--- old/src/share/vm/gc_implementation/g1/concurrentMark.cpp	2011-12-20 11:11:05.744168362 -0800
+++ new/src/share/vm/gc_implementation/g1/concurrentMark.cpp	2011-12-20 11:11:05.471883186 -0800
@@ -478,6 +478,7 @@
   _card_bm((rs.size() + CardTableModRefBS::card_size - 1) >>
            CardTableModRefBS::card_shift,
            false /* in_resource_area*/),
+
   _prevMarkBitMap(&_markBitMap1),
   _nextMarkBitMap(&_markBitMap2),
   _at_least_one_mark_complete(false),
@@ -507,7 +508,11 @@
   _total_counting_time(0.0),
   _total_rs_scrub_time(0.0),
 
-  _parallel_workers(NULL) {
+  _parallel_workers(NULL),
+
+  _count_card_bitmaps(NULL),
+  _count_marked_bytes(NULL)
+{
   CMVerboseLevel verbose_level = (CMVerboseLevel) G1MarkingVerboseLevel;
   if (verbose_level < no_verbose) {
     verbose_level = no_verbose;
@@ -541,6 +546,11 @@
   _tasks = NEW_C_HEAP_ARRAY(CMTask*, _max_task_num);
   _accum_task_vtime = NEW_C_HEAP_ARRAY(double, _max_task_num);
 
+  _count_card_bitmaps = NEW_C_HEAP_ARRAY(BitMap,  _max_task_num);
+  _count_marked_bytes = NEW_C_HEAP_ARRAY(size_t*, _max_task_num);
+
+  BitMap::idx_t card_bm_size = _card_bm.size();
+
   // so that the assertion in MarkingTaskQueue::task_queue doesn't fail
   _active_tasks = _max_task_num;
   for (int i = 0; i < (int) _max_task_num; ++i) {
@@ -548,10 +558,24 @@
     task_queue->initialize();
     _task_queues->register_queue(i, task_queue);
 
-    _tasks[i] = new CMTask(i, this, task_queue, _task_queues);
+    _count_card_bitmaps[i] = BitMap(card_bm_size, false);
+    _count_marked_bytes[i] = NEW_C_HEAP_ARRAY(size_t, max_regions);
+
+    _tasks[i] = new CMTask(i, this, 
+                           _count_marked_bytes[i],
+                           &_count_card_bitmaps[i],
+                           task_queue, _task_queues);
+
     _accum_task_vtime[i] = 0.0;
   }
 
+  // Calculate the card number for the bottom of the heap. Used
+  // in biasing indexes into the accounting card bitmaps.
+  _heap_bottom_card_num =
+    intptr_t(uintptr_t(_g1h->reserved_region().start()) >> 
+                                CardTableModRefBS::card_shift);
+
+
   if (ConcGCThreads > ParallelGCThreads) {
     vm_exit_during_initialization("Can't have more ConcGCThreads "
                                   "than ParallelGCThreads.");
@@ -675,6 +699,8 @@
   // reset all the marking data structures and any necessary flags
   clear_marking_state();
 
+  clear_all_count_data();
+
   if (verbose_low()) {
     gclog_or_tty->print_cr("[global] resetting");
   }
@@ -726,15 +752,6 @@
   clear_concurrent_marking_in_progress();
 }
 
-ConcurrentMark::~ConcurrentMark() {
-  for (int i = 0; i < (int) _max_task_num; ++i) {
-    delete _task_queues->queue(i);
-    delete _tasks[i];
-  }
-  delete _task_queues;
-  FREE_C_HEAP_ARRAY(CMTask*, _max_task_num);
-}
-
 // This closure is used to mark refs into the g1 generation
 // from external roots in the CMS bit map.
 // Called at the first checkpoint.
@@ -951,7 +968,7 @@
 }
 #endif // !PRODUCT
 
-void ConcurrentMark::grayRoot(oop p) {
+void ConcurrentMark::grayRoot(oop p, int worker_i) {
   HeapWord* addr = (HeapWord*) p;
   // We can't really check against _heap_start and _heap_end, since it
   // is possible during an evacuation pause with piggy-backed
@@ -962,7 +979,7 @@
          "address should be within the heap bounds");
 
   if (!_nextMarkBitMap->isMarked(addr)) {
-    _nextMarkBitMap->parMark(addr);
+    par_mark_and_count(p, worker_i);
   }
 }
 
@@ -1011,7 +1028,7 @@
   }
 }
 
-void ConcurrentMark::markAndGrayObjectIfNecessary(oop p) {
+void ConcurrentMark::markAndGrayObjectIfNecessary(oop p, int worker_i) {
   // The object is not marked by the caller. We need to at least mark
   // it and maybe push in on the stack.
 
@@ -1019,7 +1036,8 @@
   if (!_nextMarkBitMap->isMarked(addr)) {
     // We definitely need to mark it, irrespective whether we bail out
     // because we're done with marking.
-    if (_nextMarkBitMap->parMark(addr)) {
+
+    if (par_mark_and_count(p, worker_i)) {
       if (!concurrent_marking_in_progress() || !_should_gray_objects) {
         // If we're done with concurrent marking and we're waiting for
         // remark, then we're not pushing anything on the stack.
@@ -1221,6 +1239,10 @@
       gclog_or_tty->print_cr("\nRemark led to restart for overflow.");
     }
   } else {
+    // Aggregate the per-task counting data that we have accumulated
+    // while marking.
+    aggregate_and_clear_count_data();
+
     SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
     // We're done with marking.
     // This is the end of  the marking cycle, we're expected all
@@ -1261,45 +1283,44 @@
 
 #define CARD_BM_TEST_MODE 0
 
+// Used to calculate the # live objects per region
+// for verification purposes
 class CalcLiveObjectsClosure: public HeapRegionClosure {
 
   CMBitMapRO* _bm;
   ConcurrentMark* _cm;
-  bool _changed;
-  bool _yield;
-  size_t _words_done;
+  BitMap* _region_bm;
+  BitMap* _card_bm;
+
+  size_t _tot_words_done;
   size_t _tot_live;
   size_t _tot_used;
-  size_t _regions_done;
-  double _start_vtime_sec;
 
-  BitMap* _region_bm;
-  BitMap* _card_bm;
+  size_t _region_marked_bytes;
+
   intptr_t _bottom_card_num;
-  bool _final;
 
   void mark_card_num_range(intptr_t start_card_num, intptr_t last_card_num) {
-    for (intptr_t i = start_card_num; i <= last_card_num; i++) {
+    BitMap::idx_t start_idx = start_card_num - _bottom_card_num;
+    BitMap::idx_t last_idx = last_card_num - _bottom_card_num;
+    
+    for (BitMap::idx_t i = start_idx; i <= last_idx; i += 1) {
 #if CARD_BM_TEST_MODE
-      guarantee(_card_bm->at(i - _bottom_card_num), "Should already be set.");
+      guarantee(_card_bm->at(i), "Should already be set.");
 #else
-      _card_bm->par_at_put(i - _bottom_card_num, 1);
+      _card_bm->par_at_put(i, 1);
 #endif
     }
   }
 
 public:
-  CalcLiveObjectsClosure(bool final,
-                         CMBitMapRO *bm, ConcurrentMark *cm,
+  CalcLiveObjectsClosure(CMBitMapRO *bm, ConcurrentMark *cm,
                          BitMap* region_bm, BitMap* card_bm) :
-    _bm(bm), _cm(cm), _changed(false), _yield(true),
-    _words_done(0), _tot_live(0), _tot_used(0),
-    _region_bm(region_bm), _card_bm(card_bm),_final(final),
-    _regions_done(0), _start_vtime_sec(0.0)
+    _bm(bm), _cm(cm), _region_bm(region_bm), _card_bm(card_bm),
+    _region_marked_bytes(0), _tot_words_done(0),
+    _tot_live(0), _tot_used(0)
   {
-    _bottom_card_num =
-      intptr_t(uintptr_t(G1CollectedHeap::heap()->reserved_region().start()) >>
-               CardTableModRefBS::card_shift);
+    _bottom_card_num = cm->heap_bottom_card_num();
   }
 
   // It takes a region that's not empty (i.e., it has at least one
@@ -1316,29 +1337,16 @@
       _region_bm->par_at_put((BitMap::idx_t) index, true);
     } else {
       // Starts humongous case: calculate how many regions are part of
-      // this humongous region and then set the bit range. It might
-      // have been a bit more efficient to look at the object that
-      // spans these humongous regions to calculate their number from
-      // the object's size. However, it's a good idea to calculate
-      // this based on the metadata itself, and not the region
-      // contents, so that this code is not aware of what goes into
-      // the humongous regions (in case this changes in the future).
+      // this humongous region and then set the bit range.
       G1CollectedHeap* g1h = G1CollectedHeap::heap();
-      size_t end_index = index + 1;
-      while (end_index < g1h->n_regions()) {
-        HeapRegion* chr = g1h->region_at(end_index);
-        if (!chr->continuesHumongous()) break;
-        end_index += 1;
-      }
+      HeapRegion *last_hr = g1h->heap_region_containing_raw(hr->end() - 1);
+      size_t end_index = last_hr->hrs_index() + 1;
       _region_bm->par_at_put_range((BitMap::idx_t) index,
                                    (BitMap::idx_t) end_index, true);
     }
   }
 
   bool doHeapRegion(HeapRegion* hr) {
-    if (!_final && _regions_done == 0) {
-      _start_vtime_sec = os::elapsedVTime();
-    }
 
     if (hr->continuesHumongous()) {
       // We will ignore these here and process them when their
@@ -1352,48 +1360,40 @@
     }
 
     HeapWord* nextTop = hr->next_top_at_mark_start();
-    HeapWord* start   = hr->top_at_conc_mark_count();
-    assert(hr->bottom() <= start && start <= hr->end() &&
-           hr->bottom() <= nextTop && nextTop <= hr->end() &&
-           start <= nextTop,
-           "Preconditions.");
-    // Otherwise, record the number of word's we'll examine.
+    HeapWord* start   = hr->bottom();
+
+    assert(start <= hr->end() && start <= nextTop && nextTop <= hr->end(),
+                "Preconditions.");
+
+    // Record the number of word's we'll examine.
     size_t words_done = (nextTop - start);
+
     // Find the first marked object at or after "start".
     start = _bm->getNextMarkedWordAddress(start, nextTop);
+
     size_t marked_bytes = 0;
+    _region_marked_bytes = 0;
 
     // Below, the term "card num" means the result of shifting an address
     // by the card shift -- address 0 corresponds to card number 0.  One
     // must subtract the card num of the bottom of the heap to obtain a
     // card table index.
+
     // The first card num of the sequence of live cards currently being
     // constructed.  -1 ==> no sequence.
     intptr_t start_card_num = -1;
+
     // The last card num of the sequence of live cards currently being
     // constructed.  -1 ==> no sequence.
     intptr_t last_card_num = -1;
 
     while (start < nextTop) {
-      if (_yield && _cm->do_yield_check()) {
-        // We yielded.  It might be for a full collection, in which case
-        // all bets are off; terminate the traversal.
-        if (_cm->has_aborted()) {
-          _changed = false;
-          return true;
-        } else {
-          // Otherwise, it might be a collection pause, and the region
-          // we're looking at might be in the collection set.  We'll
-          // abandon this region.
-          return false;
-        }
-      }
       oop obj = oop(start);
       int obj_sz = obj->size();
+
       // The card num of the start of the current object.
       intptr_t obj_card_num =
         intptr_t(uintptr_t(start) >> CardTableModRefBS::card_shift);
-
       HeapWord* obj_last = start + obj_sz - 1;
       intptr_t obj_last_card_num =
         intptr_t(uintptr_t(obj_last) >> CardTableModRefBS::card_shift);
@@ -1412,108 +1412,426 @@
           }
         }
 #if CARD_BM_TEST_MODE
-        /*
-        gclog_or_tty->print_cr("Setting bits from %d/%d.",
-                               obj_card_num - _bottom_card_num,
-                               obj_last_card_num - _bottom_card_num);
-        */
         for (intptr_t j = obj_card_num; j <= obj_last_card_num; j++) {
           _card_bm->par_at_put(j - _bottom_card_num, 1);
         }
-#endif
+#endif // CARD_BM_TEST_MODE
       }
       // In any case, we set the last card num.
       last_card_num = obj_last_card_num;
 
       marked_bytes += (size_t)obj_sz * HeapWordSize;
+
       // Find the next marked object after this one.
       start = _bm->getNextMarkedWordAddress(start + 1, nextTop);
-      _changed = true;
     }
+
     // Handle the last range, if any.
     if (start_card_num != -1) {
       mark_card_num_range(start_card_num, last_card_num);
     }
-    if (_final) {
-      // Mark the allocated-since-marking portion...
-      HeapWord* tp = hr->top();
-      if (nextTop < tp) {
-        start_card_num =
-          intptr_t(uintptr_t(nextTop) >> CardTableModRefBS::card_shift);
-        last_card_num =
-          intptr_t(uintptr_t(tp) >> CardTableModRefBS::card_shift);
-        mark_card_num_range(start_card_num, last_card_num);
-        // This definitely means the region has live objects.
-        set_bit_for_region(hr);
-      }
+
+    // Mark the allocated-since-marking portion...
+    HeapWord* top = hr->top();
+    if (nextTop < top) {
+      start_card_num = intptr_t(uintptr_t(nextTop) >> CardTableModRefBS::card_shift);
+      last_card_num = intptr_t(uintptr_t(top) >> CardTableModRefBS::card_shift);
+
+      mark_card_num_range(start_card_num, last_card_num);
+      
+      // This definitely means the region has live objects.
+      set_bit_for_region(hr);
     }
 
-    hr->add_to_marked_bytes(marked_bytes);
     // Update the live region bitmap.
     if (marked_bytes > 0) {
       set_bit_for_region(hr);
     }
-    hr->set_top_at_conc_mark_count(nextTop);
+
+    // Set the marked bytes for the current region so that
+    // it can be queried by a calling verificiation routine
+    _region_marked_bytes = marked_bytes;
+
     _tot_live += hr->next_live_bytes();
     _tot_used += hr->used();
-    _words_done = words_done;
+    _tot_words_done = words_done;
 
-    if (!_final) {
-      ++_regions_done;
-      if (_regions_done % 10 == 0) {
-        double end_vtime_sec = os::elapsedVTime();
-        double elapsed_vtime_sec = end_vtime_sec - _start_vtime_sec;
-        if (elapsed_vtime_sec > (10.0 / 1000.0)) {
-          jlong sleep_time_ms =
-            (jlong) (elapsed_vtime_sec * _cm->cleanup_sleep_factor() * 1000.0);
-          os::sleep(Thread::current(), sleep_time_ms, false);
-          _start_vtime_sec = end_vtime_sec;
+    return false;
+  }
+
+  size_t region_marked_bytes() const { return _region_marked_bytes; }
+  size_t tot_words_done() const      { return _tot_words_done; }
+  size_t tot_live() const            { return _tot_live; }
+  size_t tot_used() const            { return _tot_used; }
+};
+
+// Heap region closure used for verifying the counting data
+// that was accumulated concurrently and aggregated during
+// the remark pause. This closure is applied to the heap
+// regions during the STW cleanup pause.
+
+class VerifyLiveObjectDataHRClosure: public HeapRegionClosure {
+  ConcurrentMark* _cm;
+  CalcLiveObjectsClosure _calc_cl;
+  BitMap* _region_bm;   // Region BM to be verified
+  BitMap* _card_bm;     // Card BM to be verified
+  bool _verbose;        // verbose output?
+
+  BitMap* _exp_region_bm; // Expected Region BM values
+  BitMap* _exp_card_bm;   // Expected card BM values
+
+  intptr_t _bottom_card_num; // Used for calculatint bitmap indices
+
+  int _failures;
+
+public:
+  VerifyLiveObjectDataHRClosure(ConcurrentMark* cm,
+                                BitMap* region_bm,
+                                BitMap* card_bm,
+                                BitMap* exp_region_bm,
+                                BitMap* exp_card_bm,
+                                bool verbose) :
+    _cm(cm),
+    _calc_cl(_cm->nextMarkBitMap(), _cm, exp_region_bm, exp_card_bm),
+    _region_bm(region_bm), _card_bm(card_bm), _verbose(verbose),
+    _exp_region_bm(exp_region_bm), _exp_card_bm(exp_card_bm),
+    _failures(0)
+  { 
+    _bottom_card_num = cm->heap_bottom_card_num();
+  }
+
+  int failures() const { return _failures; }
+
+  bool doHeapRegion(HeapRegion* hr) {
+    if (hr->continuesHumongous()) {
+      // We will ignore these here and process them when their
+      // associated "starts humongous" region is processed (see
+      // set_bit_for_heap_region()). Note that we cannot rely on their
+      // associated "starts humongous" region to have their bit set to
+      // 1 since, due to the region chunking in the parallel region
+      // iteration, a "continues humongous" region might be visited
+      // before its associated "starts humongous".
+      return false;
+    }
+
+    int failures = 0;
+
+    // Call the CalcLiveObjectsClosure to walk the marking bitmap for
+    // this region and set the corresponding bits in the expected region
+    // and card bitmaps.
+    bool res = _calc_cl.doHeapRegion(hr);
+    assert(res == false, "should be continuing");
+
+    // Note that the calculated count data could be a subset of the
+    // count data that was accumlated during marking. See the comment
+    // in G1ParCopyHelper::copy_to_survivor space for an explanation
+    // why.
+
+    // Verify that _top_at_conc_count == ntams
+    if (hr->top_at_conc_mark_count() != hr->next_top_at_mark_start()) {
+      if (_verbose) {
+        gclog_or_tty->print_cr("Region %d: top at conc count incorrect: expected "
+                               PTR_FORMAT", actual: "PTR_FORMAT,
+                               hr->hrs_index(), hr->next_top_at_mark_start(),
+                               hr->top_at_conc_mark_count());
+      }
+      failures += 1;
+    }
+
+    // Verify the marked bytes for this region. 
+    size_t exp_marked_bytes = _calc_cl.region_marked_bytes();
+    size_t act_marked_bytes = hr->next_marked_bytes();
+    
+    // We're not OK if expected marked bytes > actual marked bytes. It means
+    // we have missed accounting some objects during the actual marking.
+    if (exp_marked_bytes > act_marked_bytes) {
+      if (_verbose) {
+        gclog_or_tty->print_cr("Region %d: marked bytes mismatch: expected: "
+                               SIZE_FORMAT", actual: "SIZE_FORMAT,
+                               hr->hrs_index(), exp_marked_bytes, act_marked_bytes);
+      }
+      failures += 1;
+    }
+
+    // Verify the bit, for this region, in the actual and expected
+    // (which was just calculated) region bit maps.
+    // We're not OK if the expected bit is set and the actual is not set.
+    BitMap::idx_t index = (BitMap::idx_t)hr->hrs_index();
+    
+    bool expected = _exp_region_bm->at(index);
+    bool actual = _region_bm->at(index);
+    if (expected && !actual) {
+      if (_verbose) {
+        gclog_or_tty->print_cr("Region %d: region bitmap mismatch: expected: %d, actual: %d",
+                               hr->hrs_index(), expected, actual);
+      }
+      failures += 1;
+    }
+
+    // Verify that the card bit maps for the cards spanned by the current
+    // region match. The set of offsets that have set bits in the expected
+    // bitmap should be a subset of the offsets with set bits from the actual
+    // calculated card bitmap.
+    // Again it's more important that if the expected bit is set then the
+    // actual bit be set.
+    intptr_t start_card_num =
+        intptr_t(uintptr_t(hr->bottom()) >> CardTableModRefBS::card_shift);
+    intptr_t top_card_num =
+        intptr_t(uintptr_t(hr->top()) >> CardTableModRefBS::card_shift);
+
+    BitMap::idx_t start_idx = start_card_num - _bottom_card_num;
+    BitMap::idx_t end_idx = top_card_num - _bottom_card_num;
+
+    for (BitMap::idx_t i = start_idx; i < end_idx; i+=1) {
+      expected = _exp_card_bm->at(i);
+      actual = _card_bm->at(i);
+      
+      if (expected && !actual) {
+        if (_verbose) {
+          gclog_or_tty->print_cr("Region %d: card bitmap mismatch at idx %d: expected: %d, actual: %d",
+                                 hr->hrs_index(), i, expected, actual);
         }
+        failures += 1;
       }
     }
 
+    if (failures > 0 && _verbose)  {
+      gclog_or_tty->print("Region %d: bottom: "PTR_FORMAT", ntams: "
+                          PTR_FORMAT", top: "PTR_FORMAT", end: "PTR_FORMAT,
+                          hr->hrs_index(), hr->bottom(), hr->next_top_at_mark_start(),
+                          hr->top(), hr->end());
+      gclog_or_tty->print_cr(", marked_bytes: calc/actual "SIZE_FORMAT"/"SIZE_FORMAT,
+                             _calc_cl.region_marked_bytes(),
+                             hr->next_marked_bytes());
+    }
+
+    _failures += failures;
+
+    // We could stop iteration over the heap when we
+    // find the first voilating region by returning true.
     return false;
   }
+};
+
+
+class G1ParVerifyFinalCountTask: public AbstractGangTask {
+protected:
+  G1CollectedHeap* _g1h;
+  ConcurrentMark* _cm;
+  BitMap* _actual_region_bm;
+  BitMap* _actual_card_bm;
+
+  size_t _n_workers;
+
+  BitMap* _expected_region_bm;
+  BitMap* _expected_card_bm;
+
+  int  _failures;
+  bool _verbose;
+
+public:
+  G1ParVerifyFinalCountTask(G1CollectedHeap* g1h,
+                            BitMap* region_bm, BitMap* card_bm,
+                            BitMap* expected_region_bm, BitMap* expected_card_bm)
+    : AbstractGangTask("G1 verify final counting"),
+      _g1h(g1h), _cm(_g1h->concurrent_mark()),
+      _actual_region_bm(region_bm), _actual_card_bm(card_bm),
+      _expected_region_bm(expected_region_bm), _expected_card_bm(expected_card_bm),
+      _failures(0), _verbose(false),
+      _n_workers(0)
+  {
+    assert(VerifyDuringGC, "don't call this otherwise");
+
+    // Use the value already set as the number of active threads
+    // in the call to run_task().
+    if (G1CollectedHeap::use_parallel_gc_threads()) {
+      assert( _g1h->workers()->active_workers() > 0,
+        "Should have been previously set");
+      _n_workers = _g1h->workers()->active_workers();
+    } else {
+      _n_workers = 1;
+    }
+
+    assert(_expected_card_bm->size() == _actual_card_bm->size(), "sanity");
+    assert(_expected_region_bm->size() == _actual_region_bm->size(), "sanity");
+
+    _verbose = _cm->verbose_medium();
+  }
+
+  void work(int worker_i) {
+    assert((size_t) worker_i < _n_workers, "invariant");
+
+    VerifyLiveObjectDataHRClosure verify_cl(_cm,
+                                            _actual_region_bm, _actual_card_bm,
+                                            _expected_region_bm,
+                                            _expected_card_bm,
+                                            _verbose);
 
-  bool changed() { return _changed;  }
-  void reset()   { _changed = false; _words_done = 0; }
-  void no_yield() { _yield = false; }
-  size_t words_done() { return _words_done; }
-  size_t tot_live() { return _tot_live; }
-  size_t tot_used() { return _tot_used; }
+    if (G1CollectedHeap::use_parallel_gc_threads()) {
+      _g1h->heap_region_par_iterate_chunked(&verify_cl,
+                                            worker_i,
+                                            (int) _n_workers,
+                                            HeapRegion::VerifyCountClaimValue);
+    } else {
+      _g1h->heap_region_iterate(&verify_cl);
+    }
+
+    Atomic::add(verify_cl.failures(), &_failures);
+  }
+
+  int failures() const { return _failures; }
 };
 
+// Final update of count data (during cleanup).
+// Adds [top_at_count, NTAMS) to the marked bytes for each
+// region. Sets the bits in the card bitmap corresponding
+// to the interval [top_at_count, top], and sets the
+// liveness bit for each region containing live data
+// in the region bitmap.
 
-void ConcurrentMark::calcDesiredRegions() {
-  _region_bm.clear();
-  _card_bm.clear();
-  CalcLiveObjectsClosure calccl(false /*final*/,
-                                nextMarkBitMap(), this,
-                                &_region_bm, &_card_bm);
-  G1CollectedHeap *g1h = G1CollectedHeap::heap();
-  g1h->heap_region_iterate(&calccl);
+class FinalCountDataUpdateClosure: public HeapRegionClosure {
+  ConcurrentMark* _cm;
+  BitMap* _region_bm;
+  BitMap* _card_bm;
+  intptr_t _bottom_card_num;
 
-  do {
-    calccl.reset();
-    g1h->heap_region_iterate(&calccl);
-  } while (calccl.changed());
-}
+  size_t _total_live_bytes;
+  size_t _total_used_bytes;
+  size_t _total_words_done;
+
+  void mark_card_num_range(intptr_t start_card_num, intptr_t last_card_num) {
+    BitMap::idx_t start_idx = start_card_num - _bottom_card_num;
+    BitMap::idx_t last_idx = last_card_num - _bottom_card_num;
+    
+    // Inclusive bit range [start_idx, last_idx]. par_at_put_range
+    // is exclusive so we have to also set the bit for last_idx.
+    // Passing last_idx+1 to the clear_range would work in
+    // most cases but could trip an OOB assertion.
+
+    if ((last_idx - start_idx) > 0) {
+      _card_bm->par_at_put_range(start_idx, last_idx, true);
+    }
+    _card_bm->par_set_bit(last_idx);
+  }
+
+  // It takes a region that's not empty (i.e., it has at least one
+  // live object in it and sets its corresponding bit on the region
+  // bitmap to 1. If the region is "starts humongous" it will also set
+  // to 1 the bits on the region bitmap that correspond to its
+  // associated "continues humongous" regions.
+  void set_bit_for_region(HeapRegion* hr) {
+    assert(!hr->continuesHumongous(), "should have filtered those out");
+
+    size_t index = hr->hrs_index();
+    if (!hr->startsHumongous()) {
+      // Normal (non-humongous) case: just set the bit.
+      _region_bm->par_set_bit((BitMap::idx_t) index);
+    } else {
+      // Starts humongous case: calculate how many regions are part of
+      // this humongous region and then set the bit range.
+      G1CollectedHeap* g1h = G1CollectedHeap::heap();
+      HeapRegion *last_hr = g1h->heap_region_containing_raw(hr->end() - 1);
+      size_t end_index = last_hr->hrs_index() + 1;
+      _region_bm->par_at_put_range((BitMap::idx_t) index,
+                                   (BitMap::idx_t) end_index, true);
+    }
+  }
+
+ public:
+  FinalCountDataUpdateClosure(ConcurrentMark* cm,
+                              BitMap* region_bm,
+                              BitMap* card_bm) :
+    _cm(cm), _region_bm(region_bm), _card_bm(card_bm),
+    _total_words_done(0), _total_live_bytes(0), _total_used_bytes(0)
+  {
+    _bottom_card_num = cm->heap_bottom_card_num();
+  }
+
+  bool doHeapRegion(HeapRegion* hr) {
+
+    if (hr->continuesHumongous()) {
+      // We will ignore these here and process them when their
+      // associated "starts humongous" region is processed (see
+      // set_bit_for_heap_region()). Note that we cannot rely on their
+      // associated "starts humongous" region to have their bit set to
+      // 1 since, due to the region chunking in the parallel region
+      // iteration, a "continues humongous" region might be visited
+      // before its associated "starts humongous".
+      return false;
+    }
+
+    HeapWord* start = hr->top_at_conc_mark_count();
+    HeapWord* ntams = hr->next_top_at_mark_start();
+    HeapWord* top   = hr->top();
+    
+    assert(hr->bottom() <= start && start <= hr->end() &&
+           hr->bottom() <= ntams && ntams <= hr->end(), "Preconditions.");
+    
+    size_t words_done = ntams - hr->bottom();
+
+    intptr_t start_card_num = intptr_t(uintptr_t(start) >> CardTableModRefBS::card_shift);
+    intptr_t last_card_num = intptr_t(uintptr_t(top) >> CardTableModRefBS::card_shift);
+
+
+    if (start < ntams) {
+      // Region was changed between remark and cleanup pauses
+      // We need to add (ntams - start) to the marked bytes
+      // for this region, and set bits for the range
+      // [ card_num(start), card_num(ntams) ) in the
+      // card bitmap.
+      size_t live_bytes = (ntams - start) * HeapWordSize;
+      hr->add_to_marked_bytes(live_bytes);
+      
+      // Record the new top at conc count
+      hr->set_top_at_conc_mark_count(ntams);
+
+      // The setting of the bits card bitmap takes place below
+    }
+
+    // Mark the allocated-since-marking portion...
+    if (ntams < top) {
+      // This definitely means the region has live objects.
+      set_bit_for_region(hr);
+    }
+
+    // Now set the bits for [start, top]
+    mark_card_num_range(start_card_num, last_card_num);
+
+    // Set the bit for the region if it contains live data
+    if (hr->next_marked_bytes() > 0) {
+      set_bit_for_region(hr);
+    }
+
+    _total_words_done += words_done;
+    _total_used_bytes += hr->used();
+    _total_live_bytes += hr->next_marked_bytes();
+
+    return false;
+  }
+
+  size_t total_words_done() const { return _total_words_done; }
+  size_t total_live_bytes() const { return _total_live_bytes; }
+  size_t total_used_bytes() const { return _total_used_bytes; }
+};
 
 class G1ParFinalCountTask: public AbstractGangTask {
 protected:
   G1CollectedHeap* _g1h;
-  CMBitMap* _bm;
+  ConcurrentMark* _cm;
+  BitMap* _actual_region_bm;
+  BitMap* _actual_card_bm;
+  
   size_t _n_workers;
+
   size_t *_live_bytes;
   size_t *_used_bytes;
-  BitMap* _region_bm;
-  BitMap* _card_bm;
+
 public:
-  G1ParFinalCountTask(G1CollectedHeap* g1h, CMBitMap* bm,
-                      BitMap* region_bm, BitMap* card_bm)
-    : AbstractGangTask("G1 final counting"), _g1h(g1h),
-    _bm(bm), _region_bm(region_bm), _card_bm(card_bm),
-    _n_workers(0)
+  G1ParFinalCountTask(G1CollectedHeap* g1h, BitMap* region_bm, BitMap* card_bm)
+    : AbstractGangTask("G1 final counting"),
+      _g1h(g1h), _cm(_g1h->concurrent_mark()),
+      _actual_region_bm(region_bm), _actual_card_bm(card_bm),
+      _n_workers(0)
   {
     // Use the value already set as the number of active threads
     // in the call to run_task().  Needed for the allocation of
@@ -1535,30 +1853,33 @@
     FREE_C_HEAP_ARRAY(size_t, _used_bytes);
   }
 
-  void work(int i) {
-    CalcLiveObjectsClosure calccl(true /*final*/,
-                                  _bm, _g1h->concurrent_mark(),
-                                  _region_bm, _card_bm);
-    calccl.no_yield();
+  void work(int worker_i) {
+    assert((size_t) worker_i < _n_workers, "invariant");
+
+    FinalCountDataUpdateClosure final_update_cl(_cm,
+                                                _actual_region_bm,
+                                                _actual_card_bm);
+
     if (G1CollectedHeap::use_parallel_gc_threads()) {
-      _g1h->heap_region_par_iterate_chunked(&calccl, i,
+      _g1h->heap_region_par_iterate_chunked(&final_update_cl,
+                                            worker_i,
                                             (int) _n_workers,
                                             HeapRegion::FinalCountClaimValue);
     } else {
-      _g1h->heap_region_iterate(&calccl);
+      _g1h->heap_region_iterate(&final_update_cl);
     }
-    assert(calccl.complete(), "Shouldn't have yielded!");
 
-    assert((size_t) i < _n_workers, "invariant");
-    _live_bytes[i] = calccl.tot_live();
-    _used_bytes[i] = calccl.tot_used();
+    _live_bytes[worker_i] = final_update_cl.total_live_bytes();
+    _used_bytes[worker_i] = final_update_cl.total_used_bytes();
   }
+
   size_t live_bytes()  {
     size_t live_bytes = 0;
     for (size_t i = 0; i < _n_workers; ++i)
       live_bytes += _live_bytes[i];
     return live_bytes;
   }
+
   size_t used_bytes()  {
     size_t used_bytes = 0;
     for (size_t i = 0; i < _n_workers; ++i)
@@ -1766,14 +2087,17 @@
 
   HeapRegionRemSet::reset_for_cleanup_tasks();
 
+  // Clear the global region bitmap - it will be filled as part
+  // of the final counting task.
+  _region_bm.clear();
+
   size_t n_workers;
 
   // Do counting once more with the world stopped for good measure.
-  G1ParFinalCountTask g1_par_count_task(g1h, nextMarkBitMap(),
-                                        &_region_bm, &_card_bm);
+  G1ParFinalCountTask g1_par_count_task(g1h, &_region_bm, &_card_bm);
+
   if (G1CollectedHeap::use_parallel_gc_threads()) {
-    assert(g1h->check_heap_region_claim_values(
-                                               HeapRegion::InitialClaimValue),
+   assert(g1h->check_heap_region_claim_values(HeapRegion::InitialClaimValue),
            "sanity check");
 
     g1h->set_par_threads();
@@ -1784,14 +2108,42 @@
     // Done with the parallel phase so reset to 0.
     g1h->set_par_threads(0);
 
-    assert(g1h->check_heap_region_claim_values(
-                                             HeapRegion::FinalCountClaimValue),
+    assert(g1h->check_heap_region_claim_values(HeapRegion::FinalCountClaimValue),
            "sanity check");
   } else {
     n_workers = 1;
     g1_par_count_task.work(0);
   }
 
+  if (VerifyDuringGC) {
+    // Verify that the counting data accumulated during marking matches
+    // that calculated by walking the marking bitmap.
+    
+    // Bitmaps to hold expected values
+    BitMap expected_region_bm(_region_bm.size(), false);
+    BitMap expected_card_bm(_card_bm.size(), false);
+
+    G1ParVerifyFinalCountTask g1_par_verify_task(g1h,
+                                                 &_region_bm,
+                                                 &_card_bm,
+                                                 &expected_region_bm,
+                                                 &expected_card_bm);
+    
+    if (G1CollectedHeap::use_parallel_gc_threads()) {
+      g1h->set_par_threads((int)n_workers);
+      g1h->workers()->run_task(&g1_par_verify_task);
+      // Done with the parallel phase so reset to 0.
+      g1h->set_par_threads(0);
+      
+      assert(g1h->check_heap_region_claim_values(HeapRegion::VerifyCountClaimValue),
+             "sanity check");
+    } else {
+      g1_par_verify_task.work(0);
+    }
+
+    guarantee(g1_par_verify_task.failures() == 0, "Unexpected accounting failures");
+  }
+
   size_t known_garbage_bytes =
     g1_par_count_task.used_bytes() - g1_par_count_task.live_bytes();
   g1p->set_known_garbage_bytes(known_garbage_bytes);
@@ -1984,12 +2336,12 @@
 class G1CMKeepAliveClosure: public OopClosure {
   G1CollectedHeap* _g1;
   ConcurrentMark*  _cm;
-  CMBitMap*        _bitMap;
  public:
-  G1CMKeepAliveClosure(G1CollectedHeap* g1, ConcurrentMark* cm,
-                       CMBitMap* bitMap) :
-    _g1(g1), _cm(cm),
-    _bitMap(bitMap) {}
+  G1CMKeepAliveClosure(G1CollectedHeap* g1, ConcurrentMark* cm) :
+    _g1(g1), _cm(cm)
+  {
+    assert(Thread::current()->is_VM_thread(), "otherwise fix worker id");
+  }
 
   virtual void do_oop(narrowOop* p) { do_oop_work(p); }
   virtual void do_oop(      oop* p) { do_oop_work(p); }
@@ -2005,26 +2357,26 @@
     }
 
     if (_g1->is_in_g1_reserved(addr) && _g1->is_obj_ill(obj)) {
-      _bitMap->mark(addr);
+      _cm->mark_and_count(obj);
       _cm->mark_stack_push(obj);
     }
   }
 };
 
 class G1CMDrainMarkingStackClosure: public VoidClosure {
+  ConcurrentMark*               _cm;
   CMMarkStack*                  _markStack;
-  CMBitMap*                     _bitMap;
   G1CMKeepAliveClosure*         _oopClosure;
  public:
-  G1CMDrainMarkingStackClosure(CMBitMap* bitMap, CMMarkStack* markStack,
+  G1CMDrainMarkingStackClosure(ConcurrentMark* cm, CMMarkStack* markStack,
                                G1CMKeepAliveClosure* oopClosure) :
-    _bitMap(bitMap),
+    _cm(cm),
     _markStack(markStack),
     _oopClosure(oopClosure)
   {}
 
   void do_void() {
-    _markStack->drain((OopClosure*)_oopClosure, _bitMap, false);
+    _markStack->drain((OopClosure*)_oopClosure, _cm->nextMarkBitMap(), false);
   }
 };
 
@@ -2243,9 +2595,9 @@
     rp->setup_policy(clear_all_soft_refs);
     assert(_markStack.isEmpty(), "mark stack should be empty");
 
-    G1CMKeepAliveClosure g1_keep_alive(g1h, this, nextMarkBitMap());
+    G1CMKeepAliveClosure g1_keep_alive(g1h, this);
     G1CMDrainMarkingStackClosure
-      g1_drain_mark_stack(nextMarkBitMap(), &_markStack, &g1_keep_alive);
+      g1_drain_mark_stack(this, &_markStack, &g1_keep_alive);
 
     // We use the work gang from the G1CollectedHeap and we utilize all
     // the worker threads.
@@ -2623,13 +2975,13 @@
 
 public:
   void do_object(oop obj) {
-    _cm->deal_with_reference(obj);
+    _cm->deal_with_reference(obj, 0);
   }
 
   CMGlobalObjectClosure(ConcurrentMark* cm) : _cm(cm) { }
 };
 
-void ConcurrentMark::deal_with_reference(oop obj) {
+void ConcurrentMark::deal_with_reference(oop obj, int worker_i) {
   if (verbose_high()) {
     gclog_or_tty->print_cr("[global] we're dealing with reference "PTR_FORMAT,
                            (void*) obj);
@@ -2651,9 +3003,10 @@
         }
 
         // we need to mark it first
-        if (_nextMarkBitMap->parMark(objAddr)) {
+        if (par_mark_and_count(obj, hr, worker_i)) {
           // No OrderAccess:store_load() is needed. It is implicit in the
-          // CAS done in parMark(objAddr) above
+          // CAS done in the call to CMBitMap::parMark() in the above
+          // routine.
           HeapWord* finger = _finger;
           if (objAddr < finger) {
             if (verbose_high()) {
@@ -2698,7 +3051,7 @@
   ((CMBitMap*)_prevMarkBitMap)->mark((HeapWord*)p);
 }
 
-void ConcurrentMark::clear(oop p) {
+void ConcurrentMark::clear_mark(oop p) {
   assert(p != NULL && p->is_oop(), "expected an oop");
   HeapWord* addr = (HeapWord*)p;
   assert(addr >= _nextMarkBitMap->startWord() ||
@@ -2898,6 +3251,245 @@
   }
 }
 
+// Aggregate the counting data that was constructed concurrently
+// with marking.
+class AggregateCountDataHRClosure: public HeapRegionClosure {
+  ConcurrentMark* _cm;
+  BitMap* _cm_card_bm;
+  intptr_t _bottom_card_num;
+  size_t _max_task_num;
+
+ public:
+  AggregateCountDataHRClosure(ConcurrentMark *cm,
+                          BitMap* cm_card_bm,
+                          intptr_t bottom_card_num,
+                          size_t max_task_num) :
+    _cm(cm),
+    _cm_card_bm(cm_card_bm),
+    _bottom_card_num(bottom_card_num),
+    _max_task_num(max_task_num)
+  { }
+
+  bool is_card_aligned(HeapWord* p) {
+    return ((uintptr_t(p) & (CardTableModRefBS::card_size - 1)) == 0);
+  }
+
+  bool doHeapRegion(HeapRegion* hr) {
+    if (hr->continuesHumongous()) {
+      // We will ignore these here and process them when their
+      // associated "starts humongous" region is processed.
+      // Note that we cannot rely on their associated
+      // "starts humongous" region to have their bit set to 1
+      // since, due to the region chunking in the parallel region
+      // iteration, a "continues humongous" region might be visited
+      // before its associated "starts humongous".
+      return false;
+    }
+
+    HeapWord* start = hr->bottom();
+    HeapWord* limit = hr->next_top_at_mark_start();
+    HeapWord* end = hr->end();
+    
+    assert(start <= limit && limit <= hr->top() && 
+           hr->top() <= hr->end(), "Preconditions");
+
+    assert(hr->next_marked_bytes() == 0, "Precondition");
+
+    if (start == limit) {
+      // NTAMS of this region has not been set so nothing to do.
+      return false;
+    }
+
+    intptr_t start_card_num = intptr_t(uintptr_t(start) >> CardTableModRefBS::card_shift);
+    intptr_t limit_card_num = intptr_t(uintptr_t(limit) >> CardTableModRefBS::card_shift);
+    intptr_t end_card_num   = intptr_t(uintptr_t(end) >> CardTableModRefBS::card_shift);
+
+    assert(is_card_aligned(start), "sanity");
+    assert(is_card_aligned(end), "sanity");
+   
+    // If ntams is not card aligned then we bump the index for
+    // limit so that we get the card spanning ntams.
+    if (!is_card_aligned(limit)) {
+      limit_card_num += 1;
+    }
+
+    assert(limit_card_num <= end_card_num, "or else use atomics");
+
+    BitMap::idx_t start_idx = start_card_num - _bottom_card_num;
+    BitMap::idx_t limit_idx = limit_card_num - _bottom_card_num;
+
+    // Aggregate the "stripe" in the count data associated with hr.
+    size_t hrs_index = hr->hrs_index();
+    size_t marked_bytes = 0;
+
+    for (int i = 0; (size_t)i < _max_task_num; i += 1) {
+      size_t* marked_bytes_array = _cm->count_marked_bytes_array_for(i);
+      BitMap* task_card_bm = _cm->count_card_bitmap_for(i);
+  
+      // Fetch the marked_bytes in this region for task i and
+      // add it to the running total for this region.
+      marked_bytes += marked_bytes_array[hrs_index];
+
+      // Now clear the value in the task's marked bytes array
+      // for this region.
+      marked_bytes_array[hrs_index] = 0;
+  
+      // Now union the bitmaps[0,max_task_num)[start_idx..limit_idx)
+      // into the global card bitmap.
+      BitMap::idx_t scan_idx = task_card_bm->get_next_one_offset(start_idx, limit_idx);
+
+      while (scan_idx < limit_idx) {
+        assert(task_card_bm->at(scan_idx) == true, "should be");
+        _cm_card_bm->set_bit(scan_idx);
+        task_card_bm->clear_bit(scan_idx);
+        assert(_cm_card_bm->at(scan_idx) == true, "should be");
+        scan_idx = task_card_bm->get_next_one_offset(start_idx + 1, limit_idx);
+      }
+    }
+
+    // Update the marked bytes for this region.
+    hr->add_to_marked_bytes(marked_bytes);
+  
+    // Now set the top at count to NTAMS.
+    hr->set_top_at_conc_mark_count(limit);
+
+    // Next heap region
+    return false;
+  }
+};
+
+class G1AggregateCountDataTask: public AbstractGangTask {
+protected:
+  G1CollectedHeap* _g1h;
+  ConcurrentMark* _cm;
+  BitMap* _cm_card_bm;
+  intptr_t _heap_bottom_card_num;
+  size_t _max_task_num;
+  int _active_workers;
+
+public:
+  G1AggregateCountDataTask(G1CollectedHeap* g1h,
+                           ConcurrentMark* cm,
+                           BitMap* cm_card_bm,
+                           intptr_t bottom_card_num,
+                           size_t max_task_num,
+                           int n_workers) :
+    AbstractGangTask("Count Aggregation"),
+    _g1h(g1h), _cm(cm), _cm_card_bm(cm_card_bm),
+    _heap_bottom_card_num(bottom_card_num),
+    _max_task_num(max_task_num),
+    _active_workers(n_workers)
+  { }
+
+  void work(int worker_i) {
+    AggregateCountDataHRClosure cl(_cm, _cm_card_bm,
+                                 _heap_bottom_card_num, _max_task_num);
+
+    if (G1CollectedHeap::use_parallel_gc_threads()) {
+      _g1h->heap_region_par_iterate_chunked(&cl, worker_i,
+                                            _active_workers,
+                                            HeapRegion::AggregateCountClaimValue);
+    } else {
+      _g1h->heap_region_iterate(&cl);
+    }
+  }
+};
+
+
+void ConcurrentMark::aggregate_and_clear_count_data() {
+  // Clear the global card bitmap
+  _card_bm.clear();
+
+  int n_workers = (G1CollectedHeap::use_parallel_gc_threads() ?
+                        _g1h->workers()->active_workers() :
+                        1);
+
+  G1AggregateCountDataTask g1_par_agg_task(_g1h, this, &_card_bm,
+                                           _heap_bottom_card_num, _max_task_num,
+                                           n_workers);
+
+  if (G1CollectedHeap::use_parallel_gc_threads()) {
+    assert(_g1h->check_heap_region_claim_values(HeapRegion::InitialClaimValue),
+           "sanity check");
+    _g1h->set_par_threads(n_workers);
+    _g1h->workers()->run_task(&g1_par_agg_task);
+    _g1h->set_par_threads(0);
+
+    assert(_g1h->check_heap_region_claim_values(HeapRegion::AggregateCountClaimValue),
+           "sanity check");
+    _g1h->reset_heap_region_claim_values();
+  } else {
+    g1_par_agg_task.work(0);
+  }
+}
+
+// Clear the per-worker arrays used to store the per-region counting data
+void ConcurrentMark::clear_all_count_data() {
+  assert(SafepointSynchronize::is_at_safepoint() ||
+         !Universe::is_fully_initialized(), "must be");
+
+  size_t max_regions = _g1h->max_regions();
+  
+  assert(_max_task_num != 0, "unitialized");
+  assert(_count_card_bitmaps != NULL, "uninitialized");
+  assert(_count_marked_bytes != NULL, "uninitialized");
+
+  for (int i = 0; (size_t) i < _max_task_num; i += 1) {
+    BitMap* task_card_bm = count_card_bitmap_for(i);
+    size_t* marked_bytes_array = count_marked_bytes_array_for(i);
+
+    assert(task_card_bm->size() == _card_bm.size(), "size mismatch");
+    assert(marked_bytes_array != NULL, "uninitialized");
+
+    for (int j = 0; (size_t) j < max_regions; j++) {
+      marked_bytes_array[j] = 0;
+    }
+    task_card_bm->clear();
+  }
+}
+
+void ConcurrentMark::clear_count_data_for_heap_region(HeapRegion* hr) {
+  // Clears the count data for the given region from _all_ of
+  // the per-task counting data structures.
+
+  MemRegion used_region = hr->used_region();
+  HeapWord* start = used_region.start();
+  HeapWord* last = used_region.last();
+  size_t hr_index = hr->hrs_index();
+
+  intptr_t start_card_num =
+    intptr_t(uintptr_t(start) >> CardTableModRefBS::card_shift);
+  intptr_t last_card_num =
+    intptr_t(uintptr_t(last) >> CardTableModRefBS::card_shift);
+  
+  BitMap::idx_t start_idx = start_card_num - heap_bottom_card_num();
+  BitMap::idx_t last_idx = last_card_num - heap_bottom_card_num();
+
+  size_t used_region_bytes = used_region.byte_size();
+  size_t marked_bytes = 0;
+
+  for (int i=0; (size_t)i < _max_task_num; i += 1) {
+    BitMap* task_card_bm = count_card_bitmap_for(i);
+    size_t* marked_bytes_array = count_marked_bytes_array_for(i);
+
+    marked_bytes += marked_bytes_array[hr_index];
+    // clear the amount of marked bytes in the task array for this
+    // region
+    marked_bytes_array[hr_index] = 0;
+    
+    // Clear the inclusive range [start_idx, last_idx] from the
+    // card bitmap. The clear_range routine is exclusive so we
+    // need to also explicitly clear the bit at last_idx.
+    // Passing last_idx+1 to the clear_range would work in
+    // most cases but could trip an OOB assertion.
+
+    if ((last_idx - start_idx) > 0) {
+      task_card_bm->clear_range(start_idx, last_idx);
+    }
+    task_card_bm->clear_bit(last_idx);
+  }
+}
+
 void ConcurrentMark::print_stats() {
   if (verbose_stats()) {
     gclog_or_tty->print_cr("---------------------------------------------------------------------");
@@ -2914,7 +3506,6 @@
   friend class CSetMarkBitMapClosure;
 
   G1CollectedHeap* _g1h;
-  CMBitMap*        _bm;
   ConcurrentMark*  _cm;
   oop*             _ms;
   jint*            _array_ind_stack;
@@ -2974,7 +3565,6 @@
   CSetMarkOopClosure(ConcurrentMark* cm, int ms_size, int worker_i) :
     _g1h(G1CollectedHeap::heap()),
     _cm(cm),
-    _bm(cm->nextMarkBitMap()),
     _ms_size(ms_size), _ms_ind(0),
     _ms(NEW_C_HEAP_ARRAY(oop, ms_size)),
     _array_ind_stack(NEW_C_HEAP_ARRAY(jint, ms_size)),
@@ -3004,7 +3594,7 @@
     if (hr != NULL) {
       if (hr->in_collection_set()) {
         if (_g1h->is_obj_ill(obj)) {
-          if (_bm->parMark((HeapWord*)obj)) {
+          if (_cm->par_mark_and_count(obj, hr, _worker_i)) {
             if (!push(obj)) {
               gclog_or_tty->print_cr("Setting abort in CSetMarkOopClosure because push failed.");
               set_abort();
@@ -3013,7 +3603,7 @@
         }
       } else {
         // Outside the collection set; we need to gray it
-        _cm->deal_with_reference(obj);
+        _cm->deal_with_reference(obj, _worker_i);
       }
     }
   }
@@ -3289,10 +3879,9 @@
                          (_init_times.sum() + _remark_times.sum() +
                           _cleanup_times.sum())/1000.0);
   gclog_or_tty->print_cr("  Total concurrent time = %8.2f s "
-                "(%8.2f s marking, %8.2f s counting).",
+                "(%8.2f s marking).",
                 cmThread()->vtime_accum(),
-                cmThread()->vtime_mark_accum(),
-                cmThread()->vtime_count_accum());
+                cmThread()->vtime_mark_accum());
 }
 
 void ConcurrentMark::print_worker_threads_on(outputStream* st) const {
@@ -4587,6 +5176,8 @@
 
 CMTask::CMTask(int task_id,
                ConcurrentMark* cm,
+               size_t* marked_bytes,
+               BitMap* card_bm,
                CMTaskQueue* task_queue,
                CMTaskQueueSet* task_queues)
   : _g1h(G1CollectedHeap::heap()),
@@ -4596,7 +5187,9 @@
     _task_queue(task_queue),
     _task_queues(task_queues),
     _cm_oop_closure(NULL),
-    _aborted_region(MemRegion()) {
+    _aborted_region(MemRegion()),
+    _marked_bytes_array(marked_bytes),
+    _card_bm(card_bm) {
   guarantee(task_queue != NULL, "invariant");
   guarantee(task_queues != NULL, "invariant");
 
--- old/src/share/vm/gc_implementation/g1/concurrentMark.hpp	2011-12-20 11:11:08.275878533 -0800
+++ new/src/share/vm/gc_implementation/g1/concurrentMark.hpp	2011-12-20 11:11:08.021102893 -0800
@@ -430,7 +430,6 @@
   WorkGangBarrierSync     _first_overflow_barrier_sync;
   WorkGangBarrierSync     _second_overflow_barrier_sync;
 
-
   // this is set by any task, when an overflow on the global data
   // structures is detected.
   volatile bool           _has_overflown;
@@ -582,6 +581,27 @@
     }
   }
 
+  // Live Data Counting data structures...
+  // These data structures are initialized at the start of
+  // marking. They are written to while marking is active.
+  // They are aggregated during remark; the aggregated values
+  // are then used to populate the _region_bm, _card_bm, and
+  // the total live bytes, which are then subsequently updated
+  // during cleanup.
+
+  // An array of bitmaps (one bit map per task). Each bitmap
+  // is used to record the cards spanned by the live objects
+  // marked by that task/worker.
+  BitMap*  _count_card_bitmaps;
+
+  // Used to record the number of marked live bytes
+  // (for each region, by worker thread).
+  size_t** _count_marked_bytes;
+
+  // Card index of the bottom of the G1 heap. Used for biasing indices into
+  // the card bitmaps.
+  intptr_t _heap_bottom_card_num;
+
 public:
   // Manipulation of the global mark stack.
   // Notice that the first mark_stack_push is CAS-based, whereas the
@@ -703,10 +723,10 @@
 
   // It grays an object by first marking it. Then, if it's behind the
   // global finger, it also pushes it on the global stack.
-  void deal_with_reference(oop obj);
+  void deal_with_reference(oop obj, int worker_i);
 
   ConcurrentMark(ReservedSpace rs, int max_regions);
-  ~ConcurrentMark();
+
   ConcurrentMarkThread* cmThread() { return _cmThread; }
 
   CMBitMapRO* prevMarkBitMap() const { return _prevMarkBitMap; }
@@ -727,7 +747,8 @@
   // grayed and it's MT-safe. Currently, we just mark it. But, in the
   // future, we can experiment with pushing it on the stack and we can
   // do this without changing G1CollectedHeap.
-  void grayRoot(oop p);
+  void grayRoot(oop p, int worker_i);
+
   // It's used during evacuation pauses to gray a region, if
   // necessary, and it's MT-safe. It assumes that the caller has
   // marked any objects on that region. If _should_gray_objects is
@@ -735,13 +756,14 @@
   // pushed on the region stack, if it is located below the global
   // finger, otherwise we do nothing.
   void grayRegionIfNecessary(MemRegion mr);
+
   // It's used during evacuation pauses to mark and, if necessary,
   // gray a single object and it's MT-safe. It assumes the caller did
   // not mark the object. If _should_gray_objects is true and we're
   // still doing concurrent marking, the objects is pushed on the
   // global stack, if it is located below the global finger, otherwise
   // we do nothing.
-  void markAndGrayObjectIfNecessary(oop p);
+  void markAndGrayObjectIfNecessary(oop p, int worker_i);
 
   // It iterates over the heap and for each object it comes across it
   // will dump the contents of its reference fields, as well as
@@ -785,14 +807,16 @@
 
   void checkpointRootsFinal(bool clear_all_soft_refs);
   void checkpointRootsFinalWork();
-  void calcDesiredRegions();
   void cleanup();
   void completeCleanup();
 
   // Mark in the previous bitmap.  NB: this is usually read-only, so use
   // this carefully!
   void markPrev(oop p);
-  void clear(oop p);
+
+  // Clears the mark in the next bitmap for the given object.
+  void clear_mark(oop p);
+
   // Clears marks for all objects in the given range, for both prev and
   // next bitmaps.  NB: the previous bitmap is usually read-only, so use
   // this carefully!
@@ -901,6 +925,98 @@
   bool verbose_high() {
     return _MARKING_VERBOSE_ && _verbose_level >= high_verbose;
   }
+
+  // Counting data structure accessors
+
+  // Returns the card number of the bottom of the G1 heap.
+  // Used in biasing indices into accounting card bitmaps.
+  intptr_t heap_bottom_card_num() const {
+    return _heap_bottom_card_num;
+  }
+
+  // Returns the card bitmap for a given task or worker id.
+  BitMap* count_card_bitmap_for(int worker_i) {
+    assert(0 <= worker_i && (size_t) worker_i < _max_task_num, "oob");
+    assert(_count_card_bitmaps != NULL, "uninitialized");
+    BitMap* task_card_bm = &_count_card_bitmaps[worker_i];
+    assert(task_card_bm->size() == _card_bm.size(), "size mismatch");
+    return task_card_bm;
+  }
+
+  // Returns the array containing the marked bytes for each region,
+  // for the given worker or task id.
+  size_t* count_marked_bytes_array_for(int worker_i) {
+    assert(0 <= worker_i && (size_t) worker_i < _max_task_num, "oob");
+    assert(_count_marked_bytes != NULL, "uninitialized");
+    size_t* marked_bytes_array = _count_marked_bytes[worker_i];
+    assert(marked_bytes_array != NULL, "uninitialized");
+    return marked_bytes_array;
+  }
+
+  // Counts the size of the given memory region in the the given
+  // marked_bytes array slot for the given HeapRegion.
+  // Sets the bits in the given card bitmap that are associated with the
+  // cards that are spanned by the memory region.
+  inline void count_region(MemRegion mr, HeapRegion* hr,
+                           size_t* marked_bytes_array,
+                           BitMap* task_card_bm);
+  
+  // Counts the given memory region in the ask/worker counting
+  // data structures for the given worker id.
+  inline void count_region(MemRegion mr, int worker_i);
+  
+  // Counts the given object in the given task/worker counting
+  // data structures.
+  inline void count_object(oop obj, HeapRegion* hr,
+                           size_t* marked_bytes_array,
+                           BitMap* task_card_bm);
+  
+  // Counts the given object in the task/worker counting data
+  // structures for the given worker id.
+  inline void count_object(oop obj, HeapRegion* hr, int worker_i);
+  
+  // Attempts to mark the given object and, if successful, counts
+  // the object in the given task/worker counting structures.
+  inline bool par_mark_and_count(oop obj, HeapRegion* hr,
+                                 size_t* marked_bytes_array,
+                                 BitMap* task_card_bm);
+
+  // Attempts to mark the given object and, if successful, counts
+  // the object in the task/worker counting structures for the
+  // given worker id.
+  inline bool par_mark_and_count(oop obj, HeapRegion* hr, int worker_i);
+
+  // Similar to the above routine but we don't know the heap region that
+  // contains the object to be marked/counted, which this routine looks up.
+  inline bool par_mark_and_count(oop obj, int worker_i);
+
+  // Unconditionally mark the given object, and unconditinally count
+  // the object in the counting structures for worker id 0.
+  // Should *not* be called from parallel code.
+  inline bool mark_and_count(oop obj, HeapRegion* hr);
+ 
+  // Similar to the above routine but we don't know the heap region that
+  // contains the object to be marked/counted, which this routine looks up.
+  // Should *not* be called from parallel code.
+  inline bool mark_and_count(oop obj);
+
+  // Clears the count data for the given region from _all_ of
+  // the per-task counting data structures.
+  void clear_count_data_for_heap_region(HeapRegion* hr);
+
+protected:
+  // Clear all the per-task bitmaps and arrays used to store the
+  // counting data.
+  void clear_all_count_data();
+
+  // Aggregates the counting data for each worker/task
+  // that was constructed while marking. Also sets
+  // the amount of marked bytes for each region and
+  // the top at concurrent mark count.
+  void aggregate_and_clear_count_data();
+
+  // Verification routine
+  void verify_count_data();
 };
 
 // A class representing a marking task.
@@ -1019,6 +1135,12 @@
 
   TruncatedSeq                _marking_step_diffs_ms;
 
+  // Counting data structures. Embedding the task's marked_bytes_array
+  // and card bitmap into the actual task saves having to go through
+  // the ConcurrentMark object.
+  size_t*                     _marked_bytes_array;
+  BitMap*                     _card_bm;
+
   // LOTS of statistics related with this task
 #if _MARKING_STATS_
   NumberSeq                   _all_clock_intervals_ms;
@@ -1183,6 +1305,7 @@
   }
 
   CMTask(int task_num, ConcurrentMark *cm,
+         size_t* marked_bytes, BitMap* card_bm,
          CMTaskQueue* task_queue, CMTaskQueueSet* task_queues);
 
   // it prints statistics associated with this task
--- old/src/share/vm/gc_implementation/g1/concurrentMark.inline.hpp	2011-12-20 11:11:09.808248531 -0800
+++ new/src/share/vm/gc_implementation/g1/concurrentMark.inline.hpp	2011-12-20 11:11:09.564464612 -0800
@@ -28,6 +28,129 @@
 #include "gc_implementation/g1/concurrentMark.hpp"
 #include "gc_implementation/g1/g1CollectedHeap.inline.hpp"
 
+// Counts the given memory region in the given task/worker
+// counting data structures.
+inline void ConcurrentMark::count_region(MemRegion mr, HeapRegion* hr,
+                                         size_t* marked_bytes_array,
+                                         BitMap* task_card_bm) {
+  G1CollectedHeap* g1h = _g1h;
+  HeapWord* start = mr.start();
+  HeapWord* last = mr.last();
+  size_t region_size = mr.byte_size();
+  size_t index = hr->hrs_index();
+
+  assert(!hr->continuesHumongous(), "should not be HC region");
+  assert(hr == g1h->heap_region_containing(start), "sanity");
+  assert(hr == g1h->heap_region_containing(mr.last()), "sanity");
+  assert(marked_bytes_array != NULL, "pre-condition");
+  assert(task_card_bm != NULL, "pre-condition");
+
+  // Add to the task local marked bytes for this region.
+  marked_bytes_array[index] += region_size;
+
+  // Below, the term "card num" means the result of shifting an address
+  // by the card shift -- address 0 corresponds to card number 0.  One
+  // must subtract the card num of the bottom of the heap to obtain a
+  // card table index.
+
+  intptr_t start_card_num = intptr_t(uintptr_t(start) >> CardTableModRefBS::card_shift);
+  intptr_t last_card_num  = intptr_t(uintptr_t(last) >> CardTableModRefBS::card_shift);
+
+  BitMap::idx_t start_idx = start_card_num - heap_bottom_card_num();
+  BitMap::idx_t last_idx = last_card_num - heap_bottom_card_num();
+  
+  // The card bitmap is task/worker specific => no need to use 'par' routines.
+  // Set bits in the inclusive bit range [start_idx, last_idx].
+  for (BitMap::idx_t i = start_idx; i <= last_idx; i += 1) {
+    task_card_bm->set_bit(i);
+  }
+}
+
+// Counts the given memory region in the ask/worker counting
+// data structures for the given worker id.
+inline void ConcurrentMark::count_region(MemRegion mr, int worker_i) {
+  size_t* marked_bytes_array = count_marked_bytes_array_for(worker_i);
+  BitMap* task_card_bm = count_card_bitmap_for(worker_i);
+  HeapWord* addr = mr.start();
+  HeapRegion* hr = _g1h->heap_region_containing(addr);
+  count_region(mr, hr, marked_bytes_array, task_card_bm);
+}
+
+// Counts the given object in the given task/worker counting data structures.
+inline void ConcurrentMark::count_object(oop obj,
+                                         HeapRegion* hr,
+                                         size_t* marked_bytes_array,
+                                         BitMap* task_card_bm) {
+  MemRegion mr((HeapWord*)obj, obj->size());
+  count_region(mr, hr, marked_bytes_array, task_card_bm);
+}
+
+// Counts the given object in the task/worker counting data
+// structures for the given worker id.
+inline void ConcurrentMark::count_object(oop obj, HeapRegion* hr, int worker_i) {
+  size_t* marked_bytes_array = count_marked_bytes_array_for(worker_i);
+  BitMap* task_card_bm = count_card_bitmap_for(worker_i);
+  HeapWord* addr = (HeapWord*) obj;
+  count_object(obj, hr, marked_bytes_array, task_card_bm);
+}
+
+// Attempts to mark the given object and, if successful, counts
+// the object in the given task/worker counting structures.
+inline bool ConcurrentMark::par_mark_and_count(oop obj,
+                                               HeapRegion* hr,
+                                               size_t* marked_bytes_array,
+                                               BitMap* task_card_bm) {
+  HeapWord* addr = (HeapWord*)obj;
+  if (_nextMarkBitMap->parMark(addr)) {
+    // Update the task specific count data for the object.
+    count_object(obj, hr, marked_bytes_array, task_card_bm);
+    return true;
+  }
+  return false;
+}
+
+// Attempts to mark the given object and, if successful, counts
+// the object in the task/worker counting structures for the
+// given worker id.
+inline bool ConcurrentMark::par_mark_and_count(oop obj,
+                                               HeapRegion* hr,
+                                               int worker_i) {
+  HeapWord* addr = (HeapWord*)obj;
+  if (_nextMarkBitMap->parMark(addr)) {
+    // Update the task specific count data for the object.
+    count_object(obj, hr, worker_i);
+    return true;
+  }
+  return false;
+}
+
+// As above - but we don't know the heap region containing the
+// object and so have to supply it.
+inline bool ConcurrentMark::par_mark_and_count(oop obj, int worker_i) {
+  HeapWord* addr = (HeapWord*)obj;
+  HeapRegion* hr = _g1h->heap_region_containing(addr);
+  return par_mark_and_count(obj, hr, worker_i);
+}
+
+// Unconditionally mark the given object, and unconditinally count
+// the object in the counting structures for worker id 0.
+// Should *not* be called from parallel code.
+inline bool ConcurrentMark::mark_and_count(oop obj, HeapRegion* hr) {
+  HeapWord* addr = (HeapWord*)obj;
+  _nextMarkBitMap->mark(addr);
+  // Update the task specific count data for the object.
+  count_object(obj, hr, 0 /* worker_i */);
+  return true;
+}
+
+// As above - but we don't have the heap region containing the
+// object, so we have to supply it.
+inline bool ConcurrentMark::mark_and_count(oop obj) {
+  HeapWord* addr = (HeapWord*)obj;
+  HeapRegion* hr = _g1h->heap_region_containing(addr);
+  return mark_and_count(obj, hr);
+}
+
 inline void CMTask::push(oop obj) {
   HeapWord* objAddr = (HeapWord*) obj;
   assert(_g1h->is_in_g1_reserved(objAddr), "invariant");
@@ -84,7 +207,7 @@
 
   HeapWord* objAddr = (HeapWord*) obj;
   assert(obj->is_oop_or_null(true /* ignore mark word */), "Error");
- if (_g1h->is_in_g1_reserved(objAddr)) {
+  if (_g1h->is_in_g1_reserved(objAddr)) {
     assert(obj != NULL, "null check is implicit");
     if (!_nextMarkBitMap->isMarked(objAddr)) {
       // Only get the containing region if the object is not marked on the
@@ -98,9 +221,9 @@
         }
 
         // we need to mark it first
-        if (_nextMarkBitMap->parMark(objAddr)) {
+        if (_cm->par_mark_and_count(obj, hr, _marked_bytes_array, _card_bm)) {
           // No OrderAccess:store_load() is needed. It is implicit in the
-          // CAS done in parMark(objAddr) above
+          // CAS done in CMBitMap::parMark() call in the routine above.
           HeapWord* global_finger = _cm->finger();
 
 #if _CHECK_BOTH_FINGERS_
--- old/src/share/vm/gc_implementation/g1/concurrentMarkThread.cpp	2011-12-20 11:11:11.315188479 -0800
+++ new/src/share/vm/gc_implementation/g1/concurrentMarkThread.cpp	2011-12-20 11:11:11.049987891 -0800
@@ -44,8 +44,7 @@
   _started(false),
   _in_progress(false),
   _vtime_accum(0.0),
-  _vtime_mark_accum(0.0),
-  _vtime_count_accum(0.0)
+  _vtime_mark_accum(0.0)
 {
   create_and_start();
 }
@@ -148,36 +147,12 @@
         }
       } while (cm()->restart_for_overflow());
 
-      double counting_start_time = os::elapsedVTime();
-      if (!cm()->has_aborted()) {
-        double count_start_sec = os::elapsedTime();
-        if (PrintGC) {
-          gclog_or_tty->date_stamp(PrintGCDateStamps);
-          gclog_or_tty->stamp(PrintGCTimeStamps);
-          gclog_or_tty->print_cr("[GC concurrent-count-start]");
-        }
-
-        _sts.join();
-        _cm->calcDesiredRegions();
-        _sts.leave();
-
-        if (!cm()->has_aborted()) {
-          double count_end_sec = os::elapsedTime();
-          if (PrintGC) {
-            gclog_or_tty->date_stamp(PrintGCDateStamps);
-            gclog_or_tty->stamp(PrintGCTimeStamps);
-            gclog_or_tty->print_cr("[GC concurrent-count-end, %1.7lf]",
-                                   count_end_sec - count_start_sec);
-          }
-        }
-      }
-
       double end_time = os::elapsedVTime();
-      _vtime_count_accum += (end_time - counting_start_time);
       // Update the total virtual time before doing this, since it will try
       // to measure it to get the vtime for this marking.  We purposely
       // neglect the presumably-short "completeCleanup" phase here.
       _vtime_accum = (end_time - _vtime_start);
+
       if (!cm()->has_aborted()) {
         if (g1_policy->adaptive_young_list_length()) {
           double now = os::elapsedTime();
--- old/src/share/vm/gc_implementation/g1/concurrentMarkThread.hpp	2011-12-20 11:11:12.761433765 -0800
+++ new/src/share/vm/gc_implementation/g1/concurrentMarkThread.hpp	2011-12-20 11:11:12.531569554 -0800
@@ -40,7 +40,6 @@
   double _vtime_accum;  // Accumulated virtual time.
 
   double _vtime_mark_accum;
-  double _vtime_count_accum;
 
  public:
   virtual void run();
@@ -69,8 +68,6 @@
   double vtime_accum();
   // Marking virtual time so far
   double vtime_mark_accum();
-  // Counting virtual time so far.
-  double vtime_count_accum() { return _vtime_count_accum; }
 
   ConcurrentMark* cm()     { return _cm; }
 
--- old/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp	2011-12-20 11:11:14.043440125 -0800
+++ new/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp	2011-12-20 11:11:13.806508438 -0800
@@ -4054,7 +4054,7 @@
       assert(_cm->isPrevMarked(obj), "Should be marked!");
       _prev_marked_bytes += (obj_size * HeapWordSize);
       if (_g1->mark_in_progress() && !_g1->is_obj_ill(obj)) {
-        _cm->markAndGrayObjectIfNecessary(obj);
+        _cm->markAndGrayObjectIfNecessary(obj, 0 /* worker_i */);
       }
       obj->set_mark(markOopDesc::prototype());
       // While we were processing RSet buffers during the
@@ -4173,7 +4173,8 @@
 oop
 G1CollectedHeap::handle_evacuation_failure_par(OopsInHeapRegionClosure* cl,
                                                oop old,
-                                               bool should_mark_root) {
+                                               bool should_mark_root,
+                                               int worker_i) {
   assert(obj_in_cs(old),
          err_msg("obj: "PTR_FORMAT" should still be in the CSet",
                  (HeapWord*) old));
@@ -4188,7 +4189,7 @@
     // object is also responsible for marking the object.
     if (should_mark_root) {
       assert(!oopDesc::is_null(old), "shouldn't be");
-      _cm->grayRoot(old);
+      _cm->grayRoot(old, worker_i);
     }
 
     if (_evac_failure_closure != cl) {
@@ -4294,10 +4295,45 @@
 }
 #endif // PRODUCT
 
-G1ParGCAllocBuffer::G1ParGCAllocBuffer(size_t gclab_word_size) :
+void GCLabBitMap::retire(int worker_i) {
+  guarantee(use_local_bitmaps, "invariant");
+  assert(fields_well_formed(), "invariant");
+
+  if (_start_word != NULL) {
+    CMBitMap* mark_bitmap = _cm->nextMarkBitMap();
+
+    // this means that the bitmap was set up for the GCLab
+    assert(_real_start_word != NULL && _real_end_word != NULL, "invariant");
+
+    mark_bitmap->mostly_disjoint_range_union(this,
+                              0, // always start from the start of the bitmap
+                              _start_word,
+                              gclab_real_word_size());
+
+    // Note: Even though that not all objects copied into the LAB will
+    // have their bit set in the LAB bitmap (the LAB bitmap is used to
+    // propagate marks), we can just add the entire lab and its bitmap
+    // to the count of the marked data. It's OK (but inaccurate) to
+    // count a dead object but we can't miss counting a live object.
+    MemRegion lab_region(_real_start_word, _real_end_word);
+    _cm->count_region(lab_region, worker_i);
+    _cm->grayRegionIfNecessary(lab_region);
+
+#ifndef PRODUCT
+    if (use_local_bitmaps && verify_local_bitmaps) {
+      verify();
+    }
+#endif // PRODUCT
+  } else {
+    assert(_real_start_word == NULL && _real_end_word == NULL, "invariant");
+  }
+}
+
+G1ParGCAllocBuffer::G1ParGCAllocBuffer(size_t gclab_word_size, int worker_i) :
   ParGCAllocBuffer(gclab_word_size),
   _should_mark_objects(false),
   _bitmap(G1CollectedHeap::heap()->reserved_region().start(), gclab_word_size),
+  _worker_i(worker_i),
   _retired(false)
 {
   //_should_mark_objects is set to true when G1ParCopyHelper needs to
@@ -4319,8 +4355,8 @@
     _g1_rem(g1h->g1_rem_set()),
     _hash_seed(17), _queue_num(queue_num),
     _term_attempts(0),
-    _surviving_alloc_buffer(g1h->desired_plab_sz(GCAllocForSurvived)),
-    _tenured_alloc_buffer(g1h->desired_plab_sz(GCAllocForTenured)),
+    _surviving_alloc_buffer(g1h->desired_plab_sz(GCAllocForSurvived), queue_num),
+    _tenured_alloc_buffer(g1h->desired_plab_sz(GCAllocForTenured), queue_num),
     _age_table(false),
     _strong_roots_time(0), _term_time(0),
     _alloc_buffer_waste(0), _undo_waste(0)
@@ -4432,6 +4468,7 @@
 G1ParClosureSuper::G1ParClosureSuper(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state) :
   _g1(g1), _g1_rem(_g1->g1_rem_set()), _cm(_g1->concurrent_mark()),
   _par_scan_state(par_scan_state),
+  _worker_i(par_scan_state->queue_num()),
   _during_initial_mark(_g1->g1_policy()->during_initial_mark_pause()),
   _mark_in_progress(_g1->mark_in_progress()) { }
 
@@ -4445,7 +4482,7 @@
     oop obj = oopDesc::decode_heap_oop(heap_oop);
     HeapWord* addr = (HeapWord*)obj;
     if (_g1->is_in_g1_reserved(addr)) {
-      _cm->grayRoot(oop(addr));
+      _cm->grayRoot(oop(addr), _worker_i);
     }
   }
 }
@@ -4471,7 +4508,7 @@
     // This will either forward-to-self, or detect that someone else has
     // installed a forwarding pointer.
     OopsInHeapRegionClosure* cl = _par_scan_state->evac_failure_closure();
-    return _g1->handle_evacuation_failure_par(cl, old, should_mark_root);
+    return _g1->handle_evacuation_failure_par(cl, old, should_mark_root, _worker_i);
   }
 
   // We're going to allocate linearly, so might as well prefetch ahead.
@@ -4514,13 +4551,29 @@
         // if we couldn't mark it on the local bitmap (this happens when
         // the object was not allocated in the GCLab), we have to bite
         // the bullet and do the standard parallel mark
-        _cm->markAndGrayObjectIfNecessary(obj);
+        _cm->markAndGrayObjectIfNecessary(obj, _worker_i);
       }
 
       if (_g1->isMarkedNext(old)) {
         // Unmark the object's old location so that marking
         // doesn't think the old object is alive.
         _cm->nextMarkBitMap()->parClear((HeapWord*)old);
+
+        // We could clear the count data for the old object here but
+        // currently we do not. Why don't we do this? The thread/task
+        // that marks a newly copied object is likely _not_ the thread/task
+        // that originally marked the old object. So, to clear the count
+        // data for the old object, we would have to scan the count
+        // data for all of the tasks (and clear the data for the old object
+        // in parallel with other threads adding to the count data). Even
+        // then we could clear a bit incorrectly (e.g. if the old object
+        // does not start or end on a card boundary). It's more important
+        // that we don't have missed bits that should've been set than
+        // having extra bits set.
+        //
+        // As a result the accumulated count data could be a superset
+        // of the data that is/would have been calculated by walking
+        // the marking bitmap.
       }
     }
 
@@ -4607,7 +4660,8 @@
     }
     // When scanning the RS, we only care about objs in CS.
     if (barrier == G1BarrierRS) {
-      _par_scan_state->update_rs(_from, p, _par_scan_state->queue_num());
+      assert(_worker_i == _par_scan_state->queue_num(), "sanity");
+      _par_scan_state->update_rs(_from, p, _worker_i);
     }
   } else {
     // The object is not in collection set. If we're a root scanning
@@ -4619,7 +4673,8 @@
   }
 
   if (barrier == G1BarrierEvac && obj != NULL) {
-    _par_scan_state->update_rs(_from, p, _par_scan_state->queue_num());
+    assert(_worker_i == _par_scan_state->queue_num(), "sanity");
+    _par_scan_state->update_rs(_from, p, _worker_i);
   }
 
   if (do_gen_barrier && obj != NULL) {
@@ -5843,8 +5898,9 @@
       if (mark_in_progress()) {
         assert(!g1_policy()->during_initial_mark_pause(), "sanity");
         _cm->nextMarkBitMap()->clearRange(used_mr);
+        // Need to remove values from the count info
+         _cm->clear_count_data_for_heap_region(cur);
       }
-
       free_region(cur, &pre_used, &local_free_list, false /* par */);
     } else {
       cur->uninstall_surv_rate_group();
--- old/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp	2011-12-20 11:11:15.874446676 -0800
+++ new/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp	2011-12-20 11:11:15.640002377 -0800
@@ -862,7 +862,8 @@
 
   // An attempt to evacuate "obj" has failed; take necessary steps.
   oop handle_evacuation_failure_par(OopsInHeapRegionClosure* cl, oop obj,
-                                    bool should_mark_root);
+                                    bool should_mark_root,
+                                    int worker_i);
   void handle_evacuation_failure_common(oop obj, markOop m);
 
   // ("Weak") Reference processing support.
@@ -1730,6 +1731,7 @@
     // enough bits in the bitmap.
     return bits_in_bitmap + BitsPerWord - 1;
   }
+
 public:
   GCLabBitMap(HeapWord* heap_start, size_t gclab_word_size)
     : BitMap(bitmap_size_in_bits(gclab_word_size)),
@@ -1819,30 +1821,7 @@
   }
 #endif // PRODUCT
 
-  void retire() {
-    guarantee(use_local_bitmaps, "invariant");
-    assert(fields_well_formed(), "invariant");
-
-    if (_start_word != NULL) {
-      CMBitMap*       mark_bitmap = _cm->nextMarkBitMap();
-
-      // this means that the bitmap was set up for the GCLab
-      assert(_real_start_word != NULL && _real_end_word != NULL, "invariant");
-
-      mark_bitmap->mostly_disjoint_range_union(this,
-                                0, // always start from the start of the bitmap
-                                _start_word,
-                                gclab_real_word_size());
-      _cm->grayRegionIfNecessary(MemRegion(_real_start_word, _real_end_word));
-
-#ifndef PRODUCT
-      if (use_local_bitmaps && verify_local_bitmaps)
-        verify();
-#endif // PRODUCT
-    } else {
-      assert(_real_start_word == NULL && _real_end_word == NULL, "invariant");
-    }
-  }
+  void retire(int worker_i);
 
   size_t bitmap_size_in_words() const {
     return (bitmap_size_in_bits(gclab_word_size()) + BitsPerWord - 1) / BitsPerWord;
@@ -1855,9 +1834,10 @@
   bool        _retired;
   bool        _should_mark_objects;
   GCLabBitMap _bitmap;
+  int         _worker_i;
 
 public:
-  G1ParGCAllocBuffer(size_t gclab_word_size);
+  G1ParGCAllocBuffer(size_t gclab_word_size, int worker_i);
 
   inline bool mark(HeapWord* addr) {
     guarantee(use_local_bitmaps, "invariant");
@@ -1877,7 +1857,7 @@
     if (_retired)
       return;
     if (use_local_bitmaps && _should_mark_objects) {
-      _bitmap.retire();
+      _bitmap.retire(_worker_i);
     }
     ParGCAllocBuffer::retire(end_of_gc, retain);
     _retired = true;
--- old/src/share/vm/gc_implementation/g1/g1OopClosures.hpp	2011-12-20 11:11:17.575992629 -0800
+++ new/src/share/vm/gc_implementation/g1/g1OopClosures.hpp	2011-12-20 11:11:17.326809729 -0800
@@ -51,6 +51,7 @@
   G1RemSet* _g1_rem;
   ConcurrentMark* _cm;
   G1ParScanThreadState* _par_scan_state;
+  int _worker_i;
   bool _during_initial_mark;
   bool _mark_in_progress;
 public:
--- old/src/share/vm/gc_implementation/g1/heapRegion.hpp	2011-12-20 11:11:19.294649270 -0800
+++ new/src/share/vm/gc_implementation/g1/heapRegion.hpp	2011-12-20 11:11:18.865022183 -0800
@@ -373,7 +373,9 @@
     ScrubRemSetClaimValue      = 3,
     ParVerifyClaimValue        = 4,
     RebuildRSClaimValue        = 5,
-    CompleteMarkCSetClaimValue = 6
+    CompleteMarkCSetClaimValue = 6,
+    AggregateCountClaimValue   = 7,
+    VerifyCountClaimValue      = 8
   };
 
   inline HeapWord* par_allocate_no_bot_updates(size_t word_size) {