1 /*
   2  * Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #ifndef SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_CONCURRENTMARKSWEEPGENERATION_HPP
  26 #define SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_CONCURRENTMARKSWEEPGENERATION_HPP
  27 
  28 #include "gc_implementation/shared/gcHeapSummary.hpp"
  29 #include "gc_implementation/shared/gSpaceCounters.hpp"
  30 #include "gc_implementation/shared/gcStats.hpp"
  31 #include "gc_implementation/shared/gcWhen.hpp"
  32 #include "gc_implementation/shared/generationCounters.hpp"
  33 #include "memory/freeBlockDictionary.hpp"
  34 #include "memory/generation.hpp"
  35 #include "runtime/mutexLocker.hpp"
  36 #include "runtime/virtualspace.hpp"
  37 #include "services/memoryService.hpp"
  38 #include "utilities/bitMap.inline.hpp"
  39 #include "utilities/stack.inline.hpp"
  40 #include "utilities/taskqueue.hpp"
  41 #include "utilities/yieldingWorkgroup.hpp"
  42 
  43 // ConcurrentMarkSweepGeneration is in support of a concurrent
  44 // mark-sweep old generation in the Detlefs-Printezis--Boehm-Demers-Schenker
  45 // style. We assume, for now, that this generation is always the
  46 // seniormost generation and for simplicity
  47 // in the first implementation, that this generation is a single compactible
  48 // space. Neither of these restrictions appears essential, and will be
  49 // relaxed in the future when more time is available to implement the
  50 // greater generality (and there's a need for it).
  51 //
  52 // Concurrent mode failures are currently handled by
  53 // means of a sliding mark-compact.
  54 
  55 class CMSAdaptiveSizePolicy;
  56 class CMSConcMarkingTask;
  57 class CMSGCAdaptivePolicyCounters;
  58 class CMSTracer;
  59 class ConcurrentGCTimer;
  60 class ConcurrentMarkSweepGeneration;
  61 class ConcurrentMarkSweepPolicy;
  62 class ConcurrentMarkSweepThread;
  63 class CompactibleFreeListSpace;
  64 class FreeChunk;
  65 class PromotionInfo;
  66 class ScanMarkedObjectsAgainCarefullyClosure;
  67 class TenuredGeneration;
  68 class SerialOldTracer;
  69 
  70 // A generic CMS bit map. It's the basis for both the CMS marking bit map
  71 // as well as for the mod union table (in each case only a subset of the
  72 // methods are used). This is essentially a wrapper around the BitMap class,
  73 // with one bit per (1<<_shifter) HeapWords. (i.e. for the marking bit map,
  74 // we have _shifter == 0. and for the mod union table we have
  75 // shifter == CardTableModRefBS::card_shift - LogHeapWordSize.)
  76 // XXX 64-bit issues in BitMap?
  77 class CMSBitMap VALUE_OBJ_CLASS_SPEC {
  78   friend class VMStructs;
  79 
  80   HeapWord* _bmStartWord;   // base address of range covered by map
  81   size_t    _bmWordSize;    // map size (in #HeapWords covered)
  82   const int _shifter;       // shifts to convert HeapWord to bit position
  83   VirtualSpace _virtual_space; // underlying the bit map
  84   BitMap    _bm;            // the bit map itself
  85  public:
  86   Mutex* const _lock;       // mutex protecting _bm;
  87 
  88  public:
  89   // constructor
  90   CMSBitMap(int shifter, int mutex_rank, const char* mutex_name);
  91 
  92   // allocates the actual storage for the map
  93   bool allocate(MemRegion mr);
  94   // field getter
  95   Mutex* lock() const { return _lock; }
  96   // locking verifier convenience function
  97   void assert_locked() const PRODUCT_RETURN;
  98 
  99   // inquiries
 100   HeapWord* startWord()   const { return _bmStartWord; }
 101   size_t    sizeInWords() const { return _bmWordSize;  }
 102   size_t    sizeInBits()  const { return _bm.size();   }
 103   // the following is one past the last word in space
 104   HeapWord* endWord()     const { return _bmStartWord + _bmWordSize; }
 105 
 106   // reading marks
 107   bool isMarked(HeapWord* addr) const;
 108   bool par_isMarked(HeapWord* addr) const; // do not lock checks
 109   bool isUnmarked(HeapWord* addr) const;
 110   bool isAllClear() const;
 111 
 112   // writing marks
 113   void mark(HeapWord* addr);
 114   // For marking by parallel GC threads;
 115   // returns true if we did, false if another thread did
 116   bool par_mark(HeapWord* addr);
 117 
 118   void mark_range(MemRegion mr);
 119   void par_mark_range(MemRegion mr);
 120   void mark_large_range(MemRegion mr);
 121   void par_mark_large_range(MemRegion mr);
 122   void par_clear(HeapWord* addr); // For unmarking by parallel GC threads.
 123   void clear_range(MemRegion mr);
 124   void par_clear_range(MemRegion mr);
 125   void clear_large_range(MemRegion mr);
 126   void par_clear_large_range(MemRegion mr);
 127   void clear_all();
 128   void clear_all_incrementally();  // Not yet implemented!!
 129 
 130   NOT_PRODUCT(
 131     // checks the memory region for validity
 132     void region_invariant(MemRegion mr);
 133   )
 134 
 135   // iteration
 136   void iterate(BitMapClosure* cl) {
 137     _bm.iterate(cl);
 138   }
 139   void iterate(BitMapClosure* cl, HeapWord* left, HeapWord* right);
 140   void dirty_range_iterate_clear(MemRegionClosure* cl);
 141   void dirty_range_iterate_clear(MemRegion mr, MemRegionClosure* cl);
 142 
 143   // auxiliary support for iteration
 144   HeapWord* getNextMarkedWordAddress(HeapWord* addr) const;
 145   HeapWord* getNextMarkedWordAddress(HeapWord* start_addr,
 146                                             HeapWord* end_addr) const;
 147   HeapWord* getNextUnmarkedWordAddress(HeapWord* addr) const;
 148   HeapWord* getNextUnmarkedWordAddress(HeapWord* start_addr,
 149                                               HeapWord* end_addr) const;
 150   MemRegion getAndClearMarkedRegion(HeapWord* addr);
 151   MemRegion getAndClearMarkedRegion(HeapWord* start_addr,
 152                                            HeapWord* end_addr);
 153 
 154   // conversion utilities
 155   HeapWord* offsetToHeapWord(size_t offset) const;
 156   size_t    heapWordToOffset(HeapWord* addr) const;
 157   size_t    heapWordDiffToOffsetDiff(size_t diff) const;
 158 
 159   void print_on_error(outputStream* st, const char* prefix) const;
 160 
 161   // debugging
 162   // is this address range covered by the bit-map?
 163   NOT_PRODUCT(
 164     bool covers(MemRegion mr) const;
 165     bool covers(HeapWord* start, size_t size = 0) const;
 166   )
 167   void verifyNoOneBitsInRange(HeapWord* left, HeapWord* right) PRODUCT_RETURN;
 168 };
 169 
 170 // Represents a marking stack used by the CMS collector.
 171 // Ideally this should be GrowableArray<> just like MSC's marking stack(s).
 172 class CMSMarkStack: public CHeapObj<mtGC>  {
 173   //
 174   friend class CMSCollector;   // to get at expasion stats further below
 175   //
 176 
 177   VirtualSpace _virtual_space;  // space for the stack
 178   oop*   _base;      // bottom of stack
 179   size_t _index;     // one more than last occupied index
 180   size_t _capacity;  // max #elements
 181   Mutex  _par_lock;  // an advisory lock used in case of parallel access
 182   NOT_PRODUCT(size_t _max_depth;)  // max depth plumbed during run
 183 
 184  protected:
 185   size_t _hit_limit;      // we hit max stack size limit
 186   size_t _failed_double;  // we failed expansion before hitting limit
 187 
 188  public:
 189   CMSMarkStack():
 190     _par_lock(Mutex::event, "CMSMarkStack._par_lock", true),
 191     _hit_limit(0),
 192     _failed_double(0) {}
 193 
 194   bool allocate(size_t size);
 195 
 196   size_t capacity() const { return _capacity; }
 197 
 198   oop pop() {
 199     if (!isEmpty()) {
 200       return _base[--_index] ;
 201     }
 202     return NULL;
 203   }
 204 
 205   bool push(oop ptr) {
 206     if (isFull()) {
 207       return false;
 208     } else {
 209       _base[_index++] = ptr;
 210       NOT_PRODUCT(_max_depth = MAX2(_max_depth, _index));
 211       return true;
 212     }
 213   }
 214 
 215   bool isEmpty() const { return _index == 0; }
 216   bool isFull()  const {
 217     assert(_index <= _capacity, "buffer overflow");
 218     return _index == _capacity;
 219   }
 220 
 221   size_t length() { return _index; }
 222 
 223   // "Parallel versions" of some of the above
 224   oop par_pop() {
 225     // lock and pop
 226     MutexLockerEx x(&_par_lock, Mutex::_no_safepoint_check_flag);
 227     return pop();
 228   }
 229 
 230   bool par_push(oop ptr) {
 231     // lock and push
 232     MutexLockerEx x(&_par_lock, Mutex::_no_safepoint_check_flag);
 233     return push(ptr);
 234   }
 235 
 236   // Forcibly reset the stack, losing all of its contents.
 237   void reset() {
 238     _index = 0;
 239   }
 240 
 241   // Expand the stack, typically in response to an overflow condition
 242   void expand();
 243 
 244   // Compute the least valued stack element.
 245   oop least_value(HeapWord* low) {
 246      oop least = (oop)low;
 247      for (size_t i = 0; i < _index; i++) {
 248        least = MIN2(least, _base[i]);
 249      }
 250      return least;
 251   }
 252 
 253   // Exposed here to allow stack expansion in || case
 254   Mutex* par_lock() { return &_par_lock; }
 255 };
 256 
 257 class CardTableRS;
 258 class CMSParGCThreadState;
 259 
 260 class ModUnionClosure: public MemRegionClosure {
 261  protected:
 262   CMSBitMap* _t;
 263  public:
 264   ModUnionClosure(CMSBitMap* t): _t(t) { }
 265   void do_MemRegion(MemRegion mr);
 266 };
 267 
 268 class ModUnionClosurePar: public ModUnionClosure {
 269  public:
 270   ModUnionClosurePar(CMSBitMap* t): ModUnionClosure(t) { }
 271   void do_MemRegion(MemRegion mr);
 272 };
 273 
 274 // Survivor Chunk Array in support of parallelization of
 275 // Survivor Space rescan.
 276 class ChunkArray: public CHeapObj<mtGC> {
 277   size_t _index;
 278   size_t _capacity;
 279   size_t _overflows;
 280   HeapWord** _array;   // storage for array
 281 
 282  public:
 283   ChunkArray() : _index(0), _capacity(0), _overflows(0), _array(NULL) {}
 284   ChunkArray(HeapWord** a, size_t c):
 285     _index(0), _capacity(c), _overflows(0), _array(a) {}
 286 
 287   HeapWord** array() { return _array; }
 288   void set_array(HeapWord** a) { _array = a; }
 289 
 290   size_t capacity() { return _capacity; }
 291   void set_capacity(size_t c) { _capacity = c; }
 292 
 293   size_t end() {
 294     assert(_index <= capacity(),
 295            err_msg("_index (" SIZE_FORMAT ") > _capacity (" SIZE_FORMAT "): out of bounds",
 296                    _index, _capacity));
 297     return _index;
 298   }  // exclusive
 299 
 300   HeapWord* nth(size_t n) {
 301     assert(n < end(), "Out of bounds access");
 302     return _array[n];
 303   }
 304 
 305   void reset() {
 306     _index = 0;
 307     if (_overflows > 0 && PrintCMSStatistics > 1) {
 308       warning("CMS: ChunkArray[" SIZE_FORMAT "] overflowed " SIZE_FORMAT " times",
 309               _capacity, _overflows);
 310     }
 311     _overflows = 0;
 312   }
 313 
 314   void record_sample(HeapWord* p, size_t sz) {
 315     // For now we do not do anything with the size
 316     if (_index < _capacity) {
 317       _array[_index++] = p;
 318     } else {
 319       ++_overflows;
 320       assert(_index == _capacity,
 321              err_msg("_index (" SIZE_FORMAT ") > _capacity (" SIZE_FORMAT
 322                      "): out of bounds at overflow#" SIZE_FORMAT,
 323                      _index, _capacity, _overflows));
 324     }
 325   }
 326 };
 327 
 328 //
 329 // Timing, allocation and promotion statistics for gc scheduling and incremental
 330 // mode pacing.  Most statistics are exponential averages.
 331 //
 332 class CMSStats VALUE_OBJ_CLASS_SPEC {
 333  private:
 334   ConcurrentMarkSweepGeneration* const _cms_gen;   // The cms (old) gen.
 335 
 336   // The following are exponential averages with factor alpha:
 337   //   avg = (100 - alpha) * avg + alpha * cur_sample
 338   //
 339   //   The durations measure:  end_time[n] - start_time[n]
 340   //   The periods measure:    start_time[n] - start_time[n-1]
 341   //
 342   // The cms period and duration include only concurrent collections; time spent
 343   // in foreground cms collections due to System.gc() or because of a failure to
 344   // keep up are not included.
 345   //
 346   // There are 3 alphas to "bootstrap" the statistics.  The _saved_alpha is the
 347   // real value, but is used only after the first period.  A value of 100 is
 348   // used for the first sample so it gets the entire weight.
 349   unsigned int _saved_alpha; // 0-100
 350   unsigned int _gc0_alpha;
 351   unsigned int _cms_alpha;
 352 
 353   double _gc0_duration;
 354   double _gc0_period;
 355   size_t _gc0_promoted;         // bytes promoted per gc0
 356   double _cms_duration;
 357   double _cms_duration_pre_sweep; // time from initiation to start of sweep
 358   double _cms_duration_per_mb;
 359   double _cms_period;
 360   size_t _cms_allocated;        // bytes of direct allocation per gc0 period
 361 
 362   // Timers.
 363   elapsedTimer _cms_timer;
 364   TimeStamp    _gc0_begin_time;
 365   TimeStamp    _cms_begin_time;
 366   TimeStamp    _cms_end_time;
 367 
 368   // Snapshots of the amount used in the CMS generation.
 369   size_t _cms_used_at_gc0_begin;
 370   size_t _cms_used_at_gc0_end;
 371   size_t _cms_used_at_cms_begin;
 372 
 373   // Used to prevent the duty cycle from being reduced in the middle of a cms
 374   // cycle.
 375   bool _allow_duty_cycle_reduction;
 376 
 377   enum {
 378     _GC0_VALID = 0x1,
 379     _CMS_VALID = 0x2,
 380     _ALL_VALID = _GC0_VALID | _CMS_VALID
 381   };
 382 
 383   unsigned int _valid_bits;
 384 
 385   unsigned int _icms_duty_cycle;        // icms duty cycle (0-100).
 386 
 387  protected:
 388 
 389   // Return a duty cycle that avoids wild oscillations, by limiting the amount
 390   // of change between old_duty_cycle and new_duty_cycle (the latter is treated
 391   // as a recommended value).
 392   static unsigned int icms_damped_duty_cycle(unsigned int old_duty_cycle,
 393                                              unsigned int new_duty_cycle);
 394   unsigned int icms_update_duty_cycle_impl();
 395 
 396   // In support of adjusting of cms trigger ratios based on history
 397   // of concurrent mode failure.
 398   double cms_free_adjustment_factor(size_t free) const;
 399   void   adjust_cms_free_adjustment_factor(bool fail, size_t free);
 400 
 401  public:
 402   CMSStats(ConcurrentMarkSweepGeneration* cms_gen,
 403            unsigned int alpha = CMSExpAvgFactor);
 404 
 405   // Whether or not the statistics contain valid data; higher level statistics
 406   // cannot be called until this returns true (they require at least one young
 407   // gen and one cms cycle to have completed).
 408   bool valid() const;
 409 
 410   // Record statistics.
 411   void record_gc0_begin();
 412   void record_gc0_end(size_t cms_gen_bytes_used);
 413   void record_cms_begin();
 414   void record_cms_end();
 415 
 416   // Allow management of the cms timer, which must be stopped/started around
 417   // yield points.
 418   elapsedTimer& cms_timer()     { return _cms_timer; }
 419   void start_cms_timer()        { _cms_timer.start(); }
 420   void stop_cms_timer()         { _cms_timer.stop(); }
 421 
 422   // Basic statistics; units are seconds or bytes.
 423   double gc0_period() const     { return _gc0_period; }
 424   double gc0_duration() const   { return _gc0_duration; }
 425   size_t gc0_promoted() const   { return _gc0_promoted; }
 426   double cms_period() const          { return _cms_period; }
 427   double cms_duration() const        { return _cms_duration; }
 428   double cms_duration_per_mb() const { return _cms_duration_per_mb; }
 429   size_t cms_allocated() const       { return _cms_allocated; }
 430 
 431   size_t cms_used_at_gc0_end() const { return _cms_used_at_gc0_end;}
 432 
 433   // Seconds since the last background cms cycle began or ended.
 434   double cms_time_since_begin() const;
 435   double cms_time_since_end() const;
 436 
 437   // Higher level statistics--caller must check that valid() returns true before
 438   // calling.
 439 
 440   // Returns bytes promoted per second of wall clock time.
 441   double promotion_rate() const;
 442 
 443   // Returns bytes directly allocated per second of wall clock time.
 444   double cms_allocation_rate() const;
 445 
 446   // Rate at which space in the cms generation is being consumed (sum of the
 447   // above two).
 448   double cms_consumption_rate() const;
 449 
 450   // Returns an estimate of the number of seconds until the cms generation will
 451   // fill up, assuming no collection work is done.
 452   double time_until_cms_gen_full() const;
 453 
 454   // Returns an estimate of the number of seconds remaining until
 455   // the cms generation collection should start.
 456   double time_until_cms_start() const;
 457 
 458   // End of higher level statistics.
 459 
 460   // Returns the cms incremental mode duty cycle, as a percentage (0-100).
 461   unsigned int icms_duty_cycle() const { return _icms_duty_cycle; }
 462 
 463   // Update the duty cycle and return the new value.
 464   unsigned int icms_update_duty_cycle();
 465 
 466   // Debugging.
 467   void print_on(outputStream* st) const PRODUCT_RETURN;
 468   void print() const { print_on(gclog_or_tty); }
 469 };
 470 
 471 // A closure related to weak references processing which
 472 // we embed in the CMSCollector, since we need to pass
 473 // it to the reference processor for secondary filtering
 474 // of references based on reachability of referent;
 475 // see role of _is_alive_non_header closure in the
 476 // ReferenceProcessor class.
 477 // For objects in the CMS generation, this closure checks
 478 // if the object is "live" (reachable). Used in weak
 479 // reference processing.
 480 class CMSIsAliveClosure: public BoolObjectClosure {
 481   const MemRegion  _span;
 482   const CMSBitMap* _bit_map;
 483 
 484   friend class CMSCollector;
 485  public:
 486   CMSIsAliveClosure(MemRegion span,
 487                     CMSBitMap* bit_map):
 488     _span(span),
 489     _bit_map(bit_map) {
 490     assert(!span.is_empty(), "Empty span could spell trouble");
 491   }
 492 
 493   bool do_object_b(oop obj);
 494 };
 495 
 496 
 497 // Implements AbstractRefProcTaskExecutor for CMS.
 498 class CMSRefProcTaskExecutor: public AbstractRefProcTaskExecutor {
 499 public:
 500 
 501   CMSRefProcTaskExecutor(CMSCollector& collector)
 502     : _collector(collector)
 503   { }
 504 
 505   // Executes a task using worker threads.
 506   virtual void execute(ProcessTask& task);
 507   virtual void execute(EnqueueTask& task);
 508 private:
 509   CMSCollector& _collector;
 510 };
 511 
 512 
 513 class CMSCollector: public CHeapObj<mtGC> {
 514   friend class VMStructs;
 515   friend class ConcurrentMarkSweepThread;
 516   friend class ConcurrentMarkSweepGeneration;
 517   friend class CompactibleFreeListSpace;
 518   friend class CMSParRemarkTask;
 519   friend class CMSConcMarkingTask;
 520   friend class CMSRefProcTaskProxy;
 521   friend class CMSRefProcTaskExecutor;
 522   friend class ScanMarkedObjectsAgainCarefullyClosure;  // for sampling eden
 523   friend class SurvivorSpacePrecleanClosure;            // --- ditto -------
 524   friend class PushOrMarkClosure;             // to access _restart_addr
 525   friend class Par_PushOrMarkClosure;             // to access _restart_addr
 526   friend class MarkFromRootsClosure;          //  -- ditto --
 527                                               // ... and for clearing cards
 528   friend class Par_MarkFromRootsClosure;      //  to access _restart_addr
 529                                               // ... and for clearing cards
 530   friend class Par_ConcMarkingClosure;        //  to access _restart_addr etc.
 531   friend class MarkFromRootsVerifyClosure;    // to access _restart_addr
 532   friend class PushAndMarkVerifyClosure;      //  -- ditto --
 533   friend class MarkRefsIntoAndScanClosure;    // to access _overflow_list
 534   friend class PushAndMarkClosure;            //  -- ditto --
 535   friend class Par_PushAndMarkClosure;        //  -- ditto --
 536   friend class CMSKeepAliveClosure;           //  -- ditto --
 537   friend class CMSDrainMarkingStackClosure;   //  -- ditto --
 538   friend class CMSInnerParMarkAndPushClosure; //  -- ditto --
 539   NOT_PRODUCT(friend class ScanMarkedObjectsAgainClosure;) //  assertion on _overflow_list
 540   friend class ReleaseForegroundGC;  // to access _foregroundGCShouldWait
 541   friend class VM_CMS_Operation;
 542   friend class VM_CMS_Initial_Mark;
 543   friend class VM_CMS_Final_Remark;
 544   friend class TraceCMSMemoryManagerStats;
 545 
 546  private:
 547   jlong _time_of_last_gc;
 548   void update_time_of_last_gc(jlong now) {
 549     _time_of_last_gc = now;
 550   }
 551 
 552   OopTaskQueueSet* _task_queues;
 553 
 554   // Overflow list of grey objects, threaded through mark-word
 555   // Manipulated with CAS in the parallel/multi-threaded case.
 556   oop _overflow_list;
 557   // The following array-pair keeps track of mark words
 558   // displaced for accomodating overflow list above.
 559   // This code will likely be revisited under RFE#4922830.
 560   Stack<oop, mtGC>     _preserved_oop_stack;
 561   Stack<markOop, mtGC> _preserved_mark_stack;
 562 
 563   int*             _hash_seed;
 564 
 565   // In support of multi-threaded concurrent phases
 566   YieldingFlexibleWorkGang* _conc_workers;
 567 
 568   // Performance Counters
 569   CollectorCounters* _gc_counters;
 570 
 571   // Initialization Errors
 572   bool _completed_initialization;
 573 
 574   // In support of ExplicitGCInvokesConcurrent
 575   static bool _full_gc_requested;
 576   static GCCause::Cause _full_gc_cause;
 577   unsigned int _collection_count_start;
 578 
 579   // Should we unload classes this concurrent cycle?
 580   bool _should_unload_classes;
 581   unsigned int  _concurrent_cycles_since_last_unload;
 582   unsigned int concurrent_cycles_since_last_unload() const {
 583     return _concurrent_cycles_since_last_unload;
 584   }
 585   // Did we (allow) unload classes in the previous concurrent cycle?
 586   bool unloaded_classes_last_cycle() const {
 587     return concurrent_cycles_since_last_unload() == 0;
 588   }
 589   // Root scanning options for perm gen
 590   int _roots_scanning_options;
 591   int roots_scanning_options() const      { return _roots_scanning_options; }
 592   void add_root_scanning_option(int o)    { _roots_scanning_options |= o;   }
 593   void remove_root_scanning_option(int o) { _roots_scanning_options &= ~o;  }
 594 
 595   // Verification support
 596   CMSBitMap     _verification_mark_bm;
 597   void verify_after_remark_work_1();
 598   void verify_after_remark_work_2();
 599 
 600   // true if any verification flag is on.
 601   bool _verifying;
 602   bool verifying() const { return _verifying; }
 603   void set_verifying(bool v) { _verifying = v; }
 604 
 605   // Collector policy
 606   ConcurrentMarkSweepPolicy* _collector_policy;
 607   ConcurrentMarkSweepPolicy* collector_policy() { return _collector_policy; }
 608 
 609   void set_did_compact(bool v);
 610 
 611   // XXX Move these to CMSStats ??? FIX ME !!!
 612   elapsedTimer _inter_sweep_timer;   // time between sweeps
 613   elapsedTimer _intra_sweep_timer;   // time _in_ sweeps
 614   // padded decaying average estimates of the above
 615   AdaptivePaddedAverage _inter_sweep_estimate;
 616   AdaptivePaddedAverage _intra_sweep_estimate;
 617 
 618   CMSTracer* _gc_tracer_cm;
 619   ConcurrentGCTimer* _gc_timer_cm;
 620 
 621   bool _cms_start_registered;
 622 
 623   GCHeapSummary _last_heap_summary;
 624   MetaspaceSummary _last_metaspace_summary;
 625 
 626   void register_foreground_gc_start(GCCause::Cause cause);
 627   void register_gc_start(GCCause::Cause cause);
 628   void register_gc_end();
 629   void save_heap_summary();
 630   void report_heap_summary(GCWhen::Type when);
 631 
 632  protected:
 633   ConcurrentMarkSweepGeneration* _cmsGen;  // old gen (CMS)
 634   MemRegion                      _span;    // span covering above two
 635   CardTableRS*                   _ct;      // card table
 636 
 637   // CMS marking support structures
 638   CMSBitMap     _markBitMap;
 639   CMSBitMap     _modUnionTable;
 640   CMSMarkStack  _markStack;
 641 
 642   HeapWord*     _restart_addr; // in support of marking stack overflow
 643   void          lower_restart_addr(HeapWord* low);
 644 
 645   // Counters in support of marking stack / work queue overflow handling:
 646   // a non-zero value indicates certain types of overflow events during
 647   // the current CMS cycle and could lead to stack resizing efforts at
 648   // an opportune future time.
 649   size_t        _ser_pmc_preclean_ovflw;
 650   size_t        _ser_pmc_remark_ovflw;
 651   size_t        _par_pmc_remark_ovflw;
 652   size_t        _ser_kac_preclean_ovflw;
 653   size_t        _ser_kac_ovflw;
 654   size_t        _par_kac_ovflw;
 655   NOT_PRODUCT(ssize_t _num_par_pushes;)
 656 
 657   // ("Weak") Reference processing support
 658   ReferenceProcessor*            _ref_processor;
 659   CMSIsAliveClosure              _is_alive_closure;
 660       // keep this textually after _markBitMap and _span; c'tor dependency
 661 
 662   ConcurrentMarkSweepThread*     _cmsThread;   // the thread doing the work
 663   ModUnionClosure    _modUnionClosure;
 664   ModUnionClosurePar _modUnionClosurePar;
 665 
 666   // CMS abstract state machine
 667   // initial_state: Idling
 668   // next_state(Idling)            = {Marking}
 669   // next_state(Marking)           = {Precleaning, Sweeping}
 670   // next_state(Precleaning)       = {AbortablePreclean, FinalMarking}
 671   // next_state(AbortablePreclean) = {FinalMarking}
 672   // next_state(FinalMarking)      = {Sweeping}
 673   // next_state(Sweeping)          = {Resizing}
 674   // next_state(Resizing)          = {Resetting}
 675   // next_state(Resetting)         = {Idling}
 676   // The numeric values below are chosen so that:
 677   // . _collectorState <= Idling ==  post-sweep && pre-mark
 678   // . _collectorState in (Idling, Sweeping) == {initial,final}marking ||
 679   //                                            precleaning || abortablePrecleanb
 680  public:
 681   enum CollectorState {
 682     Resizing            = 0,
 683     Resetting           = 1,
 684     Idling              = 2,
 685     InitialMarking      = 3,
 686     Marking             = 4,
 687     Precleaning         = 5,
 688     AbortablePreclean   = 6,
 689     FinalMarking        = 7,
 690     Sweeping            = 8
 691   };
 692  protected:
 693   static CollectorState _collectorState;
 694 
 695   // State related to prologue/epilogue invocation for my generations
 696   bool _between_prologue_and_epilogue;
 697 
 698   // Signalling/State related to coordination between fore- and backgroud GC
 699   // Note: When the baton has been passed from background GC to foreground GC,
 700   // _foregroundGCIsActive is true and _foregroundGCShouldWait is false.
 701   static bool _foregroundGCIsActive;    // true iff foreground collector is active or
 702                                  // wants to go active
 703   static bool _foregroundGCShouldWait;  // true iff background GC is active and has not
 704                                  // yet passed the baton to the foreground GC
 705 
 706   // Support for CMSScheduleRemark (abortable preclean)
 707   bool _abort_preclean;
 708   bool _start_sampling;
 709 
 710   int    _numYields;
 711   size_t _numDirtyCards;
 712   size_t _sweep_count;
 713   // number of full gc's since the last concurrent gc.
 714   uint   _full_gcs_since_conc_gc;
 715 
 716   // occupancy used for bootstrapping stats
 717   double _bootstrap_occupancy;
 718 
 719   // timer
 720   elapsedTimer _timer;
 721 
 722   // Timing, allocation and promotion statistics, used for scheduling.
 723   CMSStats      _stats;
 724 
 725   // Allocation limits installed in the young gen, used only in
 726   // CMSIncrementalMode.  When an allocation in the young gen would cross one of
 727   // these limits, the cms generation is notified and the cms thread is started
 728   // or stopped, respectively.
 729   HeapWord*     _icms_start_limit;
 730   HeapWord*     _icms_stop_limit;
 731 
 732   enum CMS_op_type {
 733     CMS_op_checkpointRootsInitial,
 734     CMS_op_checkpointRootsFinal
 735   };
 736 
 737   void do_CMS_operation(CMS_op_type op, GCCause::Cause gc_cause);
 738   bool stop_world_and_do(CMS_op_type op);
 739 
 740   OopTaskQueueSet* task_queues() { return _task_queues; }
 741   int*             hash_seed(int i) { return &_hash_seed[i]; }
 742   YieldingFlexibleWorkGang* conc_workers() { return _conc_workers; }
 743 
 744   // Support for parallelizing Eden rescan in CMS remark phase
 745   void sample_eden(); // ... sample Eden space top
 746 
 747  private:
 748   // Support for parallelizing young gen rescan in CMS remark phase
 749   Generation* _young_gen;  // the younger gen
 750   HeapWord** _top_addr;    // ... Top of Eden
 751   HeapWord** _end_addr;    // ... End of Eden
 752   HeapWord** _eden_chunk_array; // ... Eden partitioning array
 753   size_t     _eden_chunk_index; // ... top (exclusive) of array
 754   size_t     _eden_chunk_capacity;  // ... max entries in array
 755 
 756   // Support for parallelizing survivor space rescan
 757   HeapWord** _survivor_chunk_array;
 758   size_t     _survivor_chunk_index;
 759   size_t     _survivor_chunk_capacity;
 760   size_t*    _cursor;
 761   ChunkArray* _survivor_plab_array;
 762 
 763   // Support for marking stack overflow handling
 764   bool take_from_overflow_list(size_t num, CMSMarkStack* to_stack);
 765   bool par_take_from_overflow_list(size_t num,
 766                                    OopTaskQueue* to_work_q,
 767                                    int no_of_gc_threads);
 768   void push_on_overflow_list(oop p);
 769   void par_push_on_overflow_list(oop p);
 770   // the following is, obviously, not, in general, "MT-stable"
 771   bool overflow_list_is_empty() const;
 772 
 773   void preserve_mark_if_necessary(oop p);
 774   void par_preserve_mark_if_necessary(oop p);
 775   void preserve_mark_work(oop p, markOop m);
 776   void restore_preserved_marks_if_any();
 777   NOT_PRODUCT(bool no_preserved_marks() const;)
 778   // in support of testing overflow code
 779   NOT_PRODUCT(int _overflow_counter;)
 780   NOT_PRODUCT(bool simulate_overflow();)       // sequential
 781   NOT_PRODUCT(bool par_simulate_overflow();)   // MT version
 782 
 783   // CMS work methods
 784   void checkpointRootsInitialWork(bool asynch); // initial checkpoint work
 785 
 786   // a return value of false indicates failure due to stack overflow
 787   bool markFromRootsWork(bool asynch);  // concurrent marking work
 788 
 789  public:   // FIX ME!!! only for testing
 790   bool do_marking_st(bool asynch);      // single-threaded marking
 791   bool do_marking_mt(bool asynch);      // multi-threaded  marking
 792 
 793  private:
 794 
 795   // concurrent precleaning work
 796   size_t preclean_mod_union_table(ConcurrentMarkSweepGeneration* gen,
 797                                   ScanMarkedObjectsAgainCarefullyClosure* cl);
 798   size_t preclean_card_table(ConcurrentMarkSweepGeneration* gen,
 799                              ScanMarkedObjectsAgainCarefullyClosure* cl);
 800   // Does precleaning work, returning a quantity indicative of
 801   // the amount of "useful work" done.
 802   size_t preclean_work(bool clean_refs, bool clean_survivors);
 803   void preclean_klasses(MarkRefsIntoAndScanClosure* cl, Mutex* freelistLock);
 804   void abortable_preclean(); // Preclean while looking for possible abort
 805   void initialize_sequential_subtasks_for_young_gen_rescan(int i);
 806   // Helper function for above; merge-sorts the per-thread plab samples
 807   void merge_survivor_plab_arrays(ContiguousSpace* surv, int no_of_gc_threads);
 808   // Resets (i.e. clears) the per-thread plab sample vectors
 809   void reset_survivor_plab_arrays();
 810 
 811   // final (second) checkpoint work
 812   void checkpointRootsFinalWork(bool asynch, bool clear_all_soft_refs,
 813                                 bool init_mark_was_synchronous);
 814   // work routine for parallel version of remark
 815   void do_remark_parallel();
 816   // work routine for non-parallel version of remark
 817   void do_remark_non_parallel();
 818   // reference processing work routine (during second checkpoint)
 819   void refProcessingWork(bool asynch, bool clear_all_soft_refs);
 820 
 821   // concurrent sweeping work
 822   void sweepWork(ConcurrentMarkSweepGeneration* gen, bool asynch);
 823 
 824   // (concurrent) resetting of support data structures
 825   void reset(bool asynch);
 826 
 827   // Clear _expansion_cause fields of constituent generations
 828   void clear_expansion_cause();
 829 
 830   // An auxilliary method used to record the ends of
 831   // used regions of each generation to limit the extent of sweep
 832   void save_sweep_limits();
 833 
 834   // A work method used by foreground collection to determine
 835   // what type of collection (compacting or not, continuing or fresh)
 836   // it should do.
 837   void decide_foreground_collection_type(bool clear_all_soft_refs,
 838     bool* should_compact, bool* should_start_over);
 839 
 840   // A work method used by the foreground collector to do
 841   // a mark-sweep-compact.
 842   void do_compaction_work(bool clear_all_soft_refs);
 843 
 844   // A work method used by the foreground collector to do
 845   // a mark-sweep, after taking over from a possibly on-going
 846   // concurrent mark-sweep collection.
 847   void do_mark_sweep_work(bool clear_all_soft_refs,
 848     CollectorState first_state, bool should_start_over);
 849 
 850   // Work methods for reporting concurrent mode interruption or failure
 851   bool is_external_interruption();
 852   void report_concurrent_mode_interruption();
 853 
 854   // If the backgrould GC is active, acquire control from the background
 855   // GC and do the collection.
 856   void acquire_control_and_collect(bool   full, bool clear_all_soft_refs);
 857 
 858   // For synchronizing passing of control from background to foreground
 859   // GC.  waitForForegroundGC() is called by the background
 860   // collector.  It if had to wait for a foreground collection,
 861   // it returns true and the background collection should assume
 862   // that the collection was finished by the foreground
 863   // collector.
 864   bool waitForForegroundGC();
 865 
 866   // Incremental mode triggering:  recompute the icms duty cycle and set the
 867   // allocation limits in the young gen.
 868   void icms_update_allocation_limits();
 869 
 870   size_t block_size_using_printezis_bits(HeapWord* addr) const;
 871   size_t block_size_if_printezis_bits(HeapWord* addr) const;
 872   HeapWord* next_card_start_after_block(HeapWord* addr) const;
 873 
 874   void setup_cms_unloading_and_verification_state();
 875  public:
 876   CMSCollector(ConcurrentMarkSweepGeneration* cmsGen,
 877                CardTableRS*                   ct,
 878                ConcurrentMarkSweepPolicy*     cp);
 879   ConcurrentMarkSweepThread* cmsThread() { return _cmsThread; }
 880 
 881   ReferenceProcessor* ref_processor() { return _ref_processor; }
 882   void ref_processor_init();
 883 
 884   Mutex* bitMapLock()        const { return _markBitMap.lock();    }
 885   static CollectorState abstract_state() { return _collectorState;  }
 886 
 887   bool should_abort_preclean() const; // Whether preclean should be aborted.
 888   size_t get_eden_used() const;
 889   size_t get_eden_capacity() const;
 890 
 891   ConcurrentMarkSweepGeneration* cmsGen() { return _cmsGen; }
 892 
 893   // locking checks
 894   NOT_PRODUCT(static bool have_cms_token();)
 895 
 896   // XXXPERM bool should_collect(bool full, size_t size, bool tlab);
 897   bool shouldConcurrentCollect();
 898 
 899   void collect(bool   full,
 900                bool   clear_all_soft_refs,
 901                size_t size,
 902                bool   tlab);
 903   void collect_in_background(bool clear_all_soft_refs, GCCause::Cause cause);
 904   void collect_in_foreground(bool clear_all_soft_refs, GCCause::Cause cause);
 905 
 906   // In support of ExplicitGCInvokesConcurrent
 907   static void request_full_gc(unsigned int full_gc_count, GCCause::Cause cause);
 908   // Should we unload classes in a particular concurrent cycle?
 909   bool should_unload_classes() const {
 910     return _should_unload_classes;
 911   }
 912   void update_should_unload_classes();
 913 
 914   void direct_allocated(HeapWord* start, size_t size);
 915 
 916   // Object is dead if not marked and current phase is sweeping.
 917   bool is_dead_obj(oop obj) const;
 918 
 919   // After a promotion (of "start"), do any necessary marking.
 920   // If "par", then it's being done by a parallel GC thread.
 921   // The last two args indicate if we need precise marking
 922   // and if so the size of the object so it can be dirtied
 923   // in its entirety.
 924   void promoted(bool par, HeapWord* start,
 925                 bool is_obj_array, size_t obj_size);
 926 
 927   HeapWord* allocation_limit_reached(Space* space, HeapWord* top,
 928                                      size_t word_size);
 929 
 930   void getFreelistLocks() const;
 931   void releaseFreelistLocks() const;
 932   bool haveFreelistLocks() const;
 933 
 934   // Adjust size of underlying generation
 935   void compute_new_size();
 936 
 937   // GC prologue and epilogue
 938   void gc_prologue(bool full);
 939   void gc_epilogue(bool full);
 940 
 941   jlong time_of_last_gc(jlong now) {
 942     if (_collectorState <= Idling) {
 943       // gc not in progress
 944       return _time_of_last_gc;
 945     } else {
 946       // collection in progress
 947       return now;
 948     }
 949   }
 950 
 951   // Support for parallel remark of survivor space
 952   void* get_data_recorder(int thr_num);
 953 
 954   CMSBitMap* markBitMap()  { return &_markBitMap; }
 955   void directAllocated(HeapWord* start, size_t size);
 956 
 957   // main CMS steps and related support
 958   void checkpointRootsInitial(bool asynch);
 959   bool markFromRoots(bool asynch);  // a return value of false indicates failure
 960                                     // due to stack overflow
 961   void preclean();
 962   void checkpointRootsFinal(bool asynch, bool clear_all_soft_refs,
 963                             bool init_mark_was_synchronous);
 964   void sweep(bool asynch);
 965 
 966   // Check that the currently executing thread is the expected
 967   // one (foreground collector or background collector).
 968   static void check_correct_thread_executing() PRODUCT_RETURN;
 969   // XXXPERM void print_statistics()           PRODUCT_RETURN;
 970 
 971   bool is_cms_reachable(HeapWord* addr);
 972 
 973   // Performance Counter Support
 974   CollectorCounters* counters()    { return _gc_counters; }
 975 
 976   // timer stuff
 977   void    startTimer() { assert(!_timer.is_active(), "Error"); _timer.start();   }
 978   void    stopTimer()  { assert( _timer.is_active(), "Error"); _timer.stop();    }
 979   void    resetTimer() { assert(!_timer.is_active(), "Error"); _timer.reset();   }
 980   double  timerValue() { assert(!_timer.is_active(), "Error"); return _timer.seconds(); }
 981 
 982   int  yields()          { return _numYields; }
 983   void resetYields()     { _numYields = 0;    }
 984   void incrementYields() { _numYields++;      }
 985   void resetNumDirtyCards()               { _numDirtyCards = 0; }
 986   void incrementNumDirtyCards(size_t num) { _numDirtyCards += num; }
 987   size_t  numDirtyCards()                 { return _numDirtyCards; }
 988 
 989   static bool foregroundGCShouldWait() { return _foregroundGCShouldWait; }
 990   static void set_foregroundGCShouldWait(bool v) { _foregroundGCShouldWait = v; }
 991   static bool foregroundGCIsActive() { return _foregroundGCIsActive; }
 992   static void set_foregroundGCIsActive(bool v) { _foregroundGCIsActive = v; }
 993   size_t sweep_count() const             { return _sweep_count; }
 994   void   increment_sweep_count()         { _sweep_count++; }
 995 
 996   // Timers/stats for gc scheduling and incremental mode pacing.
 997   CMSStats& stats() { return _stats; }
 998 
 999   // Convenience methods that check whether CMSIncrementalMode is enabled and
1000   // forward to the corresponding methods in ConcurrentMarkSweepThread.
1001   static void start_icms();
1002   static void stop_icms();    // Called at the end of the cms cycle.
1003   static void disable_icms(); // Called before a foreground collection.
1004   static void enable_icms();  // Called after a foreground collection.
1005   void icms_wait();          // Called at yield points.
1006 
1007   // Adaptive size policy
1008   CMSAdaptiveSizePolicy* size_policy();
1009   CMSGCAdaptivePolicyCounters* gc_adaptive_policy_counters();
1010 
1011   static void print_on_error(outputStream* st);
1012 
1013   // debugging
1014   void verify();
1015   bool verify_after_remark(bool silent = VerifySilently);
1016   void verify_ok_to_terminate() const PRODUCT_RETURN;
1017   void verify_work_stacks_empty() const PRODUCT_RETURN;
1018   void verify_overflow_empty() const PRODUCT_RETURN;
1019 
1020   // convenience methods in support of debugging
1021   static const size_t skip_header_HeapWords() PRODUCT_RETURN0;
1022   HeapWord* block_start(const void* p) const PRODUCT_RETURN0;
1023 
1024   // accessors
1025   CMSMarkStack* verification_mark_stack() { return &_markStack; }
1026   CMSBitMap*    verification_mark_bm()    { return &_verification_mark_bm; }
1027 
1028   // Initialization errors
1029   bool completed_initialization() { return _completed_initialization; }
1030 };
1031 
1032 class CMSExpansionCause : public AllStatic  {
1033  public:
1034   enum Cause {
1035     _no_expansion,
1036     _satisfy_free_ratio,
1037     _satisfy_promotion,
1038     _satisfy_allocation,
1039     _allocate_par_lab,
1040     _allocate_par_spooling_space,
1041     _adaptive_size_policy
1042   };
1043   // Return a string describing the cause of the expansion.
1044   static const char* to_string(CMSExpansionCause::Cause cause);
1045 };
1046 
1047 class ConcurrentMarkSweepGeneration: public CardGeneration {
1048   friend class VMStructs;
1049   friend class ConcurrentMarkSweepThread;
1050   friend class ConcurrentMarkSweep;
1051   friend class CMSCollector;
1052  protected:
1053   static CMSCollector*       _collector; // the collector that collects us
1054   CompactibleFreeListSpace*  _cmsSpace;  // underlying space (only one for now)
1055 
1056   // Performance Counters
1057   GenerationCounters*      _gen_counters;
1058   GSpaceCounters*          _space_counters;
1059 
1060   // Words directly allocated, used by CMSStats.
1061   size_t _direct_allocated_words;
1062 
1063   // Non-product stat counters
1064   NOT_PRODUCT(
1065     size_t _numObjectsPromoted;
1066     size_t _numWordsPromoted;
1067     size_t _numObjectsAllocated;
1068     size_t _numWordsAllocated;
1069   )
1070 
1071   // Used for sizing decisions
1072   bool _incremental_collection_failed;
1073   bool incremental_collection_failed() {
1074     return _incremental_collection_failed;
1075   }
1076   void set_incremental_collection_failed() {
1077     _incremental_collection_failed = true;
1078   }
1079   void clear_incremental_collection_failed() {
1080     _incremental_collection_failed = false;
1081   }
1082 
1083   // accessors
1084   void set_expansion_cause(CMSExpansionCause::Cause v) { _expansion_cause = v;}
1085   CMSExpansionCause::Cause expansion_cause() const { return _expansion_cause; }
1086 
1087  private:
1088   // For parallel young-gen GC support.
1089   CMSParGCThreadState** _par_gc_thread_states;
1090 
1091   // Reason generation was expanded
1092   CMSExpansionCause::Cause _expansion_cause;
1093 
1094   // In support of MinChunkSize being larger than min object size
1095   const double _dilatation_factor;
1096 
1097   enum CollectionTypes {
1098     Concurrent_collection_type          = 0,
1099     MS_foreground_collection_type       = 1,
1100     MSC_foreground_collection_type      = 2,
1101     Unknown_collection_type             = 3
1102   };
1103 
1104   CollectionTypes _debug_collection_type;
1105 
1106   // True if a compactiing collection was done.
1107   bool _did_compact;
1108   bool did_compact() { return _did_compact; }
1109 
1110   // Fraction of current occupancy at which to start a CMS collection which
1111   // will collect this generation (at least).
1112   double _initiating_occupancy;
1113 
1114  protected:
1115   // Shrink generation by specified size (returns false if unable to shrink)
1116   void shrink_free_list_by(size_t bytes);
1117 
1118   // Update statistics for GC
1119   virtual void update_gc_stats(int level, bool full);
1120 
1121   // Maximum available space in the generation (including uncommitted)
1122   // space.
1123   size_t max_available() const;
1124 
1125   // getter and initializer for _initiating_occupancy field.
1126   double initiating_occupancy() const { return _initiating_occupancy; }
1127   void   init_initiating_occupancy(intx io, uintx tr);
1128 
1129  public:
1130   ConcurrentMarkSweepGeneration(ReservedSpace rs, size_t initial_byte_size,
1131                                 int level, CardTableRS* ct,
1132                                 bool use_adaptive_freelists,
1133                                 FreeBlockDictionary<FreeChunk>::DictionaryChoice);
1134 
1135   // Accessors
1136   CMSCollector* collector() const { return _collector; }
1137   static void set_collector(CMSCollector* collector) {
1138     assert(_collector == NULL, "already set");
1139     _collector = collector;
1140   }
1141   CompactibleFreeListSpace*  cmsSpace() const { return _cmsSpace;  }
1142 
1143   Mutex* freelistLock() const;
1144 
1145   virtual Generation::Name kind() { return Generation::ConcurrentMarkSweep; }
1146 
1147   // Adaptive size policy
1148   CMSAdaptiveSizePolicy* size_policy();
1149 
1150   void set_did_compact(bool v) { _did_compact = v; }
1151 
1152   bool refs_discovery_is_atomic() const { return false; }
1153   bool refs_discovery_is_mt()     const {
1154     // Note: CMS does MT-discovery during the parallel-remark
1155     // phases. Use ReferenceProcessorMTMutator to make refs
1156     // discovery MT-safe during such phases or other parallel
1157     // discovery phases in the future. This may all go away
1158     // if/when we decide that refs discovery is sufficiently
1159     // rare that the cost of the CAS's involved is in the
1160     // noise. That's a measurement that should be done, and
1161     // the code simplified if that turns out to be the case.
1162     return ConcGCThreads > 1;
1163   }
1164 
1165   // Override
1166   virtual void ref_processor_init();
1167 
1168   // Grow generation by specified size (returns false if unable to grow)
1169   bool grow_by(size_t bytes);
1170   // Grow generation to reserved size.
1171   bool grow_to_reserved();
1172 
1173   void clear_expansion_cause() { _expansion_cause = CMSExpansionCause::_no_expansion; }
1174 
1175   // Space enquiries
1176   size_t capacity() const;
1177   size_t used() const;
1178   size_t free() const;
1179   double occupancy() const { return ((double)used())/((double)capacity()); }
1180   size_t contiguous_available() const;
1181   size_t unsafe_max_alloc_nogc() const;
1182 
1183   // over-rides
1184   MemRegion used_region() const;
1185   MemRegion used_region_at_save_marks() const;
1186 
1187   // Does a "full" (forced) collection invoked on this generation collect
1188   // all younger generations as well? Note that the second conjunct is a
1189   // hack to allow the collection of the younger gen first if the flag is
1190   // set. This is better than using th policy's should_collect_gen0_first()
1191   // since that causes us to do an extra unnecessary pair of restart-&-stop-world.
1192   virtual bool full_collects_younger_generations() const {
1193     return UseCMSCompactAtFullCollection && !CollectGen0First;
1194   }
1195 
1196   void space_iterate(SpaceClosure* blk, bool usedOnly = false);
1197 
1198   // Support for compaction
1199   CompactibleSpace* first_compaction_space() const;
1200   // Adjust quantites in the generation affected by
1201   // the compaction.
1202   void reset_after_compaction();
1203 
1204   // Allocation support
1205   HeapWord* allocate(size_t size, bool tlab);
1206   HeapWord* have_lock_and_allocate(size_t size, bool tlab);
1207   oop       promote(oop obj, size_t obj_size);
1208   HeapWord* par_allocate(size_t size, bool tlab) {
1209     return allocate(size, tlab);
1210   }
1211 
1212   // Incremental mode triggering.
1213   HeapWord* allocation_limit_reached(Space* space, HeapWord* top,
1214                                      size_t word_size);
1215 
1216   // Used by CMSStats to track direct allocation.  The value is sampled and
1217   // reset after each young gen collection.
1218   size_t direct_allocated_words() const { return _direct_allocated_words; }
1219   void reset_direct_allocated_words()   { _direct_allocated_words = 0; }
1220 
1221   // Overrides for parallel promotion.
1222   virtual oop par_promote(int thread_num,
1223                           oop obj, markOop m, size_t word_sz);
1224   // This one should not be called for CMS.
1225   virtual void par_promote_alloc_undo(int thread_num,
1226                                       HeapWord* obj, size_t word_sz);
1227   virtual void par_promote_alloc_done(int thread_num);
1228   virtual void par_oop_since_save_marks_iterate_done(int thread_num);
1229 
1230   virtual bool promotion_attempt_is_safe(size_t promotion_in_bytes) const;
1231 
1232   // Inform this (non-young) generation that a promotion failure was
1233   // encountered during a collection of a younger generation that
1234   // promotes into this generation.
1235   virtual void promotion_failure_occurred();
1236 
1237   bool should_collect(bool full, size_t size, bool tlab);
1238   virtual bool should_concurrent_collect() const;
1239   virtual bool is_too_full() const;
1240   void collect(bool   full,
1241                bool   clear_all_soft_refs,
1242                size_t size,
1243                bool   tlab);
1244 
1245   HeapWord* expand_and_allocate(size_t word_size,
1246                                 bool tlab,
1247                                 bool parallel = false);
1248 
1249   // GC prologue and epilogue
1250   void gc_prologue(bool full);
1251   void gc_prologue_work(bool full, bool registerClosure,
1252                         ModUnionClosure* modUnionClosure);
1253   void gc_epilogue(bool full);
1254   void gc_epilogue_work(bool full);
1255 
1256   // Time since last GC of this generation
1257   jlong time_of_last_gc(jlong now) {
1258     return collector()->time_of_last_gc(now);
1259   }
1260   void update_time_of_last_gc(jlong now) {
1261     collector()-> update_time_of_last_gc(now);
1262   }
1263 
1264   // Allocation failure
1265   void expand(size_t bytes, size_t expand_bytes,
1266     CMSExpansionCause::Cause cause);
1267   virtual bool expand(size_t bytes, size_t expand_bytes);
1268   void shrink(size_t bytes);
1269   void shrink_by(size_t bytes);
1270   HeapWord* expand_and_par_lab_allocate(CMSParGCThreadState* ps, size_t word_sz);
1271   bool expand_and_ensure_spooling_space(PromotionInfo* promo);
1272 
1273   // Iteration support and related enquiries
1274   void save_marks();
1275   bool no_allocs_since_save_marks();
1276   void object_iterate_since_last_GC(ObjectClosure* cl);
1277   void younger_refs_iterate(OopsInGenClosure* cl);
1278 
1279   // Iteration support specific to CMS generations
1280   void save_sweep_limit();
1281 
1282   // More iteration support
1283   virtual void oop_iterate(MemRegion mr, ExtendedOopClosure* cl);
1284   virtual void oop_iterate(ExtendedOopClosure* cl);
1285   virtual void safe_object_iterate(ObjectClosure* cl);
1286   virtual void object_iterate(ObjectClosure* cl);
1287 
1288   // Need to declare the full complement of closures, whether we'll
1289   // override them or not, or get message from the compiler:
1290   //   oop_since_save_marks_iterate_nv hides virtual function...
1291   #define CMS_SINCE_SAVE_MARKS_DECL(OopClosureType, nv_suffix) \
1292     void oop_since_save_marks_iterate##nv_suffix(OopClosureType* cl);
1293   ALL_SINCE_SAVE_MARKS_CLOSURES(CMS_SINCE_SAVE_MARKS_DECL)
1294 
1295   // Smart allocation  XXX -- move to CFLSpace?
1296   void setNearLargestChunk();
1297   bool isNearLargestChunk(HeapWord* addr);
1298 
1299   // Get the chunk at the end of the space.  Delagates to
1300   // the space.
1301   FreeChunk* find_chunk_at_end();
1302 
1303   void post_compact();
1304 
1305   // Debugging
1306   void prepare_for_verify();
1307   void verify();
1308   void print_statistics()               PRODUCT_RETURN;
1309 
1310   // Performance Counters support
1311   virtual void update_counters();
1312   virtual void update_counters(size_t used);
1313   void initialize_performance_counters();
1314   CollectorCounters* counters()  { return collector()->counters(); }
1315 
1316   // Support for parallel remark of survivor space
1317   void* get_data_recorder(int thr_num) {
1318     //Delegate to collector
1319     return collector()->get_data_recorder(thr_num);
1320   }
1321 
1322   // Printing
1323   const char* name() const;
1324   virtual const char* short_name() const { return "CMS"; }
1325   void        print() const;
1326   void printOccupancy(const char* s);
1327   bool must_be_youngest() const { return false; }
1328   bool must_be_oldest()   const { return true; }
1329 
1330   // Resize the generation after a compacting GC.  The
1331   // generation can be treated as a contiguous space
1332   // after the compaction.
1333   virtual void compute_new_size();
1334   // Resize the generation after a non-compacting
1335   // collection.
1336   void compute_new_size_free_list();
1337 
1338   CollectionTypes debug_collection_type() { return _debug_collection_type; }
1339   void rotate_debug_collection_type();
1340 };
1341 
1342 class ASConcurrentMarkSweepGeneration : public ConcurrentMarkSweepGeneration {
1343 
1344   // Return the size policy from the heap's collector
1345   // policy casted to CMSAdaptiveSizePolicy*.
1346   CMSAdaptiveSizePolicy* cms_size_policy() const;
1347 
1348   // Resize the generation based on the adaptive size
1349   // policy.
1350   void resize(size_t cur_promo, size_t desired_promo);
1351 
1352   // Return the GC counters from the collector policy
1353   CMSGCAdaptivePolicyCounters* gc_adaptive_policy_counters();
1354 
1355   virtual void shrink_by(size_t bytes);
1356 
1357  public:
1358   ASConcurrentMarkSweepGeneration(ReservedSpace rs, size_t initial_byte_size,
1359                                   int level, CardTableRS* ct,
1360                                   bool use_adaptive_freelists,
1361                                   FreeBlockDictionary<FreeChunk>::DictionaryChoice
1362                                     dictionaryChoice) :
1363     ConcurrentMarkSweepGeneration(rs, initial_byte_size, level, ct,
1364       use_adaptive_freelists, dictionaryChoice) {}
1365 
1366   virtual const char* short_name() const { return "ASCMS"; }
1367   virtual Generation::Name kind() { return Generation::ASConcurrentMarkSweep; }
1368 
1369   virtual void update_counters();
1370   virtual void update_counters(size_t used);
1371 };
1372 
1373 //
1374 // Closures of various sorts used by CMS to accomplish its work
1375 //
1376 
1377 // This closure is used to check that a certain set of oops is empty.
1378 class FalseClosure: public OopClosure {
1379  public:
1380   void do_oop(oop* p)       { guarantee(false, "Should be an empty set"); }
1381   void do_oop(narrowOop* p) { guarantee(false, "Should be an empty set"); }
1382 };
1383 
1384 // This closure is used to do concurrent marking from the roots
1385 // following the first checkpoint.
1386 class MarkFromRootsClosure: public BitMapClosure {
1387   CMSCollector*  _collector;
1388   MemRegion      _span;
1389   CMSBitMap*     _bitMap;
1390   CMSBitMap*     _mut;
1391   CMSMarkStack*  _markStack;
1392   bool           _yield;
1393   int            _skipBits;
1394   HeapWord*      _finger;
1395   HeapWord*      _threshold;
1396   DEBUG_ONLY(bool _verifying;)
1397 
1398  public:
1399   MarkFromRootsClosure(CMSCollector* collector, MemRegion span,
1400                        CMSBitMap* bitMap,
1401                        CMSMarkStack*  markStack,
1402                        bool should_yield, bool verifying = false);
1403   bool do_bit(size_t offset);
1404   void reset(HeapWord* addr);
1405   inline void do_yield_check();
1406 
1407  private:
1408   void scanOopsInOop(HeapWord* ptr);
1409   void do_yield_work();
1410 };
1411 
1412 // This closure is used to do concurrent multi-threaded
1413 // marking from the roots following the first checkpoint.
1414 // XXX This should really be a subclass of The serial version
1415 // above, but i have not had the time to refactor things cleanly.
1416 // That willbe done for Dolphin.
1417 class Par_MarkFromRootsClosure: public BitMapClosure {
1418   CMSCollector*  _collector;
1419   MemRegion      _whole_span;
1420   MemRegion      _span;
1421   CMSBitMap*     _bit_map;
1422   CMSBitMap*     _mut;
1423   OopTaskQueue*  _work_queue;
1424   CMSMarkStack*  _overflow_stack;
1425   bool           _yield;
1426   int            _skip_bits;
1427   HeapWord*      _finger;
1428   HeapWord*      _threshold;
1429   CMSConcMarkingTask* _task;
1430  public:
1431   Par_MarkFromRootsClosure(CMSConcMarkingTask* task, CMSCollector* collector,
1432                        MemRegion span,
1433                        CMSBitMap* bit_map,
1434                        OopTaskQueue* work_queue,
1435                        CMSMarkStack*  overflow_stack,
1436                        bool should_yield);
1437   bool do_bit(size_t offset);
1438   inline void do_yield_check();
1439 
1440  private:
1441   void scan_oops_in_oop(HeapWord* ptr);
1442   void do_yield_work();
1443   bool get_work_from_overflow_stack();
1444 };
1445 
1446 // The following closures are used to do certain kinds of verification of
1447 // CMS marking.
1448 class PushAndMarkVerifyClosure: public CMSOopClosure {
1449   CMSCollector*    _collector;
1450   MemRegion        _span;
1451   CMSBitMap*       _verification_bm;
1452   CMSBitMap*       _cms_bm;
1453   CMSMarkStack*    _mark_stack;
1454  protected:
1455   void do_oop(oop p);
1456   template <class T> inline void do_oop_work(T *p) {
1457     oop obj = oopDesc::load_decode_heap_oop(p);
1458     do_oop(obj);
1459   }
1460  public:
1461   PushAndMarkVerifyClosure(CMSCollector* cms_collector,
1462                            MemRegion span,
1463                            CMSBitMap* verification_bm,
1464                            CMSBitMap* cms_bm,
1465                            CMSMarkStack*  mark_stack);
1466   void do_oop(oop* p);
1467   void do_oop(narrowOop* p);
1468 
1469   // Deal with a stack overflow condition
1470   void handle_stack_overflow(HeapWord* lost);
1471 };
1472 
1473 class MarkFromRootsVerifyClosure: public BitMapClosure {
1474   CMSCollector*  _collector;
1475   MemRegion      _span;
1476   CMSBitMap*     _verification_bm;
1477   CMSBitMap*     _cms_bm;
1478   CMSMarkStack*  _mark_stack;
1479   HeapWord*      _finger;
1480   PushAndMarkVerifyClosure _pam_verify_closure;
1481  public:
1482   MarkFromRootsVerifyClosure(CMSCollector* collector, MemRegion span,
1483                              CMSBitMap* verification_bm,
1484                              CMSBitMap* cms_bm,
1485                              CMSMarkStack*  mark_stack);
1486   bool do_bit(size_t offset);
1487   void reset(HeapWord* addr);
1488 };
1489 
1490 
1491 // This closure is used to check that a certain set of bits is
1492 // "empty" (i.e. the bit vector doesn't have any 1-bits).
1493 class FalseBitMapClosure: public BitMapClosure {
1494  public:
1495   bool do_bit(size_t offset) {
1496     guarantee(false, "Should not have a 1 bit");
1497     return true;
1498   }
1499 };
1500 
1501 // This closure is used during the second checkpointing phase
1502 // to rescan the marked objects on the dirty cards in the mod
1503 // union table and the card table proper. It's invoked via
1504 // MarkFromDirtyCardsClosure below. It uses either
1505 // [Par_]MarkRefsIntoAndScanClosure (Par_ in the parallel case)
1506 // declared in genOopClosures.hpp to accomplish some of its work.
1507 // In the parallel case the bitMap is shared, so access to
1508 // it needs to be suitably synchronized for updates by embedded
1509 // closures that update it; however, this closure itself only
1510 // reads the bit_map and because it is idempotent, is immune to
1511 // reading stale values.
1512 class ScanMarkedObjectsAgainClosure: public UpwardsObjectClosure {
1513   #ifdef ASSERT
1514     CMSCollector*          _collector;
1515     MemRegion              _span;
1516     union {
1517       CMSMarkStack*        _mark_stack;
1518       OopTaskQueue*        _work_queue;
1519     };
1520   #endif // ASSERT
1521   bool                       _parallel;
1522   CMSBitMap*                 _bit_map;
1523   union {
1524     MarkRefsIntoAndScanClosure*     _scan_closure;
1525     Par_MarkRefsIntoAndScanClosure* _par_scan_closure;
1526   };
1527 
1528  public:
1529   ScanMarkedObjectsAgainClosure(CMSCollector* collector,
1530                                 MemRegion span,
1531                                 ReferenceProcessor* rp,
1532                                 CMSBitMap* bit_map,
1533                                 CMSMarkStack*  mark_stack,
1534                                 MarkRefsIntoAndScanClosure* cl):
1535     #ifdef ASSERT
1536       _collector(collector),
1537       _span(span),
1538       _mark_stack(mark_stack),
1539     #endif // ASSERT
1540     _parallel(false),
1541     _bit_map(bit_map),
1542     _scan_closure(cl) { }
1543 
1544   ScanMarkedObjectsAgainClosure(CMSCollector* collector,
1545                                 MemRegion span,
1546                                 ReferenceProcessor* rp,
1547                                 CMSBitMap* bit_map,
1548                                 OopTaskQueue* work_queue,
1549                                 Par_MarkRefsIntoAndScanClosure* cl):
1550     #ifdef ASSERT
1551       _collector(collector),
1552       _span(span),
1553       _work_queue(work_queue),
1554     #endif // ASSERT
1555     _parallel(true),
1556     _bit_map(bit_map),
1557     _par_scan_closure(cl) { }
1558 
1559   bool do_object_b(oop obj) {
1560     guarantee(false, "Call do_object_b(oop, MemRegion) form instead");
1561     return false;
1562   }
1563   bool do_object_bm(oop p, MemRegion mr);
1564 };
1565 
1566 // This closure is used during the second checkpointing phase
1567 // to rescan the marked objects on the dirty cards in the mod
1568 // union table and the card table proper. It invokes
1569 // ScanMarkedObjectsAgainClosure above to accomplish much of its work.
1570 // In the parallel case, the bit map is shared and requires
1571 // synchronized access.
1572 class MarkFromDirtyCardsClosure: public MemRegionClosure {
1573   CompactibleFreeListSpace*      _space;
1574   ScanMarkedObjectsAgainClosure  _scan_cl;
1575   size_t                         _num_dirty_cards;
1576 
1577  public:
1578   MarkFromDirtyCardsClosure(CMSCollector* collector,
1579                             MemRegion span,
1580                             CompactibleFreeListSpace* space,
1581                             CMSBitMap* bit_map,
1582                             CMSMarkStack* mark_stack,
1583                             MarkRefsIntoAndScanClosure* cl):
1584     _space(space),
1585     _num_dirty_cards(0),
1586     _scan_cl(collector, span, collector->ref_processor(), bit_map,
1587                  mark_stack, cl) { }
1588 
1589   MarkFromDirtyCardsClosure(CMSCollector* collector,
1590                             MemRegion span,
1591                             CompactibleFreeListSpace* space,
1592                             CMSBitMap* bit_map,
1593                             OopTaskQueue* work_queue,
1594                             Par_MarkRefsIntoAndScanClosure* cl):
1595     _space(space),
1596     _num_dirty_cards(0),
1597     _scan_cl(collector, span, collector->ref_processor(), bit_map,
1598              work_queue, cl) { }
1599 
1600   void do_MemRegion(MemRegion mr);
1601   void set_space(CompactibleFreeListSpace* space) { _space = space; }
1602   size_t num_dirty_cards() { return _num_dirty_cards; }
1603 };
1604 
1605 // This closure is used in the non-product build to check
1606 // that there are no MemRegions with a certain property.
1607 class FalseMemRegionClosure: public MemRegionClosure {
1608   void do_MemRegion(MemRegion mr) {
1609     guarantee(!mr.is_empty(), "Shouldn't be empty");
1610     guarantee(false, "Should never be here");
1611   }
1612 };
1613 
1614 // This closure is used during the precleaning phase
1615 // to "carefully" rescan marked objects on dirty cards.
1616 // It uses MarkRefsIntoAndScanClosure declared in genOopClosures.hpp
1617 // to accomplish some of its work.
1618 class ScanMarkedObjectsAgainCarefullyClosure: public ObjectClosureCareful {
1619   CMSCollector*                  _collector;
1620   MemRegion                      _span;
1621   bool                           _yield;
1622   Mutex*                         _freelistLock;
1623   CMSBitMap*                     _bitMap;
1624   CMSMarkStack*                  _markStack;
1625   MarkRefsIntoAndScanClosure*    _scanningClosure;
1626 
1627  public:
1628   ScanMarkedObjectsAgainCarefullyClosure(CMSCollector* collector,
1629                                          MemRegion     span,
1630                                          CMSBitMap* bitMap,
1631                                          CMSMarkStack*  markStack,
1632                                          MarkRefsIntoAndScanClosure* cl,
1633                                          bool should_yield):
1634     _collector(collector),
1635     _span(span),
1636     _yield(should_yield),
1637     _bitMap(bitMap),
1638     _markStack(markStack),
1639     _scanningClosure(cl) {
1640   }
1641 
1642   void do_object(oop p) {
1643     guarantee(false, "call do_object_careful instead");
1644   }
1645 
1646   size_t      do_object_careful(oop p) {
1647     guarantee(false, "Unexpected caller");
1648     return 0;
1649   }
1650 
1651   size_t      do_object_careful_m(oop p, MemRegion mr);
1652 
1653   void setFreelistLock(Mutex* m) {
1654     _freelistLock = m;
1655     _scanningClosure->set_freelistLock(m);
1656   }
1657 
1658  private:
1659   inline bool do_yield_check();
1660 
1661   void do_yield_work();
1662 };
1663 
1664 class SurvivorSpacePrecleanClosure: public ObjectClosureCareful {
1665   CMSCollector*                  _collector;
1666   MemRegion                      _span;
1667   bool                           _yield;
1668   CMSBitMap*                     _bit_map;
1669   CMSMarkStack*                  _mark_stack;
1670   PushAndMarkClosure*            _scanning_closure;
1671   unsigned int                   _before_count;
1672 
1673  public:
1674   SurvivorSpacePrecleanClosure(CMSCollector* collector,
1675                                MemRegion     span,
1676                                CMSBitMap*    bit_map,
1677                                CMSMarkStack* mark_stack,
1678                                PushAndMarkClosure* cl,
1679                                unsigned int  before_count,
1680                                bool          should_yield):
1681     _collector(collector),
1682     _span(span),
1683     _yield(should_yield),
1684     _bit_map(bit_map),
1685     _mark_stack(mark_stack),
1686     _scanning_closure(cl),
1687     _before_count(before_count)
1688   { }
1689 
1690   void do_object(oop p) {
1691     guarantee(false, "call do_object_careful instead");
1692   }
1693 
1694   size_t      do_object_careful(oop p);
1695 
1696   size_t      do_object_careful_m(oop p, MemRegion mr) {
1697     guarantee(false, "Unexpected caller");
1698     return 0;
1699   }
1700 
1701  private:
1702   inline void do_yield_check();
1703   void do_yield_work();
1704 };
1705 
1706 // This closure is used to accomplish the sweeping work
1707 // after the second checkpoint but before the concurrent reset
1708 // phase.
1709 //
1710 // Terminology
1711 //   left hand chunk (LHC) - block of one or more chunks currently being
1712 //     coalesced.  The LHC is available for coalescing with a new chunk.
1713 //   right hand chunk (RHC) - block that is currently being swept that is
1714 //     free or garbage that can be coalesced with the LHC.
1715 // _inFreeRange is true if there is currently a LHC
1716 // _lastFreeRangeCoalesced is true if the LHC consists of more than one chunk.
1717 // _freeRangeInFreeLists is true if the LHC is in the free lists.
1718 // _freeFinger is the address of the current LHC
1719 class SweepClosure: public BlkClosureCareful {
1720   CMSCollector*                  _collector;  // collector doing the work
1721   ConcurrentMarkSweepGeneration* _g;    // Generation being swept
1722   CompactibleFreeListSpace*      _sp;   // Space being swept
1723   HeapWord*                      _limit;// the address at or above which the sweep should stop
1724                                         // because we do not expect newly garbage blocks
1725                                         // eligible for sweeping past that address.
1726   Mutex*                         _freelistLock; // Free list lock (in space)
1727   CMSBitMap*                     _bitMap;       // Marking bit map (in
1728                                                 // generation)
1729   bool                           _inFreeRange;  // Indicates if we are in the
1730                                                 // midst of a free run
1731   bool                           _freeRangeInFreeLists;
1732                                         // Often, we have just found
1733                                         // a free chunk and started
1734                                         // a new free range; we do not
1735                                         // eagerly remove this chunk from
1736                                         // the free lists unless there is
1737                                         // a possibility of coalescing.
1738                                         // When true, this flag indicates
1739                                         // that the _freeFinger below
1740                                         // points to a potentially free chunk
1741                                         // that may still be in the free lists
1742   bool                           _lastFreeRangeCoalesced;
1743                                         // free range contains chunks
1744                                         // coalesced
1745   bool                           _yield;
1746                                         // Whether sweeping should be
1747                                         // done with yields. For instance
1748                                         // when done by the foreground
1749                                         // collector we shouldn't yield.
1750   HeapWord*                      _freeFinger;   // When _inFreeRange is set, the
1751                                                 // pointer to the "left hand
1752                                                 // chunk"
1753   size_t                         _freeRangeSize;
1754                                         // When _inFreeRange is set, this
1755                                         // indicates the accumulated size
1756                                         // of the "left hand chunk"
1757   NOT_PRODUCT(
1758     size_t                       _numObjectsFreed;
1759     size_t                       _numWordsFreed;
1760     size_t                       _numObjectsLive;
1761     size_t                       _numWordsLive;
1762     size_t                       _numObjectsAlreadyFree;
1763     size_t                       _numWordsAlreadyFree;
1764     FreeChunk*                   _last_fc;
1765   )
1766  private:
1767   // Code that is common to a free chunk or garbage when
1768   // encountered during sweeping.
1769   void do_post_free_or_garbage_chunk(FreeChunk *fc, size_t chunkSize);
1770   // Process a free chunk during sweeping.
1771   void do_already_free_chunk(FreeChunk *fc);
1772   // Work method called when processing an already free or a
1773   // freshly garbage chunk to do a lookahead and possibly a
1774   // premptive flush if crossing over _limit.
1775   void lookahead_and_flush(FreeChunk* fc, size_t chunkSize);
1776   // Process a garbage chunk during sweeping.
1777   size_t do_garbage_chunk(FreeChunk *fc);
1778   // Process a live chunk during sweeping.
1779   size_t do_live_chunk(FreeChunk* fc);
1780 
1781   // Accessors.
1782   HeapWord* freeFinger() const          { return _freeFinger; }
1783   void set_freeFinger(HeapWord* v)      { _freeFinger = v; }
1784   bool inFreeRange()    const           { return _inFreeRange; }
1785   void set_inFreeRange(bool v)          { _inFreeRange = v; }
1786   bool lastFreeRangeCoalesced() const    { return _lastFreeRangeCoalesced; }
1787   void set_lastFreeRangeCoalesced(bool v) { _lastFreeRangeCoalesced = v; }
1788   bool freeRangeInFreeLists() const     { return _freeRangeInFreeLists; }
1789   void set_freeRangeInFreeLists(bool v) { _freeRangeInFreeLists = v; }
1790 
1791   // Initialize a free range.
1792   void initialize_free_range(HeapWord* freeFinger, bool freeRangeInFreeLists);
1793   // Return this chunk to the free lists.
1794   void flush_cur_free_chunk(HeapWord* chunk, size_t size);
1795 
1796   // Check if we should yield and do so when necessary.
1797   inline void do_yield_check(HeapWord* addr);
1798 
1799   // Yield
1800   void do_yield_work(HeapWord* addr);
1801 
1802   // Debugging/Printing
1803   void print_free_block_coalesced(FreeChunk* fc) const;
1804 
1805  public:
1806   SweepClosure(CMSCollector* collector, ConcurrentMarkSweepGeneration* g,
1807                CMSBitMap* bitMap, bool should_yield);
1808   ~SweepClosure() PRODUCT_RETURN;
1809 
1810   size_t       do_blk_careful(HeapWord* addr);
1811   void         print() const { print_on(tty); }
1812   void         print_on(outputStream *st) const;
1813 };
1814 
1815 // Closures related to weak references processing
1816 
1817 // During CMS' weak reference processing, this is a
1818 // work-routine/closure used to complete transitive
1819 // marking of objects as live after a certain point
1820 // in which an initial set has been completely accumulated.
1821 // This closure is currently used both during the final
1822 // remark stop-world phase, as well as during the concurrent
1823 // precleaning of the discovered reference lists.
1824 class CMSDrainMarkingStackClosure: public VoidClosure {
1825   CMSCollector*        _collector;
1826   MemRegion            _span;
1827   CMSMarkStack*        _mark_stack;
1828   CMSBitMap*           _bit_map;
1829   CMSKeepAliveClosure* _keep_alive;
1830   bool                 _concurrent_precleaning;
1831  public:
1832   CMSDrainMarkingStackClosure(CMSCollector* collector, MemRegion span,
1833                       CMSBitMap* bit_map, CMSMarkStack* mark_stack,
1834                       CMSKeepAliveClosure* keep_alive,
1835                       bool cpc):
1836     _collector(collector),
1837     _span(span),
1838     _bit_map(bit_map),
1839     _mark_stack(mark_stack),
1840     _keep_alive(keep_alive),
1841     _concurrent_precleaning(cpc) {
1842     assert(_concurrent_precleaning == _keep_alive->concurrent_precleaning(),
1843            "Mismatch");
1844   }
1845 
1846   void do_void();
1847 };
1848 
1849 // A parallel version of CMSDrainMarkingStackClosure above.
1850 class CMSParDrainMarkingStackClosure: public VoidClosure {
1851   CMSCollector*           _collector;
1852   MemRegion               _span;
1853   OopTaskQueue*           _work_queue;
1854   CMSBitMap*              _bit_map;
1855   CMSInnerParMarkAndPushClosure _mark_and_push;
1856 
1857  public:
1858   CMSParDrainMarkingStackClosure(CMSCollector* collector,
1859                                  MemRegion span, CMSBitMap* bit_map,
1860                                  OopTaskQueue* work_queue):
1861     _collector(collector),
1862     _span(span),
1863     _bit_map(bit_map),
1864     _work_queue(work_queue),
1865     _mark_and_push(collector, span, bit_map, work_queue) { }
1866 
1867  public:
1868   void trim_queue(uint max);
1869   void do_void();
1870 };
1871 
1872 // Allow yielding or short-circuiting of reference list
1873 // prelceaning work.
1874 class CMSPrecleanRefsYieldClosure: public YieldClosure {
1875   CMSCollector* _collector;
1876   void do_yield_work();
1877  public:
1878   CMSPrecleanRefsYieldClosure(CMSCollector* collector):
1879     _collector(collector) {}
1880   virtual bool should_return();
1881 };
1882 
1883 
1884 // Convenience class that locks free list locks for given CMS collector
1885 class FreelistLocker: public StackObj {
1886  private:
1887   CMSCollector* _collector;
1888  public:
1889   FreelistLocker(CMSCollector* collector):
1890     _collector(collector) {
1891     _collector->getFreelistLocks();
1892   }
1893 
1894   ~FreelistLocker() {
1895     _collector->releaseFreelistLocks();
1896   }
1897 };
1898 
1899 // Mark all dead objects in a given space.
1900 class MarkDeadObjectsClosure: public BlkClosure {
1901   const CMSCollector*             _collector;
1902   const CompactibleFreeListSpace* _sp;
1903   CMSBitMap*                      _live_bit_map;
1904   CMSBitMap*                      _dead_bit_map;
1905 public:
1906   MarkDeadObjectsClosure(const CMSCollector* collector,
1907                          const CompactibleFreeListSpace* sp,
1908                          CMSBitMap *live_bit_map,
1909                          CMSBitMap *dead_bit_map) :
1910     _collector(collector),
1911     _sp(sp),
1912     _live_bit_map(live_bit_map),
1913     _dead_bit_map(dead_bit_map) {}
1914   size_t do_blk(HeapWord* addr);
1915 };
1916 
1917 class TraceCMSMemoryManagerStats : public TraceMemoryManagerStats {
1918 
1919  public:
1920   TraceCMSMemoryManagerStats(CMSCollector::CollectorState phase, GCCause::Cause cause);
1921 };
1922 
1923 
1924 #endif // SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_CONCURRENTMARKSWEEPGENERATION_HPP