1 /*
2 * Copyright (c) 1998, 2020, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "classfile/vmSymbols.hpp"
27 #include "logging/log.hpp"
28 #include "logging/logStream.hpp"
29 #include "jfr/jfrEvents.hpp"
30 #include "memory/allocation.inline.hpp"
31 #include "memory/metaspaceShared.hpp"
32 #include "memory/padded.hpp"
33 #include "memory/resourceArea.hpp"
34 #include "memory/universe.hpp"
35 #include "oops/markWord.hpp"
36 #include "oops/oop.inline.hpp"
37 #include "runtime/atomic.hpp"
38 #include "runtime/biasedLocking.hpp"
39 #include "runtime/handles.inline.hpp"
40 #include "runtime/handshake.hpp"
41 #include "runtime/interfaceSupport.inline.hpp"
42 #include "runtime/mutexLocker.hpp"
43 #include "runtime/objectMonitor.hpp"
44 #include "runtime/objectMonitor.inline.hpp"
45 #include "runtime/osThread.hpp"
46 #include "runtime/safepointMechanism.inline.hpp"
47 #include "runtime/safepointVerifiers.hpp"
48 #include "runtime/sharedRuntime.hpp"
49 #include "runtime/stubRoutines.hpp"
50 #include "runtime/synchronizer.hpp"
51 #include "runtime/thread.inline.hpp"
52 #include "runtime/timer.hpp"
53 #include "runtime/vframe.hpp"
54 #include "runtime/vmThread.hpp"
55 #include "utilities/align.hpp"
56 #include "utilities/dtrace.hpp"
57 #include "utilities/events.hpp"
58 #include "utilities/preserveException.hpp"
59
60 // The "core" versions of monitor enter and exit reside in this file.
61 // The interpreter and compilers contain specialized transliterated
62 // variants of the enter-exit fast-path operations. See i486.ad fast_lock(),
63 // for instance. If you make changes here, make sure to modify the
64 // interpreter, and both C1 and C2 fast-path inline locking code emission.
65 //
66 // -----------------------------------------------------------------------------
67
68 #ifdef DTRACE_ENABLED
69
70 // Only bother with this argument setup if dtrace is available
71 // TODO-FIXME: probes should not fire when caller is _blocked. assert() accordingly.
72
73 #define DTRACE_MONITOR_PROBE_COMMON(obj, thread) \
74 char* bytes = NULL; \
75 int len = 0; \
76 jlong jtid = SharedRuntime::get_java_tid(thread); \
77 Symbol* klassname = ((oop)(obj))->klass()->name(); \
78 if (klassname != NULL) { \
79 bytes = (char*)klassname->bytes(); \
80 len = klassname->utf8_length(); \
81 }
82
83 #define DTRACE_MONITOR_WAIT_PROBE(monitor, obj, thread, millis) \
84 { \
85 if (DTraceMonitorProbes) { \
86 DTRACE_MONITOR_PROBE_COMMON(obj, thread); \
87 HOTSPOT_MONITOR_WAIT(jtid, \
88 (uintptr_t)(monitor), bytes, len, (millis)); \
89 } \
90 }
91
92 #define HOTSPOT_MONITOR_PROBE_notify HOTSPOT_MONITOR_NOTIFY
93 #define HOTSPOT_MONITOR_PROBE_notifyAll HOTSPOT_MONITOR_NOTIFYALL
94 #define HOTSPOT_MONITOR_PROBE_waited HOTSPOT_MONITOR_WAITED
95
96 #define DTRACE_MONITOR_PROBE(probe, monitor, obj, thread) \
97 { \
98 if (DTraceMonitorProbes) { \
99 DTRACE_MONITOR_PROBE_COMMON(obj, thread); \
100 HOTSPOT_MONITOR_PROBE_##probe(jtid, /* probe = waited */ \
101 (uintptr_t)(monitor), bytes, len); \
102 } \
103 }
104
105 #else // ndef DTRACE_ENABLED
106
107 #define DTRACE_MONITOR_WAIT_PROBE(obj, thread, millis, mon) {;}
108 #define DTRACE_MONITOR_PROBE(probe, obj, thread, mon) {;}
109
110 #endif // ndef DTRACE_ENABLED
111
112 // This exists only as a workaround of dtrace bug 6254741
113 int dtrace_waited_probe(ObjectMonitor* monitor, Handle obj, Thread* thr) {
114 DTRACE_MONITOR_PROBE(waited, monitor, obj(), thr);
115 return 0;
116 }
117
118 #define NINFLATIONLOCKS 256
119 static volatile intptr_t gInflationLocks[NINFLATIONLOCKS];
120
121 // global list of blocks of monitors
122 PaddedObjectMonitor* ObjectSynchronizer::g_block_list = NULL;
123 bool volatile ObjectSynchronizer::_is_async_deflation_requested = false;
124 jlong ObjectSynchronizer::_last_async_deflation_time_ns = 0;
125
126 struct ObjectMonitorListGlobals {
127 char _pad_prefix[OM_CACHE_LINE_SIZE];
128 // These are highly shared list related variables.
129 // To avoid false-sharing they need to be the sole occupants of a cache line.
130
131 // Global ObjectMonitor free list. Newly allocated and deflated
132 // ObjectMonitors are prepended here.
133 ObjectMonitor* _free_list;
134 DEFINE_PAD_MINUS_SIZE(1, OM_CACHE_LINE_SIZE, sizeof(ObjectMonitor*));
135
136 // Global ObjectMonitor in-use list. When a JavaThread is exiting,
137 // ObjectMonitors on its per-thread in-use list are prepended here.
138 ObjectMonitor* _in_use_list;
139 DEFINE_PAD_MINUS_SIZE(2, OM_CACHE_LINE_SIZE, sizeof(ObjectMonitor*));
140
141 // Global ObjectMonitor wait list. Deflated ObjectMonitors wait on
142 // this list until after a handshake or a safepoint for platforms
143 // that don't support handshakes. After the handshake or safepoint,
144 // the deflated ObjectMonitors are prepended to free_list.
145 ObjectMonitor* _wait_list;
146 DEFINE_PAD_MINUS_SIZE(3, OM_CACHE_LINE_SIZE, sizeof(ObjectMonitor*));
147
148 int _free_count; // # on free_list
149 DEFINE_PAD_MINUS_SIZE(4, OM_CACHE_LINE_SIZE, sizeof(int));
150
151 int _in_use_count; // # on in_use_list
152 DEFINE_PAD_MINUS_SIZE(5, OM_CACHE_LINE_SIZE, sizeof(int));
153
154 int _population; // # Extant -- in circulation
155 DEFINE_PAD_MINUS_SIZE(6, OM_CACHE_LINE_SIZE, sizeof(int));
156
157 int _wait_count; // # on wait_list
158 DEFINE_PAD_MINUS_SIZE(7, OM_CACHE_LINE_SIZE, sizeof(int));
159 };
160 static ObjectMonitorListGlobals om_list_globals;
161
162 #define CHAINMARKER (cast_to_oop<intptr_t>(-1))
163
164
165 // =====================> Spin-lock functions
166
167 // ObjectMonitors are not lockable outside of this file. We use spin-locks
168 // implemented using a bit in the _next_om field instead of the heavier
169 // weight locking mechanisms for faster list management.
170
171 #define OM_LOCK_BIT 0x1
172
173 // Return true if the ObjectMonitor is locked.
174 // Otherwise returns false.
175 static bool is_locked(ObjectMonitor* om) {
176 return ((intptr_t)om->next_om_acquire() & OM_LOCK_BIT) == OM_LOCK_BIT;
177 }
178
179 // Mark an ObjectMonitor* with OM_LOCK_BIT and return it.
180 static ObjectMonitor* mark_om_ptr(ObjectMonitor* om) {
181 return (ObjectMonitor*)((intptr_t)om | OM_LOCK_BIT);
182 }
183
184 // Return the unmarked next field in an ObjectMonitor. Note: the next
185 // field may or may not have been marked with OM_LOCK_BIT originally.
186 static ObjectMonitor* unmarked_next(ObjectMonitor* om) {
187 return (ObjectMonitor*)((intptr_t)om->next_om() & ~OM_LOCK_BIT);
188 }
189
190 // Try to lock an ObjectMonitor. Returns true if locking was successful.
191 // Otherwise returns false.
192 static bool try_om_lock(ObjectMonitor* om) {
193 // Get current next field without any OM_LOCK_BIT value.
194 ObjectMonitor* next = unmarked_next(om);
195 if (om->try_set_next_om(next, mark_om_ptr(next)) != next) {
196 return false; // Cannot lock the ObjectMonitor.
197 }
198 return true;
199 }
200
201 // Lock an ObjectMonitor.
202 static void om_lock(ObjectMonitor* om) {
203 while (true) {
204 if (try_om_lock(om)) {
205 return;
206 }
207 }
208 }
209
210 // Unlock an ObjectMonitor.
211 static void om_unlock(ObjectMonitor* om) {
212 ObjectMonitor* next = om->next_om();
213 guarantee(((intptr_t)next & OM_LOCK_BIT) == OM_LOCK_BIT, "next=" INTPTR_FORMAT
214 " must have OM_LOCK_BIT=%x set.", p2i(next), OM_LOCK_BIT);
215
216 next = (ObjectMonitor*)((intptr_t)next & ~OM_LOCK_BIT); // Clear OM_LOCK_BIT.
217 om->release_set_next_om(next);
218 }
219
220 // Get the list head after locking it. Returns the list head or NULL
221 // if the list is empty.
222 static ObjectMonitor* get_list_head_locked(ObjectMonitor** list_p) {
223 while (true) {
224 // Acquire semantics not needed on this list load since we're
225 // checking for NULL here or following up with a cmpxchg() via
226 // try_om_lock() below and we retry on cmpxchg() failure.
227 ObjectMonitor* mid = Atomic::load(list_p);
228 if (mid == NULL) {
229 return NULL; // The list is empty.
230 }
231 if (try_om_lock(mid)) {
232 // Acquire semantics not needed on this list load since memory is
233 // already consistent due to the cmpxchg() via try_om_lock() above.
234 if (Atomic::load(list_p) != mid) {
235 // The list head changed before we could lock it so we have to retry.
236 om_unlock(mid);
237 continue;
238 }
239 return mid;
240 }
241 }
242 }
243
244 #undef OM_LOCK_BIT
245
246
247 // =====================> List Management functions
248
249 // Prepend a list of ObjectMonitors to the specified *list_p. 'tail' is
250 // the last ObjectMonitor in the list and there are 'count' on the list.
251 // Also updates the specified *count_p.
252 static void prepend_list_to_common(ObjectMonitor* list, ObjectMonitor* tail,
253 int count, ObjectMonitor** list_p,
254 int* count_p) {
255 while (true) {
256 // Acquire semantics not needed on this list load since we're
257 // following up with a cmpxchg() via try_om_lock() below and we
258 // retry on cmpxchg() failure.
259 ObjectMonitor* cur = Atomic::load(list_p);
260 // Prepend list to *list_p.
261 if (!try_om_lock(tail)) {
262 // Failed to lock tail due to a list walker so try it all again.
263 continue;
264 }
265 // Release semantics not needed on this "unlock" since memory is
266 // already consistent due to the cmpxchg() via try_om_lock() above.
267 tail->set_next_om(cur); // tail now points to cur (and unlocks tail)
268 if (cur == NULL) {
269 // No potential race with takers or other prependers since
270 // *list_p is empty.
271 if (Atomic::cmpxchg(list_p, cur, list) == cur) {
272 // Successfully switched *list_p to the list value.
273 Atomic::add(count_p, count);
274 break;
275 }
276 // Implied else: try it all again
277 } else {
278 if (!try_om_lock(cur)) {
279 continue; // failed to lock cur so try it all again
280 }
281 // We locked cur so try to switch *list_p to the list value.
282 if (Atomic::cmpxchg(list_p, cur, list) != cur) {
283 // The list head has changed so unlock cur and try again:
284 om_unlock(cur);
285 continue;
286 }
287 Atomic::add(count_p, count);
288 om_unlock(cur);
289 break;
290 }
291 }
292 }
293
294 // Prepend a newly allocated block of ObjectMonitors to g_block_list and
295 // om_list_globals._free_list. Also updates om_list_globals._population
296 // and om_list_globals._free_count.
297 void ObjectSynchronizer::prepend_block_to_lists(PaddedObjectMonitor* new_blk) {
298 // First we handle g_block_list:
299 while (true) {
300 PaddedObjectMonitor* cur = Atomic::load(&g_block_list);
301 // Prepend new_blk to g_block_list. The first ObjectMonitor in
302 // a block is reserved for use as linkage to the next block.
303 new_blk[0].set_next_om(cur);
304 if (Atomic::cmpxchg(&g_block_list, cur, new_blk) == cur) {
305 // Successfully switched g_block_list to the new_blk value.
306 Atomic::add(&om_list_globals._population, _BLOCKSIZE - 1);
307 break;
308 }
309 // Implied else: try it all again
310 }
311
312 // Second we handle om_list_globals._free_list:
313 prepend_list_to_common(new_blk + 1, &new_blk[_BLOCKSIZE - 1], _BLOCKSIZE - 1,
314 &om_list_globals._free_list, &om_list_globals._free_count);
315 }
316
317 // Prepend a list of ObjectMonitors to om_list_globals._free_list.
318 // 'tail' is the last ObjectMonitor in the list and there are 'count'
319 // on the list. Also updates om_list_globals._free_count.
320 static void prepend_list_to_global_free_list(ObjectMonitor* list,
321 ObjectMonitor* tail, int count) {
322 prepend_list_to_common(list, tail, count, &om_list_globals._free_list,
323 &om_list_globals._free_count);
324 }
325
326 // Prepend a list of ObjectMonitors to om_list_globals._wait_list.
327 // 'tail' is the last ObjectMonitor in the list and there are 'count'
328 // on the list. Also updates om_list_globals._wait_count.
329 static void prepend_list_to_global_wait_list(ObjectMonitor* list,
330 ObjectMonitor* tail, int count) {
331 prepend_list_to_common(list, tail, count, &om_list_globals._wait_list,
332 &om_list_globals._wait_count);
333 }
334
335 // Prepend a list of ObjectMonitors to om_list_globals._in_use_list.
336 // 'tail' is the last ObjectMonitor in the list and there are 'count'
337 // on the list. Also updates om_list_globals._in_use_list.
338 static void prepend_list_to_global_in_use_list(ObjectMonitor* list,
339 ObjectMonitor* tail, int count) {
340 prepend_list_to_common(list, tail, count, &om_list_globals._in_use_list,
341 &om_list_globals._in_use_count);
342 }
343
344 // Prepend an ObjectMonitor to the specified list. Also updates
345 // the specified counter.
346 static void prepend_to_common(ObjectMonitor* m, ObjectMonitor** list_p,
347 int* count_p) {
348 while (true) {
349 om_lock(m); // Lock m so we can safely update its next field.
350 ObjectMonitor* cur = NULL;
351 // Lock the list head to guard against races with a list walker
352 // or async deflater thread (which only races in om_in_use_list):
353 if ((cur = get_list_head_locked(list_p)) != NULL) {
354 // List head is now locked so we can safely switch it. Release
355 // semantics not needed on this "unlock" since memory is already
356 // consistent due to the cmpxchg() via get_list_head_locked() above.
357 m->set_next_om(cur); // m now points to cur (and unlocks m)
358 OrderAccess::storestore(); // Make sure set_next_om() is seen first.
359 Atomic::store(list_p, m); // Switch list head to unlocked m.
360 om_unlock(cur);
361 break;
362 }
363 // The list is empty so try to set the list head.
364 assert(cur == NULL, "cur must be NULL: cur=" INTPTR_FORMAT, p2i(cur));
365 // Release semantics not needed on this "unlock" since memory
366 // is already consistent.
367 m->set_next_om(cur); // m now points to NULL (and unlocks m)
368 if (Atomic::cmpxchg(list_p, cur, m) == cur) {
369 // List head is now unlocked m.
370 break;
371 }
372 // Implied else: try it all again
373 }
374 Atomic::inc(count_p);
375 }
376
377 // Prepend an ObjectMonitor to a per-thread om_free_list.
378 // Also updates the per-thread om_free_count.
379 static void prepend_to_om_free_list(Thread* self, ObjectMonitor* m) {
380 prepend_to_common(m, &self->om_free_list, &self->om_free_count);
381 }
382
383 // Prepend an ObjectMonitor to a per-thread om_in_use_list.
384 // Also updates the per-thread om_in_use_count.
385 static void prepend_to_om_in_use_list(Thread* self, ObjectMonitor* m) {
386 prepend_to_common(m, &self->om_in_use_list, &self->om_in_use_count);
387 }
388
389 // Take an ObjectMonitor from the start of the specified list. Also
390 // decrements the specified counter. Returns NULL if none are available.
391 static ObjectMonitor* take_from_start_of_common(ObjectMonitor** list_p,
392 int* count_p) {
393 ObjectMonitor* take = NULL;
394 // Lock the list head to guard against races with a list walker
395 // or async deflater thread (which only races in om_list_globals._free_list):
396 if ((take = get_list_head_locked(list_p)) == NULL) {
397 return NULL; // None are available.
398 }
399 ObjectMonitor* next = unmarked_next(take);
400 // Switch locked list head to next (which unlocks the list head, but
401 // leaves take locked). Release semantics not needed on this "unlock"
402 // since memory is already consistent due to the cmpxchg() via
403 // get_list_head_locked() above.
404 Atomic::store(list_p, next);
405 Atomic::dec(count_p);
406 // Unlock take, but leave the next value for any lagging list
407 // walkers. It will get cleaned up when take is prepended to
408 // the in-use list:
409 om_unlock(take);
410 return take;
411 }
412
413 // Take an ObjectMonitor from the start of the om_list_globals._free_list.
414 // Also updates om_list_globals._free_count. Returns NULL if none are
415 // available.
416 static ObjectMonitor* take_from_start_of_global_free_list() {
417 return take_from_start_of_common(&om_list_globals._free_list,
418 &om_list_globals._free_count);
419 }
420
421 // Take an ObjectMonitor from the start of a per-thread free-list.
422 // Also updates om_free_count. Returns NULL if none are available.
423 static ObjectMonitor* take_from_start_of_om_free_list(Thread* self) {
424 return take_from_start_of_common(&self->om_free_list, &self->om_free_count);
425 }
426
427
428 // =====================> Quick functions
429
430 // The quick_* forms are special fast-path variants used to improve
431 // performance. In the simplest case, a "quick_*" implementation could
432 // simply return false, in which case the caller will perform the necessary
433 // state transitions and call the slow-path form.
434 // The fast-path is designed to handle frequently arising cases in an efficient
435 // manner and is just a degenerate "optimistic" variant of the slow-path.
436 // returns true -- to indicate the call was satisfied.
437 // returns false -- to indicate the call needs the services of the slow-path.
438 // A no-loitering ordinance is in effect for code in the quick_* family
439 // operators: safepoints or indefinite blocking (blocking that might span a
440 // safepoint) are forbidden. Generally the thread_state() is _in_Java upon
441 // entry.
442 //
443 // Consider: An interesting optimization is to have the JIT recognize the
444 // following common idiom:
445 // synchronized (someobj) { .... ; notify(); }
446 // That is, we find a notify() or notifyAll() call that immediately precedes
447 // the monitorexit operation. In that case the JIT could fuse the operations
448 // into a single notifyAndExit() runtime primitive.
449
450 bool ObjectSynchronizer::quick_notify(oopDesc* obj, Thread* self, bool all) {
451 assert(!SafepointSynchronize::is_at_safepoint(), "invariant");
452 assert(self->is_Java_thread(), "invariant");
453 assert(((JavaThread *) self)->thread_state() == _thread_in_Java, "invariant");
454 NoSafepointVerifier nsv;
455 if (obj == NULL) return false; // slow-path for invalid obj
456 const markWord mark = obj->mark();
457
458 if (mark.has_locker() && self->is_lock_owned((address)mark.locker())) {
459 // Degenerate notify
460 // stack-locked by caller so by definition the implied waitset is empty.
461 return true;
462 }
463
464 if (mark.has_monitor()) {
465 ObjectMonitor* const mon = mark.monitor();
466 assert(mon->object() == obj, "invariant");
467 if (mon->owner() != self) return false; // slow-path for IMS exception
468
469 if (mon->first_waiter() != NULL) {
470 // We have one or more waiters. Since this is an inflated monitor
471 // that we own, we can transfer one or more threads from the waitset
472 // to the entrylist here and now, avoiding the slow-path.
473 if (all) {
474 DTRACE_MONITOR_PROBE(notifyAll, mon, obj, self);
475 } else {
476 DTRACE_MONITOR_PROBE(notify, mon, obj, self);
477 }
478 int free_count = 0;
479 do {
480 mon->INotify(self);
481 ++free_count;
482 } while (mon->first_waiter() != NULL && all);
483 OM_PERFDATA_OP(Notifications, inc(free_count));
484 }
485 return true;
486 }
487
488 // biased locking and any other IMS exception states take the slow-path
489 return false;
490 }
491
492
493 // The LockNode emitted directly at the synchronization site would have
494 // been too big if it were to have included support for the cases of inflated
495 // recursive enter and exit, so they go here instead.
496 // Note that we can't safely call AsyncPrintJavaStack() from within
497 // quick_enter() as our thread state remains _in_Java.
498
499 bool ObjectSynchronizer::quick_enter(oop obj, Thread* self,
500 BasicLock * lock) {
501 assert(!SafepointSynchronize::is_at_safepoint(), "invariant");
502 assert(self->is_Java_thread(), "invariant");
503 assert(((JavaThread *) self)->thread_state() == _thread_in_Java, "invariant");
504 NoSafepointVerifier nsv;
505 if (obj == NULL) return false; // Need to throw NPE
506
507 const markWord mark = obj->mark();
508
509 if (mark.has_monitor()) {
510 ObjectMonitor* const m = mark.monitor();
511 // An async deflation can race us before we manage to make the
512 // ObjectMonitor busy by setting the owner below. If we detect
513 // that race we just bail out to the slow-path here.
514 if (m->object() == NULL) {
515 return false;
516 }
517 Thread* const owner = (Thread *) m->_owner;
518
519 // Lock contention and Transactional Lock Elision (TLE) diagnostics
520 // and observability
521 // Case: light contention possibly amenable to TLE
522 // Case: TLE inimical operations such as nested/recursive synchronization
523
524 if (owner == self) {
525 m->_recursions++;
526 return true;
527 }
528
529 // This Java Monitor is inflated so obj's header will never be
530 // displaced to this thread's BasicLock. Make the displaced header
531 // non-NULL so this BasicLock is not seen as recursive nor as
532 // being locked. We do this unconditionally so that this thread's
533 // BasicLock cannot be mis-interpreted by any stack walkers. For
534 // performance reasons, stack walkers generally first check for
535 // Biased Locking in the object's header, the second check is for
536 // stack-locking in the object's header, the third check is for
537 // recursive stack-locking in the displaced header in the BasicLock,
538 // and last are the inflated Java Monitor (ObjectMonitor) checks.
539 lock->set_displaced_header(markWord::unused_mark());
540
541 if (owner == NULL && m->try_set_owner_from(NULL, self) == NULL) {
542 assert(m->_recursions == 0, "invariant");
543 return true;
544 }
545 }
546
547 // Note that we could inflate in quick_enter.
548 // This is likely a useful optimization
549 // Critically, in quick_enter() we must not:
550 // -- perform bias revocation, or
551 // -- block indefinitely, or
552 // -- reach a safepoint
553
554 return false; // revert to slow-path
555 }
556
557 // -----------------------------------------------------------------------------
558 // Monitor Enter/Exit
559 // The interpreter and compiler assembly code tries to lock using the fast path
560 // of this algorithm. Make sure to update that code if the following function is
561 // changed. The implementation is extremely sensitive to race condition. Be careful.
562
563 void ObjectSynchronizer::enter(Handle obj, BasicLock* lock, TRAPS) {
564 if (UseBiasedLocking) {
565 if (!SafepointSynchronize::is_at_safepoint()) {
566 BiasedLocking::revoke(obj, THREAD);
567 } else {
568 BiasedLocking::revoke_at_safepoint(obj);
569 }
570 }
571
572 markWord mark = obj->mark();
573 assert(!mark.has_bias_pattern(), "should not see bias pattern here");
574
575 if (mark.is_neutral()) {
576 // Anticipate successful CAS -- the ST of the displaced mark must
577 // be visible <= the ST performed by the CAS.
578 lock->set_displaced_header(mark);
579 if (mark == obj()->cas_set_mark(markWord::from_pointer(lock), mark)) {
580 return;
581 }
582 // Fall through to inflate() ...
583 } else if (mark.has_locker() &&
584 THREAD->is_lock_owned((address)mark.locker())) {
585 assert(lock != mark.locker(), "must not re-lock the same lock");
586 assert(lock != (BasicLock*)obj->mark().value(), "don't relock with same BasicLock");
587 lock->set_displaced_header(markWord::from_pointer(NULL));
588 return;
589 }
590
591 // The object header will never be displaced to this lock,
592 // so it does not matter what the value is, except that it
593 // must be non-zero to avoid looking like a re-entrant lock,
594 // and must not look locked either.
595 lock->set_displaced_header(markWord::unused_mark());
596 // An async deflation can race after the inflate() call and before
597 // enter() can make the ObjectMonitor busy. enter() returns false if
598 // we have lost the race to async deflation and we simply try again.
599 while (true) {
600 ObjectMonitor* monitor = inflate(THREAD, obj(), inflate_cause_monitor_enter);
601 if (monitor->enter(THREAD)) {
602 return;
603 }
604 }
605 }
606
607 void ObjectSynchronizer::exit(oop object, BasicLock* lock, TRAPS) {
608 markWord mark = object->mark();
609 // We cannot check for Biased Locking if we are racing an inflation.
610 assert(mark == markWord::INFLATING() ||
611 !mark.has_bias_pattern(), "should not see bias pattern here");
612
613 markWord dhw = lock->displaced_header();
614 if (dhw.value() == 0) {
615 // If the displaced header is NULL, then this exit matches up with
616 // a recursive enter. No real work to do here except for diagnostics.
617 #ifndef PRODUCT
618 if (mark != markWord::INFLATING()) {
619 // Only do diagnostics if we are not racing an inflation. Simply
620 // exiting a recursive enter of a Java Monitor that is being
621 // inflated is safe; see the has_monitor() comment below.
622 assert(!mark.is_neutral(), "invariant");
623 assert(!mark.has_locker() ||
624 THREAD->is_lock_owned((address)mark.locker()), "invariant");
625 if (mark.has_monitor()) {
626 // The BasicLock's displaced_header is marked as a recursive
627 // enter and we have an inflated Java Monitor (ObjectMonitor).
628 // This is a special case where the Java Monitor was inflated
629 // after this thread entered the stack-lock recursively. When a
630 // Java Monitor is inflated, we cannot safely walk the Java
631 // Monitor owner's stack and update the BasicLocks because a
632 // Java Monitor can be asynchronously inflated by a thread that
633 // does not own the Java Monitor.
634 ObjectMonitor* m = mark.monitor();
635 assert(((oop)(m->object()))->mark() == mark, "invariant");
636 assert(m->is_entered(THREAD), "invariant");
637 }
638 }
639 #endif
640 return;
641 }
642
643 if (mark == markWord::from_pointer(lock)) {
644 // If the object is stack-locked by the current thread, try to
645 // swing the displaced header from the BasicLock back to the mark.
646 assert(dhw.is_neutral(), "invariant");
647 if (object->cas_set_mark(dhw, mark) == mark) {
648 return;
649 }
650 }
651
652 // We have to take the slow-path of possible inflation and then exit.
653 // The ObjectMonitor* can't be async deflated until ownership is
654 // dropped inside exit() and the ObjectMonitor* must be !is_busy().
655 ObjectMonitor* monitor = inflate(THREAD, object, inflate_cause_vm_internal);
656 monitor->exit(true, THREAD);
657 }
658
659 // -----------------------------------------------------------------------------
660 // Class Loader support to workaround deadlocks on the class loader lock objects
661 // Also used by GC
662 // complete_exit()/reenter() are used to wait on a nested lock
663 // i.e. to give up an outer lock completely and then re-enter
664 // Used when holding nested locks - lock acquisition order: lock1 then lock2
665 // 1) complete_exit lock1 - saving recursion count
666 // 2) wait on lock2
667 // 3) when notified on lock2, unlock lock2
668 // 4) reenter lock1 with original recursion count
669 // 5) lock lock2
670 // NOTE: must use heavy weight monitor to handle complete_exit/reenter()
671 intx ObjectSynchronizer::complete_exit(Handle obj, TRAPS) {
672 if (UseBiasedLocking) {
673 BiasedLocking::revoke(obj, THREAD);
674 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
675 }
676
677 // The ObjectMonitor* can't be async deflated until ownership is
678 // dropped inside exit() and the ObjectMonitor* must be !is_busy().
679 ObjectMonitor* monitor = inflate(THREAD, obj(), inflate_cause_vm_internal);
680 intptr_t ret_code = monitor->complete_exit(THREAD);
681 return ret_code;
682 }
683
684 // NOTE: must use heavy weight monitor to handle complete_exit/reenter()
685 void ObjectSynchronizer::reenter(Handle obj, intx recursions, TRAPS) {
686 if (UseBiasedLocking) {
687 BiasedLocking::revoke(obj, THREAD);
688 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
689 }
690
691 // An async deflation can race after the inflate() call and before
692 // reenter() -> enter() can make the ObjectMonitor busy. reenter() ->
693 // enter() returns false if we have lost the race to async deflation
694 // and we simply try again.
695 while (true) {
696 ObjectMonitor* monitor = inflate(THREAD, obj(), inflate_cause_vm_internal);
697 if (monitor->reenter(recursions, THREAD)) {
698 return;
699 }
700 }
701 }
702
703 // -----------------------------------------------------------------------------
704 // JNI locks on java objects
705 // NOTE: must use heavy weight monitor to handle jni monitor enter
706 void ObjectSynchronizer::jni_enter(Handle obj, TRAPS) {
707 // the current locking is from JNI instead of Java code
708 if (UseBiasedLocking) {
709 BiasedLocking::revoke(obj, THREAD);
710 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
711 }
712 THREAD->set_current_pending_monitor_is_from_java(false);
713 // An async deflation can race after the inflate() call and before
714 // enter() can make the ObjectMonitor busy. enter() returns false if
715 // we have lost the race to async deflation and we simply try again.
716 while (true) {
717 ObjectMonitor* monitor = inflate(THREAD, obj(), inflate_cause_jni_enter);
718 if (monitor->enter(THREAD)) {
719 break;
720 }
721 }
722 THREAD->set_current_pending_monitor_is_from_java(true);
723 }
724
725 // NOTE: must use heavy weight monitor to handle jni monitor exit
726 void ObjectSynchronizer::jni_exit(oop obj, Thread* THREAD) {
727 if (UseBiasedLocking) {
728 Handle h_obj(THREAD, obj);
729 BiasedLocking::revoke(h_obj, THREAD);
730 obj = h_obj();
731 }
732 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
733
734 // The ObjectMonitor* can't be async deflated until ownership is
735 // dropped inside exit() and the ObjectMonitor* must be !is_busy().
736 ObjectMonitor* monitor = inflate(THREAD, obj, inflate_cause_jni_exit);
737 // If this thread has locked the object, exit the monitor. We
738 // intentionally do not use CHECK here because we must exit the
739 // monitor even if an exception is pending.
740 if (monitor->check_owner(THREAD)) {
741 monitor->exit(true, THREAD);
742 }
743 }
744
745 // -----------------------------------------------------------------------------
746 // Internal VM locks on java objects
747 // standard constructor, allows locking failures
748 ObjectLocker::ObjectLocker(Handle obj, Thread* thread, bool do_lock) {
749 _dolock = do_lock;
750 _thread = thread;
751 _thread->check_for_valid_safepoint_state();
752 _obj = obj;
753
754 if (_dolock) {
755 ObjectSynchronizer::enter(_obj, &_lock, _thread);
756 }
757 }
758
759 ObjectLocker::~ObjectLocker() {
760 if (_dolock) {
761 ObjectSynchronizer::exit(_obj(), &_lock, _thread);
762 }
763 }
764
765
766 // -----------------------------------------------------------------------------
767 // Wait/Notify/NotifyAll
768 // NOTE: must use heavy weight monitor to handle wait()
769 int ObjectSynchronizer::wait(Handle obj, jlong millis, TRAPS) {
770 if (UseBiasedLocking) {
771 BiasedLocking::revoke(obj, THREAD);
772 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
773 }
774 if (millis < 0) {
775 THROW_MSG_0(vmSymbols::java_lang_IllegalArgumentException(), "timeout value is negative");
776 }
777 // The ObjectMonitor* can't be async deflated because the _waiters
778 // field is incremented before ownership is dropped and decremented
779 // after ownership is regained.
780 ObjectMonitor* monitor = inflate(THREAD, obj(), inflate_cause_wait);
781
782 DTRACE_MONITOR_WAIT_PROBE(monitor, obj(), THREAD, millis);
783 monitor->wait(millis, true, THREAD);
784
785 // This dummy call is in place to get around dtrace bug 6254741. Once
786 // that's fixed we can uncomment the following line, remove the call
787 // and change this function back into a "void" func.
788 // DTRACE_MONITOR_PROBE(waited, monitor, obj(), THREAD);
789 int ret_code = dtrace_waited_probe(monitor, obj, THREAD);
790 return ret_code;
791 }
792
793 void ObjectSynchronizer::wait_uninterruptibly(Handle obj, jlong millis, TRAPS) {
794 if (UseBiasedLocking) {
795 BiasedLocking::revoke(obj, THREAD);
796 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
797 }
798 if (millis < 0) {
799 THROW_MSG(vmSymbols::java_lang_IllegalArgumentException(), "timeout value is negative");
800 }
801 // The ObjectMonitor* can't be async deflated because the _waiters
802 // field is incremented before ownership is dropped and decremented
803 // after ownership is regained.
804 ObjectMonitor* monitor = inflate(THREAD, obj(), inflate_cause_wait);
805 monitor->wait(millis, false, THREAD);
806 }
807
808 void ObjectSynchronizer::notify(Handle obj, TRAPS) {
809 if (UseBiasedLocking) {
810 BiasedLocking::revoke(obj, THREAD);
811 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
812 }
813
814 markWord mark = obj->mark();
815 if (mark.has_locker() && THREAD->is_lock_owned((address)mark.locker())) {
816 return;
817 }
818 // The ObjectMonitor* can't be async deflated until ownership is
819 // dropped by the calling thread.
820 ObjectMonitor* monitor = inflate(THREAD, obj(), inflate_cause_notify);
821 monitor->notify(THREAD);
822 }
823
824 // NOTE: see comment of notify()
825 void ObjectSynchronizer::notifyall(Handle obj, TRAPS) {
826 if (UseBiasedLocking) {
827 BiasedLocking::revoke(obj, THREAD);
828 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
829 }
830
831 markWord mark = obj->mark();
832 if (mark.has_locker() && THREAD->is_lock_owned((address)mark.locker())) {
833 return;
834 }
835 // The ObjectMonitor* can't be async deflated until ownership is
836 // dropped by the calling thread.
837 ObjectMonitor* monitor = inflate(THREAD, obj(), inflate_cause_notify);
838 monitor->notifyAll(THREAD);
839 }
840
841 // -----------------------------------------------------------------------------
842 // Hash Code handling
843 //
844 // Performance concern:
845 // OrderAccess::storestore() calls release() which at one time stored 0
846 // into the global volatile OrderAccess::dummy variable. This store was
847 // unnecessary for correctness. Many threads storing into a common location
848 // causes considerable cache migration or "sloshing" on large SMP systems.
849 // As such, I avoided using OrderAccess::storestore(). In some cases
850 // OrderAccess::fence() -- which incurs local latency on the executing
851 // processor -- is a better choice as it scales on SMP systems.
852 //
853 // See http://blogs.oracle.com/dave/entry/biased_locking_in_hotspot for
854 // a discussion of coherency costs. Note that all our current reference
855 // platforms provide strong ST-ST order, so the issue is moot on IA32,
856 // x64, and SPARC.
857 //
858 // As a general policy we use "volatile" to control compiler-based reordering
859 // and explicit fences (barriers) to control for architectural reordering
860 // performed by the CPU(s) or platform.
861
862 struct SharedGlobals {
863 char _pad_prefix[OM_CACHE_LINE_SIZE];
864 // These are highly shared mostly-read variables.
865 // To avoid false-sharing they need to be the sole occupants of a cache line.
866 volatile int stw_random;
867 volatile int stw_cycle;
868 DEFINE_PAD_MINUS_SIZE(1, OM_CACHE_LINE_SIZE, sizeof(volatile int) * 2);
869 // Hot RW variable -- Sequester to avoid false-sharing
870 volatile int hc_sequence;
871 DEFINE_PAD_MINUS_SIZE(2, OM_CACHE_LINE_SIZE, sizeof(volatile int));
872 };
873
874 static SharedGlobals GVars;
875
876 static markWord read_stable_mark(oop obj) {
877 markWord mark = obj->mark();
878 if (!mark.is_being_inflated()) {
879 return mark; // normal fast-path return
880 }
881
882 int its = 0;
883 for (;;) {
884 markWord mark = obj->mark();
885 if (!mark.is_being_inflated()) {
886 return mark; // normal fast-path return
887 }
888
889 // The object is being inflated by some other thread.
890 // The caller of read_stable_mark() must wait for inflation to complete.
891 // Avoid live-lock
892 // TODO: consider calling SafepointSynchronize::do_call_back() while
893 // spinning to see if there's a safepoint pending. If so, immediately
894 // yielding or blocking would be appropriate. Avoid spinning while
895 // there is a safepoint pending.
896 // TODO: add inflation contention performance counters.
897 // TODO: restrict the aggregate number of spinners.
898
899 ++its;
900 if (its > 10000 || !os::is_MP()) {
901 if (its & 1) {
902 os::naked_yield();
903 } else {
904 // Note that the following code attenuates the livelock problem but is not
905 // a complete remedy. A more complete solution would require that the inflating
906 // thread hold the associated inflation lock. The following code simply restricts
907 // the number of spinners to at most one. We'll have N-2 threads blocked
908 // on the inflationlock, 1 thread holding the inflation lock and using
909 // a yield/park strategy, and 1 thread in the midst of inflation.
910 // A more refined approach would be to change the encoding of INFLATING
911 // to allow encapsulation of a native thread pointer. Threads waiting for
912 // inflation to complete would use CAS to push themselves onto a singly linked
913 // list rooted at the markword. Once enqueued, they'd loop, checking a per-thread flag
914 // and calling park(). When inflation was complete the thread that accomplished inflation
915 // would detach the list and set the markword to inflated with a single CAS and
916 // then for each thread on the list, set the flag and unpark() the thread.
917 // This is conceptually similar to muxAcquire-muxRelease, except that muxRelease
918 // wakes at most one thread whereas we need to wake the entire list.
919 int ix = (cast_from_oop<intptr_t>(obj) >> 5) & (NINFLATIONLOCKS-1);
920 int YieldThenBlock = 0;
921 assert(ix >= 0 && ix < NINFLATIONLOCKS, "invariant");
922 assert((NINFLATIONLOCKS & (NINFLATIONLOCKS-1)) == 0, "invariant");
923 Thread::muxAcquire(gInflationLocks + ix, "gInflationLock");
924 while (obj->mark() == markWord::INFLATING()) {
925 // Beware: NakedYield() is advisory and has almost no effect on some platforms
926 // so we periodically call self->_ParkEvent->park(1).
927 // We use a mixed spin/yield/block mechanism.
928 if ((YieldThenBlock++) >= 16) {
929 Thread::current()->_ParkEvent->park(1);
930 } else {
931 os::naked_yield();
932 }
933 }
934 Thread::muxRelease(gInflationLocks + ix);
935 }
936 } else {
937 SpinPause(); // SMP-polite spinning
938 }
939 }
940 }
941
942 // hashCode() generation :
943 //
944 // Possibilities:
945 // * MD5Digest of {obj,stw_random}
946 // * CRC32 of {obj,stw_random} or any linear-feedback shift register function.
947 // * A DES- or AES-style SBox[] mechanism
948 // * One of the Phi-based schemes, such as:
949 // 2654435761 = 2^32 * Phi (golden ratio)
950 // HashCodeValue = ((uintptr_t(obj) >> 3) * 2654435761) ^ GVars.stw_random ;
951 // * A variation of Marsaglia's shift-xor RNG scheme.
952 // * (obj ^ stw_random) is appealing, but can result
953 // in undesirable regularity in the hashCode values of adjacent objects
954 // (objects allocated back-to-back, in particular). This could potentially
955 // result in hashtable collisions and reduced hashtable efficiency.
956 // There are simple ways to "diffuse" the middle address bits over the
957 // generated hashCode values:
958
959 static inline intptr_t get_next_hash(Thread* self, oop obj) {
960 intptr_t value = 0;
961 if (hashCode == 0) {
962 // This form uses global Park-Miller RNG.
963 // On MP system we'll have lots of RW access to a global, so the
964 // mechanism induces lots of coherency traffic.
965 value = os::random();
966 } else if (hashCode == 1) {
967 // This variation has the property of being stable (idempotent)
968 // between STW operations. This can be useful in some of the 1-0
969 // synchronization schemes.
970 intptr_t addr_bits = cast_from_oop<intptr_t>(obj) >> 3;
971 value = addr_bits ^ (addr_bits >> 5) ^ GVars.stw_random;
972 } else if (hashCode == 2) {
973 value = 1; // for sensitivity testing
974 } else if (hashCode == 3) {
975 value = ++GVars.hc_sequence;
976 } else if (hashCode == 4) {
977 value = cast_from_oop<intptr_t>(obj);
978 } else {
979 // Marsaglia's xor-shift scheme with thread-specific state
980 // This is probably the best overall implementation -- we'll
981 // likely make this the default in future releases.
982 unsigned t = self->_hashStateX;
983 t ^= (t << 11);
984 self->_hashStateX = self->_hashStateY;
985 self->_hashStateY = self->_hashStateZ;
986 self->_hashStateZ = self->_hashStateW;
987 unsigned v = self->_hashStateW;
988 v = (v ^ (v >> 19)) ^ (t ^ (t >> 8));
989 self->_hashStateW = v;
990 value = v;
991 }
992
993 value &= markWord::hash_mask;
994 if (value == 0) value = 0xBAD;
995 assert(value != markWord::no_hash, "invariant");
996 return value;
997 }
998
999 intptr_t ObjectSynchronizer::FastHashCode(Thread* self, oop obj) {
1000 if (UseBiasedLocking) {
1001 // NOTE: many places throughout the JVM do not expect a safepoint
1002 // to be taken here. However, we only ever bias Java instances and all
1003 // of the call sites of identity_hash that might revoke biases have
1004 // been checked to make sure they can handle a safepoint. The
1005 // added check of the bias pattern is to avoid useless calls to
1006 // thread-local storage.
1007 if (obj->mark().has_bias_pattern()) {
1008 // Handle for oop obj in case of STW safepoint
1009 Handle hobj(self, obj);
1010 // Relaxing assertion for bug 6320749.
1011 assert(Universe::verify_in_progress() ||
1012 !SafepointSynchronize::is_at_safepoint(),
1013 "biases should not be seen by VM thread here");
1014 BiasedLocking::revoke(hobj, JavaThread::current());
1015 obj = hobj();
1016 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
1017 }
1018 }
1019
1020 while (true) {
1021 ObjectMonitor* monitor = NULL;
1022 markWord temp, test;
1023 intptr_t hash;
1024 markWord mark = read_stable_mark(obj);
1025
1026 // object should remain ineligible for biased locking
1027 assert(!mark.has_bias_pattern(), "invariant");
1028
1029 if (mark.is_neutral()) { // if this is a normal header
1030 hash = mark.hash();
1031 if (hash != 0) { // if it has a hash, just return it
1032 return hash;
1033 }
1034 hash = get_next_hash(self, obj); // get a new hash
1035 temp = mark.copy_set_hash(hash); // merge the hash into header
1036 // try to install the hash
1037 test = obj->cas_set_mark(temp, mark);
1038 if (test == mark) { // if the hash was installed, return it
1039 return hash;
1040 }
1041 // Failed to install the hash. It could be that another thread
1042 // installed the hash just before our attempt or inflation has
1043 // occurred or... so we fall thru to inflate the monitor for
1044 // stability and then install the hash.
1045 } else if (mark.has_monitor()) {
1046 monitor = mark.monitor();
1047 temp = monitor->header();
1048 assert(temp.is_neutral(), "invariant: header=" INTPTR_FORMAT, temp.value());
1049 hash = temp.hash();
1050 if (hash != 0) {
1051 // It has a hash.
1052
1053 // Separate load of dmw/header above from the loads in
1054 // is_being_async_deflated().
1055 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
1056 // A non-multiple copy atomic (nMCA) machine needs a bigger
1057 // hammer to separate the load above and the loads below.
1058 OrderAccess::fence();
1059 } else {
1060 OrderAccess::loadload();
1061 }
1062 if (monitor->is_being_async_deflated()) {
1063 // But we can't safely use the hash if we detect that async
1064 // deflation has occurred. So we attempt to restore the
1065 // header/dmw to the object's header so that we only retry
1066 // once if the deflater thread happens to be slow.
1067 monitor->install_displaced_markword_in_object(obj);
1068 continue;
1069 }
1070 return hash;
1071 }
1072 // Fall thru so we only have one place that installs the hash in
1073 // the ObjectMonitor.
1074 } else if (self->is_lock_owned((address)mark.locker())) {
1075 // This is a stack lock owned by the calling thread so fetch the
1076 // displaced markWord from the BasicLock on the stack.
1077 temp = mark.displaced_mark_helper();
1078 assert(temp.is_neutral(), "invariant: header=" INTPTR_FORMAT, temp.value());
1079 hash = temp.hash();
1080 if (hash != 0) { // if it has a hash, just return it
1081 return hash;
1082 }
1083 // WARNING:
1084 // The displaced header in the BasicLock on a thread's stack
1085 // is strictly immutable. It CANNOT be changed in ANY cases.
1086 // So we have to inflate the stack lock into an ObjectMonitor
1087 // even if the current thread owns the lock. The BasicLock on
1088 // a thread's stack can be asynchronously read by other threads
1089 // during an inflate() call so any change to that stack memory
1090 // may not propagate to other threads correctly.
1091 }
1092
1093 // Inflate the monitor to set the hash.
1094
1095 // An async deflation can race after the inflate() call and before we
1096 // can update the ObjectMonitor's header with the hash value below.
1097 monitor = inflate(self, obj, inflate_cause_hash_code);
1098 // Load ObjectMonitor's header/dmw field and see if it has a hash.
1099 mark = monitor->header();
1100 assert(mark.is_neutral(), "invariant: header=" INTPTR_FORMAT, mark.value());
1101 hash = mark.hash();
1102 if (hash == 0) { // if it does not have a hash
1103 hash = get_next_hash(self, obj); // get a new hash
1104 temp = mark.copy_set_hash(hash); // merge the hash into header
1105 assert(temp.is_neutral(), "invariant: header=" INTPTR_FORMAT, temp.value());
1106 uintptr_t v = Atomic::cmpxchg((volatile uintptr_t*)monitor->header_addr(), mark.value(), temp.value());
1107 test = markWord(v);
1108 if (test != mark) {
1109 // The attempt to update the ObjectMonitor's header/dmw field
1110 // did not work. This can happen if another thread managed to
1111 // merge in the hash just before our cmpxchg().
1112 // If we add any new usages of the header/dmw field, this code
1113 // will need to be updated.
1114 hash = test.hash();
1115 assert(test.is_neutral(), "invariant: header=" INTPTR_FORMAT, test.value());
1116 assert(hash != 0, "should only have lost the race to a thread that set a non-zero hash");
1117 }
1118 if (monitor->is_being_async_deflated()) {
1119 // If we detect that async deflation has occurred, then we
1120 // attempt to restore the header/dmw to the object's header
1121 // so that we only retry once if the deflater thread happens
1122 // to be slow.
1123 monitor->install_displaced_markword_in_object(obj);
1124 continue;
1125 }
1126 }
1127 // We finally get the hash.
1128 return hash;
1129 }
1130 }
1131
1132 // Deprecated -- use FastHashCode() instead.
1133
1134 intptr_t ObjectSynchronizer::identity_hash_value_for(Handle obj) {
1135 return FastHashCode(Thread::current(), obj());
1136 }
1137
1138
1139 bool ObjectSynchronizer::current_thread_holds_lock(JavaThread* thread,
1140 Handle h_obj) {
1141 if (UseBiasedLocking) {
1142 BiasedLocking::revoke(h_obj, thread);
1143 assert(!h_obj->mark().has_bias_pattern(), "biases should be revoked by now");
1144 }
1145
1146 assert(thread == JavaThread::current(), "Can only be called on current thread");
1147 oop obj = h_obj();
1148
1149 markWord mark = read_stable_mark(obj);
1150
1151 // Uncontended case, header points to stack
1152 if (mark.has_locker()) {
1153 return thread->is_lock_owned((address)mark.locker());
1154 }
1155 // Contended case, header points to ObjectMonitor (tagged pointer)
1156 if (mark.has_monitor()) {
1157 // The first stage of async deflation does not affect any field
1158 // used by this comparison so the ObjectMonitor* is usable here.
1159 ObjectMonitor* monitor = mark.monitor();
1160 return monitor->is_entered(thread) != 0;
1161 }
1162 // Unlocked case, header in place
1163 assert(mark.is_neutral(), "sanity check");
1164 return false;
1165 }
1166
1167 // Be aware of this method could revoke bias of the lock object.
1168 // This method queries the ownership of the lock handle specified by 'h_obj'.
1169 // If the current thread owns the lock, it returns owner_self. If no
1170 // thread owns the lock, it returns owner_none. Otherwise, it will return
1171 // owner_other.
1172 ObjectSynchronizer::LockOwnership ObjectSynchronizer::query_lock_ownership
1173 (JavaThread *self, Handle h_obj) {
1174 // The caller must beware this method can revoke bias, and
1175 // revocation can result in a safepoint.
1176 assert(!SafepointSynchronize::is_at_safepoint(), "invariant");
1177 assert(self->thread_state() != _thread_blocked, "invariant");
1178
1179 // Possible mark states: neutral, biased, stack-locked, inflated
1180
1181 if (UseBiasedLocking && h_obj()->mark().has_bias_pattern()) {
1182 // CASE: biased
1183 BiasedLocking::revoke(h_obj, self);
1184 assert(!h_obj->mark().has_bias_pattern(),
1185 "biases should be revoked by now");
1186 }
1187
1188 assert(self == JavaThread::current(), "Can only be called on current thread");
1189 oop obj = h_obj();
1190 markWord mark = read_stable_mark(obj);
1191
1192 // CASE: stack-locked. Mark points to a BasicLock on the owner's stack.
1193 if (mark.has_locker()) {
1194 return self->is_lock_owned((address)mark.locker()) ?
1195 owner_self : owner_other;
1196 }
1197
1198 // CASE: inflated. Mark (tagged pointer) points to an ObjectMonitor.
1199 if (mark.has_monitor()) {
1200 // The first stage of async deflation does not affect any field
1201 // used by this comparison so the ObjectMonitor* is usable here.
1202 ObjectMonitor* monitor = mark.monitor();
1203 void* owner = monitor->owner();
1204 if (owner == NULL) return owner_none;
1205 return (owner == self ||
1206 self->is_lock_owned((address)owner)) ? owner_self : owner_other;
1207 }
1208
1209 // CASE: neutral
1210 assert(mark.is_neutral(), "sanity check");
1211 return owner_none; // it's unlocked
1212 }
1213
1214 // FIXME: jvmti should call this
1215 JavaThread* ObjectSynchronizer::get_lock_owner(ThreadsList * t_list, Handle h_obj) {
1216 if (UseBiasedLocking) {
1217 if (SafepointSynchronize::is_at_safepoint()) {
1218 BiasedLocking::revoke_at_safepoint(h_obj);
1219 } else {
1220 BiasedLocking::revoke(h_obj, JavaThread::current());
1221 }
1222 assert(!h_obj->mark().has_bias_pattern(), "biases should be revoked by now");
1223 }
1224
1225 oop obj = h_obj();
1226 address owner = NULL;
1227
1228 markWord mark = read_stable_mark(obj);
1229
1230 // Uncontended case, header points to stack
1231 if (mark.has_locker()) {
1232 owner = (address) mark.locker();
1233 }
1234
1235 // Contended case, header points to ObjectMonitor (tagged pointer)
1236 else if (mark.has_monitor()) {
1237 // The first stage of async deflation does not affect any field
1238 // used by this comparison so the ObjectMonitor* is usable here.
1239 ObjectMonitor* monitor = mark.monitor();
1240 assert(monitor != NULL, "monitor should be non-null");
1241 owner = (address) monitor->owner();
1242 }
1243
1244 if (owner != NULL) {
1245 // owning_thread_from_monitor_owner() may also return NULL here
1246 return Threads::owning_thread_from_monitor_owner(t_list, owner);
1247 }
1248
1249 // Unlocked case, header in place
1250 // Cannot have assertion since this object may have been
1251 // locked by another thread when reaching here.
1252 // assert(mark.is_neutral(), "sanity check");
1253
1254 return NULL;
1255 }
1256
1257 // Visitors ...
1258
1259 void ObjectSynchronizer::monitors_iterate(MonitorClosure* closure) {
1260 PaddedObjectMonitor* block = Atomic::load(&g_block_list);
1261 while (block != NULL) {
1262 assert(block->object() == CHAINMARKER, "must be a block header");
1263 for (int i = _BLOCKSIZE - 1; i > 0; i--) {
1264 ObjectMonitor* mid = (ObjectMonitor *)(block + i);
1265 if (mid->object() != NULL) {
1266 // Only process with closure if the object is set.
1267
1268 // monitors_iterate() is only called at a safepoint or when the
1269 // target thread is suspended or when the target thread is
1270 // operating on itself. The current closures in use today are
1271 // only interested in an owned ObjectMonitor and ownership
1272 // cannot be dropped under the calling contexts so the
1273 // ObjectMonitor cannot be async deflated.
1274 closure->do_monitor(mid);
1275 }
1276 }
1277 // unmarked_next() is not needed with g_block_list (no locking
1278 // used with block linkage _next_om fields).
1279 block = (PaddedObjectMonitor*)block->next_om();
1280 }
1281 }
1282
1283 static bool monitors_used_above_threshold() {
1284 int population = Atomic::load(&om_list_globals._population);
1285 if (population == 0) {
1286 return false;
1287 }
1288 if (MonitorUsedDeflationThreshold > 0) {
1289 int monitors_used = population - Atomic::load(&om_list_globals._free_count) -
1290 Atomic::load(&om_list_globals._wait_count);
1291 int monitor_usage = (monitors_used * 100LL) / population;
1292 return monitor_usage > MonitorUsedDeflationThreshold;
1293 }
1294 return false;
1295 }
1296
1297 bool ObjectSynchronizer::is_async_deflation_needed() {
1298 if (is_async_deflation_requested()) {
1299 // Async deflation request.
1300 return true;
1301 }
1302 if (AsyncDeflationInterval > 0 &&
1303 time_since_last_async_deflation_ms() > AsyncDeflationInterval &&
1304 monitors_used_above_threshold()) {
1305 // It's been longer than our specified deflate interval and there
1306 // are too many monitors in use. We don't deflate more frequently
1307 // than AsyncDeflationInterval (unless is_async_deflation_requested)
1308 // in order to not swamp the ServiceThread.
1309 return true;
1310 }
1311 return false;
1312 }
1313
1314 bool ObjectSynchronizer::request_deflate_idle_monitors() {
1315 bool is_JavaThread = Thread::current()->is_Java_thread();
1316 bool ret_code = false;
1317
1318 jlong last_time = last_async_deflation_time_ns();
1319 set_is_async_deflation_requested(true);
1320 {
1321 MonitorLocker ml(Service_lock, Mutex::_no_safepoint_check_flag);
1322 ml.notify_all();
1323 }
1324 const int N_CHECKS = 5;
1325 for (int i = 0; i < N_CHECKS; i++) { // sleep for at most 5 seconds
1326 if (last_async_deflation_time_ns() > last_time) {
1327 log_info(monitorinflation)("Async Deflation happened after %d check(s).", i);
1328 ret_code = true;
1329 break;
1330 }
1331 if (is_JavaThread) {
1332 // JavaThread has to honor the blocking protocol.
1333 ThreadBlockInVM tbivm(JavaThread::current());
1334 os::naked_short_sleep(999); // sleep for almost 1 second
1335 } else {
1336 os::naked_short_sleep(999); // sleep for almost 1 second
1337 }
1338 }
1339 if (!ret_code) {
1340 log_info(monitorinflation)("Async Deflation DID NOT happen after %d checks.", N_CHECKS);
1341 }
1342
1343 return ret_code;
1344 }
1345
1346 jlong ObjectSynchronizer::time_since_last_async_deflation_ms() {
1347 return (os::javaTimeNanos() - last_async_deflation_time_ns()) / (NANOUNITS / MILLIUNITS);
1348 }
1349
1350 void ObjectSynchronizer::oops_do(OopClosure* f) {
1351 // We only scan the global used list here (for moribund threads), and
1352 // the thread-local monitors in Thread::oops_do().
1353 global_used_oops_do(f);
1354 }
1355
1356 void ObjectSynchronizer::global_used_oops_do(OopClosure* f) {
1357 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
1358 // Acquire semantics not needed since we're at a safepoint.
1359 list_oops_do(Atomic::load(&om_list_globals._in_use_list), f);
1360 }
1361
1362 void ObjectSynchronizer::thread_local_used_oops_do(Thread* thread, OopClosure* f) {
1363 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
1364 list_oops_do(thread->om_in_use_list, f);
1365 }
1366
1367 void ObjectSynchronizer::list_oops_do(ObjectMonitor* list, OopClosure* f) {
1368 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
1369 // The oops_do() phase does not overlap with monitor deflation
1370 // so no need to lock ObjectMonitors for the list traversal.
1371 for (ObjectMonitor* mid = list; mid != NULL; mid = unmarked_next(mid)) {
1372 if (mid->object() != NULL) {
1373 f->do_oop((oop*)mid->object_addr());
1374 }
1375 }
1376 }
1377
1378
1379 // -----------------------------------------------------------------------------
1380 // ObjectMonitor Lifecycle
1381 // -----------------------
1382 // Inflation unlinks monitors from om_list_globals._free_list or a per-thread
1383 // free list and associates them with objects. Async deflation disassociates
1384 // idle monitors from objects. Such scavenged monitors are returned to the
1385 // om_list_globals._free_list.
1386 //
1387 // ObjectMonitors reside in type-stable memory (TSM) and are immortal.
1388 //
1389 // Lifecycle:
1390 // -- unassigned and on the om_list_globals._free_list
1391 // -- unassigned and on a per-thread free list
1392 // -- assigned to an object. The object is inflated and the mark refers
1393 // to the ObjectMonitor.
1394
1395 ObjectMonitor* ObjectSynchronizer::om_alloc(Thread* self) {
1396 // A large MAXPRIVATE value reduces both list lock contention
1397 // and list coherency traffic, but also tends to increase the
1398 // number of ObjectMonitors in circulation as well as the
1399 // scavenge costs. As usual, we lean toward time in space-time
1400 // tradeoffs.
1401 const int MAXPRIVATE = 1024;
1402 NoSafepointVerifier nsv;
1403
1404 for (;;) {
1405 ObjectMonitor* m;
1406
1407 // 1: try to allocate from the thread's local om_free_list.
1408 // Threads will attempt to allocate first from their local list, then
1409 // from the global list, and only after those attempts fail will the
1410 // thread attempt to instantiate new monitors. Thread-local free lists
1411 // improve allocation latency, as well as reducing coherency traffic
1412 // on the shared global list.
1413 m = take_from_start_of_om_free_list(self);
1414 if (m != NULL) {
1415 guarantee(m->object() == NULL, "invariant");
1416 m->set_allocation_state(ObjectMonitor::New);
1417 prepend_to_om_in_use_list(self, m);
1418 return m;
1419 }
1420
1421 // 2: try to allocate from the global om_list_globals._free_list
1422 // If we're using thread-local free lists then try
1423 // to reprovision the caller's free list.
1424 // Acquire semantics not needed on this list load since memory
1425 // is already consistent due to the cmpxchg() via
1426 // take_from_start_of_om_free_list() above.
1427 if (Atomic::load(&om_list_globals._free_list) != NULL) {
1428 // Reprovision the thread's om_free_list.
1429 // Use bulk transfers to reduce the allocation rate and heat
1430 // on various locks.
1431 for (int i = self->om_free_provision; --i >= 0;) {
1432 ObjectMonitor* take = take_from_start_of_global_free_list();
1433 if (take == NULL) {
1434 break; // No more are available.
1435 }
1436 guarantee(take->object() == NULL, "invariant");
1437 // We allowed 3 field values to linger during async deflation.
1438 // Clear or restore them as appropriate.
1439 take->set_header(markWord::zero());
1440 // DEFLATER_MARKER is the only non-NULL value we should see here.
1441 take->try_set_owner_from(DEFLATER_MARKER, NULL);
1442 if (take->contentions() < 0) {
1443 // Add back max_jint to restore the contentions field to its
1444 // proper value.
1445 take->add_to_contentions(max_jint);
1446
1447 #ifdef ASSERT
1448 jint l_contentions = take->contentions();
1449 assert(l_contentions >= 0, "must not be negative: l_contentions=%d, contentions=%d",
1450 l_contentions, take->contentions());
1451 #endif
1452 }
1453 take->Recycle();
1454 // Since we're taking from the global free-list, take must be Free.
1455 // om_release() also sets the allocation state to Free because it
1456 // is called from other code paths.
1457 assert(take->is_free(), "invariant");
1458 om_release(self, take, false);
1459 }
1460 self->om_free_provision += 1 + (self->om_free_provision / 2);
1461 if (self->om_free_provision > MAXPRIVATE) self->om_free_provision = MAXPRIVATE;
1462 continue;
1463 }
1464
1465 // 3: allocate a block of new ObjectMonitors
1466 // Both the local and global free lists are empty -- resort to malloc().
1467 // In the current implementation ObjectMonitors are TSM - immortal.
1468 // Ideally, we'd write "new ObjectMonitor[_BLOCKSIZE], but we want
1469 // each ObjectMonitor to start at the beginning of a cache line,
1470 // so we use align_up().
1471 // A better solution would be to use C++ placement-new.
1472 // BEWARE: As it stands currently, we don't run the ctors!
1473 assert(_BLOCKSIZE > 1, "invariant");
1474 size_t neededsize = sizeof(PaddedObjectMonitor) * _BLOCKSIZE;
1475 PaddedObjectMonitor* temp;
1476 size_t aligned_size = neededsize + (OM_CACHE_LINE_SIZE - 1);
1477 void* real_malloc_addr = NEW_C_HEAP_ARRAY(char, aligned_size, mtInternal);
1478 temp = (PaddedObjectMonitor*)align_up(real_malloc_addr, OM_CACHE_LINE_SIZE);
1479 (void)memset((void *) temp, 0, neededsize);
1480
1481 // Format the block.
1482 // initialize the linked list, each monitor points to its next
1483 // forming the single linked free list, the very first monitor
1484 // will points to next block, which forms the block list.
1485 // The trick of using the 1st element in the block as g_block_list
1486 // linkage should be reconsidered. A better implementation would
1487 // look like: class Block { Block * next; int N; ObjectMonitor Body [N] ; }
1488
1489 for (int i = 1; i < _BLOCKSIZE; i++) {
1490 temp[i].set_next_om((ObjectMonitor*)&temp[i + 1]);
1491 assert(temp[i].is_free(), "invariant");
1492 }
1493
1494 // terminate the last monitor as the end of list
1495 temp[_BLOCKSIZE - 1].set_next_om((ObjectMonitor*)NULL);
1496
1497 // Element [0] is reserved for global list linkage
1498 temp[0].set_object(CHAINMARKER);
1499
1500 // Consider carving out this thread's current request from the
1501 // block in hand. This avoids some lock traffic and redundant
1502 // list activity.
1503
1504 prepend_block_to_lists(temp);
1505 }
1506 }
1507
1508 // Place "m" on the caller's private per-thread om_free_list.
1509 // In practice there's no need to clamp or limit the number of
1510 // monitors on a thread's om_free_list as the only non-allocation time
1511 // we'll call om_release() is to return a monitor to the free list after
1512 // a CAS attempt failed. This doesn't allow unbounded #s of monitors to
1513 // accumulate on a thread's free list.
1514 //
1515 // Key constraint: all ObjectMonitors on a thread's free list and the global
1516 // free list must have their object field set to null. This prevents the
1517 // scavenger -- deflate_monitor_list_using_JT() -- from reclaiming them
1518 // while we are trying to release them.
1519
1520 void ObjectSynchronizer::om_release(Thread* self, ObjectMonitor* m,
1521 bool from_per_thread_alloc) {
1522 guarantee(m->header().value() == 0, "invariant");
1523 guarantee(m->object() == NULL, "invariant");
1524 NoSafepointVerifier nsv;
1525
1526 if ((m->is_busy() | m->_recursions) != 0) {
1527 stringStream ss;
1528 fatal("freeing in-use monitor: %s, recursions=" INTX_FORMAT,
1529 m->is_busy_to_string(&ss), m->_recursions);
1530 }
1531 m->set_allocation_state(ObjectMonitor::Free);
1532 // _next_om is used for both per-thread in-use and free lists so
1533 // we have to remove 'm' from the in-use list first (as needed).
1534 if (from_per_thread_alloc) {
1535 // Need to remove 'm' from om_in_use_list.
1536 ObjectMonitor* mid = NULL;
1537 ObjectMonitor* next = NULL;
1538
1539 // This list walk can race with another list walker or with async
1540 // deflation so we have to worry about an ObjectMonitor being
1541 // removed from this list while we are walking it.
1542
1543 // Lock the list head to avoid racing with another list walker
1544 // or with async deflation.
1545 if ((mid = get_list_head_locked(&self->om_in_use_list)) == NULL) {
1546 fatal("thread=" INTPTR_FORMAT " in-use list must not be empty.", p2i(self));
1547 }
1548 next = unmarked_next(mid);
1549 if (m == mid) {
1550 // First special case:
1551 // 'm' matches mid, is the list head and is locked. Switch the list
1552 // head to next which unlocks the list head, but leaves the extracted
1553 // mid locked. Release semantics not needed on this "unlock" since
1554 // memory is already consistent due to the get_list_head_locked()
1555 // above.
1556 Atomic::store(&self->om_in_use_list, next);
1557 } else if (m == next) {
1558 // Second special case:
1559 // 'm' matches next after the list head and we already have the list
1560 // head locked so set mid to what we are extracting:
1561 mid = next;
1562 // Lock mid to prevent races with a list walker or an async
1563 // deflater thread that's ahead of us. The locked list head
1564 // prevents races from behind us.
1565 om_lock(mid);
1566 // Update next to what follows mid (if anything):
1567 next = unmarked_next(mid);
1568 // Switch next after the list head to new next which unlocks the
1569 // list head, but leaves the extracted mid locked. Release semantics
1570 // not needed on this "unlock" since memory is already consistent
1571 // due to the get_list_head_locked() above.
1572 self->om_in_use_list->set_next_om(next);
1573 } else {
1574 // We have to search the list to find 'm'.
1575 guarantee(next != NULL, "thread=" INTPTR_FORMAT ": om_in_use_list=" INTPTR_FORMAT
1576 " is too short.", p2i(self), p2i(self->om_in_use_list));
1577 // Our starting anchor is next after the list head which is the
1578 // last ObjectMonitor we checked:
1579 ObjectMonitor* anchor = next;
1580 // Lock anchor to prevent races with a list walker or an async
1581 // deflater thread that's ahead of us. The locked list head
1582 // prevents races from behind us.
1583 om_lock(anchor);
1584 om_unlock(mid); // Unlock the list head now that anchor is locked.
1585 while ((mid = unmarked_next(anchor)) != NULL) {
1586 if (m == mid) {
1587 // We found 'm' on the per-thread in-use list so extract it.
1588 // Update next to what follows mid (if anything):
1589 next = unmarked_next(mid);
1590 // Switch next after the anchor to new next which unlocks the
1591 // anchor, but leaves the extracted mid locked. Release semantics
1592 // not needed on this "unlock" since memory is already consistent
1593 // due to the om_unlock() above before entering the loop or the
1594 // om_unlock() below before looping again.
1595 anchor->set_next_om(next);
1596 break;
1597 } else {
1598 // Lock the next anchor to prevent races with a list walker
1599 // or an async deflater thread that's ahead of us. The locked
1600 // current anchor prevents races from behind us.
1601 om_lock(mid);
1602 // Unlock current anchor now that next anchor is locked:
1603 om_unlock(anchor);
1604 anchor = mid; // Advance to new anchor and try again.
1605 }
1606 }
1607 }
1608
1609 if (mid == NULL) {
1610 // Reached end of the list and didn't find 'm' so:
1611 fatal("thread=" INTPTR_FORMAT " must find m=" INTPTR_FORMAT "on om_in_use_list="
1612 INTPTR_FORMAT, p2i(self), p2i(m), p2i(self->om_in_use_list));
1613 }
1614
1615 // At this point mid is disconnected from the in-use list so
1616 // its lock no longer has any effects on the in-use list.
1617 Atomic::dec(&self->om_in_use_count);
1618 // Unlock mid, but leave the next value for any lagging list
1619 // walkers. It will get cleaned up when mid is prepended to
1620 // the thread's free list:
1621 om_unlock(mid);
1622 }
1623
1624 prepend_to_om_free_list(self, m);
1625 guarantee(m->is_free(), "invariant");
1626 }
1627
1628 // Return ObjectMonitors on a moribund thread's free and in-use
1629 // lists to the appropriate global lists. The ObjectMonitors on the
1630 // per-thread in-use list may still be in use by other threads.
1631 //
1632 // We currently call om_flush() from Threads::remove() before the
1633 // thread has been excised from the thread list and is no longer a
1634 // mutator. In particular, this ensures that the thread's in-use
1635 // monitors are scanned by a GC safepoint, either via Thread::oops_do()
1636 // (before om_flush() is called) or via ObjectSynchronizer::oops_do()
1637 // (after om_flush() is called).
1638 //
1639 // deflate_global_idle_monitors_using_JT() and
1640 // deflate_per_thread_idle_monitors_using_JT() (in another thread) can
1641 // run at the same time as om_flush() so we have to follow a careful
1642 // protocol to prevent list corruption.
1643
1644 void ObjectSynchronizer::om_flush(Thread* self) {
1645 // Process the per-thread in-use list first to be consistent.
1646 int in_use_count = 0;
1647 ObjectMonitor* in_use_list = NULL;
1648 ObjectMonitor* in_use_tail = NULL;
1649 NoSafepointVerifier nsv;
1650
1651 // This function can race with a list walker or with an async
1652 // deflater thread so we lock the list head to prevent confusion.
1653 // An async deflater thread checks to see if the target thread
1654 // is exiting, but if it has made it past that check before we
1655 // started exiting, then it is racing to get to the in-use list.
1656 if ((in_use_list = get_list_head_locked(&self->om_in_use_list)) != NULL) {
1657 // At this point, we have locked the in-use list head so a racing
1658 // thread cannot come in after us. However, a racing thread could
1659 // be ahead of us; we'll detect that and delay to let it finish.
1660 //
1661 // The thread is going away, however the ObjectMonitors on the
1662 // om_in_use_list may still be in-use by other threads. Link
1663 // them to in_use_tail, which will be linked into the global
1664 // in-use list (om_list_globals._in_use_list) below.
1665 //
1666 // Account for the in-use list head before the loop since it is
1667 // already locked (by this thread):
1668 in_use_tail = in_use_list;
1669 in_use_count++;
1670 for (ObjectMonitor* cur_om = unmarked_next(in_use_list); cur_om != NULL;) {
1671 if (is_locked(cur_om)) {
1672 // cur_om is locked so there must be a racing walker or async
1673 // deflater thread ahead of us so we'll give it a chance to finish.
1674 while (is_locked(cur_om)) {
1675 os::naked_short_sleep(1);
1676 }
1677 // Refetch the possibly changed next field and try again.
1678 cur_om = unmarked_next(in_use_tail);
1679 continue;
1680 }
1681 if (cur_om->object() == NULL) {
1682 // cur_om was deflated and the object ref was cleared while it
1683 // was locked. We happened to see it just after it was unlocked
1684 // (and added to the free list). Refetch the possibly changed
1685 // next field and try again.
1686 cur_om = unmarked_next(in_use_tail);
1687 continue;
1688 }
1689 in_use_tail = cur_om;
1690 in_use_count++;
1691 cur_om = unmarked_next(cur_om);
1692 }
1693 guarantee(in_use_tail != NULL, "invariant");
1694 #ifdef ASSERT
1695 int l_om_in_use_count = Atomic::load(&self->om_in_use_count);
1696 assert(l_om_in_use_count == in_use_count, "in-use counts don't match: "
1697 "l_om_in_use_count=%d, in_use_count=%d", l_om_in_use_count, in_use_count);
1698 #endif
1699 Atomic::store(&self->om_in_use_count, 0);
1700 OrderAccess::storestore(); // Make sure counter update is seen first.
1701 // Clear the in-use list head (which also unlocks it):
1702 Atomic::store(&self->om_in_use_list, (ObjectMonitor*)NULL);
1703 om_unlock(in_use_list);
1704 }
1705
1706 int free_count = 0;
1707 ObjectMonitor* free_list = NULL;
1708 ObjectMonitor* free_tail = NULL;
1709 // This function can race with a list walker thread so we lock the
1710 // list head to prevent confusion.
1711 if ((free_list = get_list_head_locked(&self->om_free_list)) != NULL) {
1712 // At this point, we have locked the free list head so a racing
1713 // thread cannot come in after us. However, a racing thread could
1714 // be ahead of us; we'll detect that and delay to let it finish.
1715 //
1716 // The thread is going away. Set 'free_tail' to the last per-thread free
1717 // monitor which will be linked to om_list_globals._free_list below.
1718 //
1719 // Account for the free list head before the loop since it is
1720 // already locked (by this thread):
1721 free_tail = free_list;
1722 free_count++;
1723 for (ObjectMonitor* s = unmarked_next(free_list); s != NULL; s = unmarked_next(s)) {
1724 if (is_locked(s)) {
1725 // s is locked so there must be a racing walker thread ahead
1726 // of us so we'll give it a chance to finish.
1727 while (is_locked(s)) {
1728 os::naked_short_sleep(1);
1729 }
1730 }
1731 free_tail = s;
1732 free_count++;
1733 guarantee(s->object() == NULL, "invariant");
1734 if (s->is_busy()) {
1735 stringStream ss;
1736 fatal("must be !is_busy: %s", s->is_busy_to_string(&ss));
1737 }
1738 }
1739 guarantee(free_tail != NULL, "invariant");
1740 #ifdef ASSERT
1741 int l_om_free_count = Atomic::load(&self->om_free_count);
1742 assert(l_om_free_count == free_count, "free counts don't match: "
1743 "l_om_free_count=%d, free_count=%d", l_om_free_count, free_count);
1744 #endif
1745 Atomic::store(&self->om_free_count, 0);
1746 OrderAccess::storestore(); // Make sure counter update is seen first.
1747 Atomic::store(&self->om_free_list, (ObjectMonitor*)NULL);
1748 om_unlock(free_list);
1749 }
1750
1751 if (free_tail != NULL) {
1752 prepend_list_to_global_free_list(free_list, free_tail, free_count);
1753 }
1754
1755 if (in_use_tail != NULL) {
1756 prepend_list_to_global_in_use_list(in_use_list, in_use_tail, in_use_count);
1757 }
1758
1759 LogStreamHandle(Debug, monitorinflation) lsh_debug;
1760 LogStreamHandle(Info, monitorinflation) lsh_info;
1761 LogStream* ls = NULL;
1762 if (log_is_enabled(Debug, monitorinflation)) {
1763 ls = &lsh_debug;
1764 } else if ((free_count != 0 || in_use_count != 0) &&
1765 log_is_enabled(Info, monitorinflation)) {
1766 ls = &lsh_info;
1767 }
1768 if (ls != NULL) {
1769 ls->print_cr("om_flush: jt=" INTPTR_FORMAT ", free_count=%d"
1770 ", in_use_count=%d" ", om_free_provision=%d",
1771 p2i(self), free_count, in_use_count, self->om_free_provision);
1772 }
1773 }
1774
1775 static void post_monitor_inflate_event(EventJavaMonitorInflate* event,
1776 const oop obj,
1777 ObjectSynchronizer::InflateCause cause) {
1778 assert(event != NULL, "invariant");
1779 assert(event->should_commit(), "invariant");
1780 event->set_monitorClass(obj->klass());
1781 event->set_address((uintptr_t)(void*)obj);
1782 event->set_cause((u1)cause);
1783 event->commit();
1784 }
1785
1786 // Fast path code shared by multiple functions
1787 void ObjectSynchronizer::inflate_helper(oop obj) {
1788 markWord mark = obj->mark();
1789 if (mark.has_monitor()) {
1790 ObjectMonitor* monitor = mark.monitor();
1791 assert(ObjectSynchronizer::verify_objmon_isinpool(monitor), "monitor=" INTPTR_FORMAT " is invalid", p2i(monitor));
1792 markWord dmw = monitor->header();
1793 assert(dmw.is_neutral(), "sanity check: header=" INTPTR_FORMAT, dmw.value());
1794 return;
1795 }
1796 (void)inflate(Thread::current(), obj, inflate_cause_vm_internal);
1797 }
1798
1799 ObjectMonitor* ObjectSynchronizer::inflate(Thread* self, oop object,
1800 const InflateCause cause) {
1801 EventJavaMonitorInflate event;
1802
1803 for (;;) {
1804 const markWord mark = object->mark();
1805 assert(!mark.has_bias_pattern(), "invariant");
1806
1807 // The mark can be in one of the following states:
1808 // * Inflated - just return
1809 // * Stack-locked - coerce it to inflated
1810 // * INFLATING - busy wait for conversion to complete
1811 // * Neutral - aggressively inflate the object.
1812 // * BIASED - Illegal. We should never see this
1813
1814 // CASE: inflated
1815 if (mark.has_monitor()) {
1816 ObjectMonitor* inf = mark.monitor();
1817 markWord dmw = inf->header();
1818 assert(dmw.is_neutral(), "invariant: header=" INTPTR_FORMAT, dmw.value());
1819 assert(ObjectSynchronizer::verify_objmon_isinpool(inf), "monitor is invalid");
1820 return inf;
1821 }
1822
1823 // CASE: inflation in progress - inflating over a stack-lock.
1824 // Some other thread is converting from stack-locked to inflated.
1825 // Only that thread can complete inflation -- other threads must wait.
1826 // The INFLATING value is transient.
1827 // Currently, we spin/yield/park and poll the markword, waiting for inflation to finish.
1828 // We could always eliminate polling by parking the thread on some auxiliary list.
1829 if (mark == markWord::INFLATING()) {
1830 read_stable_mark(object);
1831 continue;
1832 }
1833
1834 // CASE: stack-locked
1835 // Could be stack-locked either by this thread or by some other thread.
1836 //
1837 // Note that we allocate the objectmonitor speculatively, _before_ attempting
1838 // to install INFLATING into the mark word. We originally installed INFLATING,
1839 // allocated the objectmonitor, and then finally STed the address of the
1840 // objectmonitor into the mark. This was correct, but artificially lengthened
1841 // the interval in which INFLATED appeared in the mark, thus increasing
1842 // the odds of inflation contention.
1843 //
1844 // We now use per-thread private objectmonitor free lists.
1845 // These list are reprovisioned from the global free list outside the
1846 // critical INFLATING...ST interval. A thread can transfer
1847 // multiple objectmonitors en-mass from the global free list to its local free list.
1848 // This reduces coherency traffic and lock contention on the global free list.
1849 // Using such local free lists, it doesn't matter if the om_alloc() call appears
1850 // before or after the CAS(INFLATING) operation.
1851 // See the comments in om_alloc().
1852
1853 LogStreamHandle(Trace, monitorinflation) lsh;
1854
1855 if (mark.has_locker()) {
1856 ObjectMonitor* m = om_alloc(self);
1857 // Optimistically prepare the objectmonitor - anticipate successful CAS
1858 // We do this before the CAS in order to minimize the length of time
1859 // in which INFLATING appears in the mark.
1860 m->Recycle();
1861 m->_Responsible = NULL;
1862 m->_SpinDuration = ObjectMonitor::Knob_SpinLimit; // Consider: maintain by type/class
1863
1864 markWord cmp = object->cas_set_mark(markWord::INFLATING(), mark);
1865 if (cmp != mark) {
1866 // om_release() will reset the allocation state from New to Free.
1867 om_release(self, m, true);
1868 continue; // Interference -- just retry
1869 }
1870
1871 // We've successfully installed INFLATING (0) into the mark-word.
1872 // This is the only case where 0 will appear in a mark-word.
1873 // Only the singular thread that successfully swings the mark-word
1874 // to 0 can perform (or more precisely, complete) inflation.
1875 //
1876 // Why do we CAS a 0 into the mark-word instead of just CASing the
1877 // mark-word from the stack-locked value directly to the new inflated state?
1878 // Consider what happens when a thread unlocks a stack-locked object.
1879 // It attempts to use CAS to swing the displaced header value from the
1880 // on-stack BasicLock back into the object header. Recall also that the
1881 // header value (hash code, etc) can reside in (a) the object header, or
1882 // (b) a displaced header associated with the stack-lock, or (c) a displaced
1883 // header in an ObjectMonitor. The inflate() routine must copy the header
1884 // value from the BasicLock on the owner's stack to the ObjectMonitor, all
1885 // the while preserving the hashCode stability invariants. If the owner
1886 // decides to release the lock while the value is 0, the unlock will fail
1887 // and control will eventually pass from slow_exit() to inflate. The owner
1888 // will then spin, waiting for the 0 value to disappear. Put another way,
1889 // the 0 causes the owner to stall if the owner happens to try to
1890 // drop the lock (restoring the header from the BasicLock to the object)
1891 // while inflation is in-progress. This protocol avoids races that might
1892 // would otherwise permit hashCode values to change or "flicker" for an object.
1893 // Critically, while object->mark is 0 mark.displaced_mark_helper() is stable.
1894 // 0 serves as a "BUSY" inflate-in-progress indicator.
1895
1896
1897 // fetch the displaced mark from the owner's stack.
1898 // The owner can't die or unwind past the lock while our INFLATING
1899 // object is in the mark. Furthermore the owner can't complete
1900 // an unlock on the object, either.
1901 markWord dmw = mark.displaced_mark_helper();
1902 // Catch if the object's header is not neutral (not locked and
1903 // not marked is what we care about here).
1904 assert(dmw.is_neutral(), "invariant: header=" INTPTR_FORMAT, dmw.value());
1905
1906 // Setup monitor fields to proper values -- prepare the monitor
1907 m->set_header(dmw);
1908
1909 // Optimization: if the mark.locker stack address is associated
1910 // with this thread we could simply set m->_owner = self.
1911 // Note that a thread can inflate an object
1912 // that it has stack-locked -- as might happen in wait() -- directly
1913 // with CAS. That is, we can avoid the xchg-NULL .... ST idiom.
1914 m->set_owner_from(NULL, DEFLATER_MARKER, mark.locker());
1915 m->set_object(object);
1916 // TODO-FIXME: assert BasicLock->dhw != 0.
1917
1918 // Must preserve store ordering. The monitor state must
1919 // be stable at the time of publishing the monitor address.
1920 guarantee(object->mark() == markWord::INFLATING(), "invariant");
1921 // Release semantics so that above set_object() is seen first.
1922 object->release_set_mark(markWord::encode(m));
1923
1924 // Once ObjectMonitor is configured and the object is associated
1925 // with the ObjectMonitor, it is safe to allow async deflation:
1926 assert(m->is_new(), "freshly allocated monitor must be new");
1927 // Release semantics needed to keep allocation_state from floating up.
1928 m->release_set_allocation_state(ObjectMonitor::Old);
1929
1930 // Hopefully the performance counters are allocated on distinct cache lines
1931 // to avoid false sharing on MP systems ...
1932 OM_PERFDATA_OP(Inflations, inc());
1933 if (log_is_enabled(Trace, monitorinflation)) {
1934 RESOURCE_MARK_(self);
1935 lsh.print_cr("inflate(has_locker): object=" INTPTR_FORMAT ", mark="
1936 INTPTR_FORMAT ", type='%s'", p2i(object),
1937 object->mark().value(), object->klass()->external_name());
1938 }
1939 if (event.should_commit()) {
1940 post_monitor_inflate_event(&event, object, cause);
1941 }
1942 return m;
1943 }
1944
1945 // CASE: neutral
1946 // TODO-FIXME: for entry we currently inflate and then try to CAS _owner.
1947 // If we know we're inflating for entry it's better to inflate by swinging a
1948 // pre-locked ObjectMonitor pointer into the object header. A successful
1949 // CAS inflates the object *and* confers ownership to the inflating thread.
1950 // In the current implementation we use a 2-step mechanism where we CAS()
1951 // to inflate and then CAS() again to try to swing _owner from NULL to self.
1952 // An inflateTry() method that we could call from enter() would be useful.
1953
1954 // Catch if the object's header is not neutral (not locked and
1955 // not marked is what we care about here).
1956 assert(mark.is_neutral(), "invariant: header=" INTPTR_FORMAT, mark.value());
1957 ObjectMonitor* m = om_alloc(self);
1958 // prepare m for installation - set monitor to initial state
1959 m->Recycle();
1960 m->set_header(mark);
1961 // DEFLATER_MARKER is the only non-NULL value we should see here.
1962 m->try_set_owner_from(DEFLATER_MARKER, NULL);
1963 m->set_object(object);
1964 m->_Responsible = NULL;
1965 m->_SpinDuration = ObjectMonitor::Knob_SpinLimit; // consider: keep metastats by type/class
1966
1967 if (object->cas_set_mark(markWord::encode(m), mark) != mark) {
1968 m->set_header(markWord::zero());
1969 m->set_object(NULL);
1970 m->Recycle();
1971 // om_release() will reset the allocation state from New to Free.
1972 om_release(self, m, true);
1973 m = NULL;
1974 continue;
1975 // interference - the markword changed - just retry.
1976 // The state-transitions are one-way, so there's no chance of
1977 // live-lock -- "Inflated" is an absorbing state.
1978 }
1979
1980 // Once the ObjectMonitor is configured and object is associated
1981 // with the ObjectMonitor, it is safe to allow async deflation:
1982 assert(m->is_new(), "freshly allocated monitor must be new");
1983 // Release semantics are not needed to keep allocation_state from
1984 // floating up since cas_set_mark() takes care of it.
1985 m->set_allocation_state(ObjectMonitor::Old);
1986
1987 // Hopefully the performance counters are allocated on distinct
1988 // cache lines to avoid false sharing on MP systems ...
1989 OM_PERFDATA_OP(Inflations, inc());
1990 if (log_is_enabled(Trace, monitorinflation)) {
1991 RESOURCE_MARK_(self);
1992 lsh.print_cr("inflate(neutral): object=" INTPTR_FORMAT ", mark="
1993 INTPTR_FORMAT ", type='%s'", p2i(object),
1994 object->mark().value(), object->klass()->external_name());
1995 }
1996 if (event.should_commit()) {
1997 post_monitor_inflate_event(&event, object, cause);
1998 }
1999 return m;
2000 }
2001 }
2002
2003
2004 // An async deflation request is registered with the ServiceThread
2005 // and it is notified.
2006 void ObjectSynchronizer::do_safepoint_work() {
2007 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
2008
2009 log_debug(monitorinflation)("requesting async deflation of idle monitors.");
2010 // Request deflation of idle monitors by the ServiceThread:
2011 set_is_async_deflation_requested(true);
2012 MonitorLocker ml(Service_lock, Mutex::_no_safepoint_check_flag);
2013 ml.notify_all();
2014
2015 if (log_is_enabled(Debug, monitorinflation)) {
2016 // exit_globals()'s call to audit_and_print_stats() is done
2017 // at the Info level and not at a safepoint.
2018 ObjectSynchronizer::audit_and_print_stats(false /* on_exit */);
2019 }
2020 }
2021
2022 // Deflate the specified ObjectMonitor if not in-use using a JavaThread.
2023 // Returns true if it was deflated and false otherwise.
2024 //
2025 // The async deflation protocol sets owner to DEFLATER_MARKER and
2026 // makes contentions negative as signals to contending threads that
2027 // an async deflation is in progress. There are a number of checks
2028 // as part of the protocol to make sure that the calling thread has
2029 // not lost the race to a contending thread.
2030 //
2031 // The ObjectMonitor has been successfully async deflated when:
2032 // (contentions < 0)
2033 // Contending threads that see that condition know to retry their operation.
2034 //
2035 bool ObjectSynchronizer::deflate_monitor_using_JT(ObjectMonitor* mid,
2036 ObjectMonitor** free_head_p,
2037 ObjectMonitor** free_tail_p) {
2038 assert(Thread::current()->is_Java_thread(), "precondition");
2039 // A newly allocated ObjectMonitor should not be seen here so we
2040 // avoid an endless inflate/deflate cycle.
2041 assert(mid->is_old(), "must be old: allocation_state=%d",
2042 (int) mid->allocation_state());
2043
2044 if (mid->is_busy()) {
2045 // Easy checks are first - the ObjectMonitor is busy so no deflation.
2046 return false;
2047 }
2048
2049 // Set a NULL owner to DEFLATER_MARKER to force any contending thread
2050 // through the slow path. This is just the first part of the async
2051 // deflation dance.
2052 if (mid->try_set_owner_from(NULL, DEFLATER_MARKER) != NULL) {
2053 // The owner field is no longer NULL so we lost the race since the
2054 // ObjectMonitor is now busy.
2055 return false;
2056 }
2057
2058 if (mid->contentions() > 0 || mid->_waiters != 0) {
2059 // Another thread has raced to enter the ObjectMonitor after
2060 // mid->is_busy() above or has already entered and waited on
2061 // it which makes it busy so no deflation. Restore owner to
2062 // NULL if it is still DEFLATER_MARKER.
2063 if (mid->try_set_owner_from(DEFLATER_MARKER, NULL) != DEFLATER_MARKER) {
2064 // Deferred decrement for the JT EnterI() that cancelled the async deflation.
2065 mid->add_to_contentions(-1);
2066 }
2067 return false;
2068 }
2069
2070 // Make a zero contentions field negative to force any contending threads
2071 // to retry. This is the second part of the async deflation dance.
2072 if (Atomic::cmpxchg(&mid->_contentions, (jint)0, -max_jint) != 0) {
2073 // Contentions was no longer 0 so we lost the race since the
2074 // ObjectMonitor is now busy. Restore owner to NULL if it is
2075 // still DEFLATER_MARKER:
2076 if (mid->try_set_owner_from(DEFLATER_MARKER, NULL) != DEFLATER_MARKER) {
2077 // Deferred decrement for the JT EnterI() that cancelled the async deflation.
2078 mid->add_to_contentions(-1);
2079 }
2080 return false;
2081 }
2082
2083 // Sanity checks for the races:
2084 guarantee(mid->owner_is_DEFLATER_MARKER(), "must be deflater marker");
2085 guarantee(mid->contentions() < 0, "must be negative: contentions=%d",
2086 mid->contentions());
2087 guarantee(mid->_waiters == 0, "must be 0: waiters=%d", mid->_waiters);
2088 guarantee(mid->_cxq == NULL, "must be no contending threads: cxq="
2089 INTPTR_FORMAT, p2i(mid->_cxq));
2090 guarantee(mid->_EntryList == NULL,
2091 "must be no entering threads: EntryList=" INTPTR_FORMAT,
2092 p2i(mid->_EntryList));
2093
2094 const oop obj = (oop) mid->object();
2095 if (log_is_enabled(Trace, monitorinflation)) {
2096 RESOURCE_MARK;
2097 log_trace(monitorinflation)("deflate_monitor_using_JT: "
2098 "object=" INTPTR_FORMAT ", mark="
2099 INTPTR_FORMAT ", type='%s'",
2100 p2i(obj), obj->mark().value(),
2101 obj->klass()->external_name());
2102 }
2103
2104 // Install the old mark word if nobody else has already done it.
2105 mid->install_displaced_markword_in_object(obj);
2106 mid->clear_common();
2107
2108 assert(mid->object() == NULL, "must be NULL: object=" INTPTR_FORMAT,
2109 p2i(mid->object()));
2110 assert(mid->is_free(), "must be free: allocation_state=%d",
2111 (int)mid->allocation_state());
2112
2113 // Move the deflated ObjectMonitor to the working free list
2114 // defined by free_head_p and free_tail_p.
2115 if (*free_head_p == NULL) {
2116 // First one on the list.
2117 *free_head_p = mid;
2118 }
2119 if (*free_tail_p != NULL) {
2120 // We append to the list so the caller can use mid->_next_om
2121 // to fix the linkages in its context.
2122 ObjectMonitor* prevtail = *free_tail_p;
2123 // prevtail should have been cleaned up by the caller:
2124 #ifdef ASSERT
2125 ObjectMonitor* l_next_om = unmarked_next(prevtail);
2126 assert(l_next_om == NULL, "must be NULL: _next_om=" INTPTR_FORMAT, p2i(l_next_om));
2127 #endif
2128 om_lock(prevtail);
2129 prevtail->set_next_om(mid); // prevtail now points to mid (and is unlocked)
2130 }
2131 *free_tail_p = mid;
2132
2133 // At this point, mid->_next_om still refers to its current
2134 // value and another ObjectMonitor's _next_om field still
2135 // refers to this ObjectMonitor. Those linkages have to be
2136 // cleaned up by the caller who has the complete context.
2137
2138 // We leave owner == DEFLATER_MARKER and contentions < 0
2139 // to force any racing threads to retry.
2140 return true; // Success, ObjectMonitor has been deflated.
2141 }
2142
2143 // Walk a given ObjectMonitor list and deflate idle ObjectMonitors using
2144 // a JavaThread. Returns the number of deflated ObjectMonitors. The given
2145 // list could be a per-thread in-use list or the global in-use list.
2146 // If a safepoint has started, then we save state via saved_mid_in_use_p
2147 // and return to the caller to honor the safepoint.
2148 //
2149 int ObjectSynchronizer::deflate_monitor_list_using_JT(ObjectMonitor** list_p,
2150 int* count_p,
2151 ObjectMonitor** free_head_p,
2152 ObjectMonitor** free_tail_p,
2153 ObjectMonitor** saved_mid_in_use_p) {
2154 JavaThread* self = JavaThread::current();
2155
2156 ObjectMonitor* cur_mid_in_use = NULL;
2157 ObjectMonitor* mid = NULL;
2158 ObjectMonitor* next = NULL;
2159 ObjectMonitor* next_next = NULL;
2160 int deflated_count = 0;
2161 NoSafepointVerifier nsv;
2162
2163 // We use the more complicated lock-cur_mid_in_use-and-mid-as-we-go
2164 // protocol because om_release() can do list deletions in parallel;
2165 // this also prevents races with a list walker thread. We also
2166 // lock-next-next-as-we-go to prevent an om_flush() that is behind
2167 // this thread from passing us.
2168 if (*saved_mid_in_use_p == NULL) {
2169 // No saved state so start at the beginning.
2170 // Lock the list head so we can possibly deflate it:
2171 if ((mid = get_list_head_locked(list_p)) == NULL) {
2172 return 0; // The list is empty so nothing to deflate.
2173 }
2174 next = unmarked_next(mid);
2175 } else {
2176 // We're restarting after a safepoint so restore the necessary state
2177 // before we resume.
2178 cur_mid_in_use = *saved_mid_in_use_p;
2179 // Lock cur_mid_in_use so we can possibly update its
2180 // next field to extract a deflated ObjectMonitor.
2181 om_lock(cur_mid_in_use);
2182 mid = unmarked_next(cur_mid_in_use);
2183 if (mid == NULL) {
2184 om_unlock(cur_mid_in_use);
2185 *saved_mid_in_use_p = NULL;
2186 return 0; // The remainder is empty so nothing more to deflate.
2187 }
2188 // Lock mid so we can possibly deflate it:
2189 om_lock(mid);
2190 next = unmarked_next(mid);
2191 }
2192
2193 while (true) {
2194 // The current mid is locked at this point. If we have a
2195 // cur_mid_in_use, then it is also locked at this point.
2196
2197 if (next != NULL) {
2198 // We lock next so that an om_flush() thread that is behind us
2199 // cannot pass us when we unlock the current mid.
2200 om_lock(next);
2201 next_next = unmarked_next(next);
2202 }
2203
2204 // Only try to deflate if there is an associated Java object and if
2205 // mid is old (is not newly allocated and is not newly freed).
2206 if (mid->object() != NULL && mid->is_old() &&
2207 deflate_monitor_using_JT(mid, free_head_p, free_tail_p)) {
2208 // Deflation succeeded and already updated free_head_p and
2209 // free_tail_p as needed. Finish the move to the local free list
2210 // by unlinking mid from the global or per-thread in-use list.
2211 if (cur_mid_in_use == NULL) {
2212 // mid is the list head and it is locked. Switch the list head
2213 // to next which is also locked (if not NULL) and also leave
2214 // mid locked. Release semantics needed since not all code paths
2215 // in deflate_monitor_using_JT() ensure memory consistency.
2216 Atomic::release_store(list_p, next);
2217 } else {
2218 ObjectMonitor* locked_next = mark_om_ptr(next);
2219 // mid and cur_mid_in_use are locked. Switch cur_mid_in_use's
2220 // next field to locked_next and also leave mid locked.
2221 // Release semantics needed since not all code paths in
2222 // deflate_monitor_using_JT() ensure memory consistency.
2223 cur_mid_in_use->release_set_next_om(locked_next);
2224 }
2225 // At this point mid is disconnected from the in-use list so
2226 // its lock longer has any effects on in-use list.
2227 deflated_count++;
2228 Atomic::dec(count_p);
2229 // mid is current tail in the free_head_p list so NULL terminate
2230 // it (which also unlocks it). No release semantics needed since
2231 // Atomic::dec() already provides it.
2232 mid->set_next_om(NULL);
2233
2234 // All the list management is done so move on to the next one:
2235 mid = next; // mid keeps non-NULL next's locked state
2236 next = next_next;
2237 } else {
2238 // mid is considered in-use if it does not have an associated
2239 // Java object or mid is not old or deflation did not succeed.
2240 // A mid->is_new() node can be seen here when it is freshly
2241 // returned by om_alloc() (and skips the deflation code path).
2242 // A mid->is_old() node can be seen here when deflation failed.
2243 // A mid->is_free() node can be seen here when a fresh node from
2244 // om_alloc() is released by om_release() due to losing the race
2245 // in inflate().
2246
2247 // All the list management is done so move on to the next one:
2248 if (cur_mid_in_use != NULL) {
2249 om_unlock(cur_mid_in_use);
2250 }
2251 // The next cur_mid_in_use keeps mid's lock state so
2252 // that it is stable for a possible next field change. It
2253 // cannot be modified by om_release() while it is locked.
2254 cur_mid_in_use = mid;
2255 mid = next; // mid keeps non-NULL next's locked state
2256 next = next_next;
2257
2258 if (SafepointMechanism::should_block(self) &&
2259 // Acquire semantics are not needed on this list load since
2260 // it is not dependent on the following load which does have
2261 // acquire semantics.
2262 cur_mid_in_use != Atomic::load(list_p) && cur_mid_in_use->is_old()) {
2263 // If a safepoint has started and cur_mid_in_use is not the list
2264 // head and is old, then it is safe to use as saved state. Return
2265 // to the caller before blocking.
2266 *saved_mid_in_use_p = cur_mid_in_use;
2267 om_unlock(cur_mid_in_use);
2268 if (mid != NULL) {
2269 om_unlock(mid);
2270 }
2271 return deflated_count;
2272 }
2273 }
2274 if (mid == NULL) {
2275 if (cur_mid_in_use != NULL) {
2276 om_unlock(cur_mid_in_use);
2277 }
2278 break; // Reached end of the list so nothing more to deflate.
2279 }
2280
2281 // The current mid's next field is locked at this point. If we have
2282 // a cur_mid_in_use, then it is also locked at this point.
2283 }
2284 // We finished the list without a safepoint starting so there's
2285 // no need to save state.
2286 *saved_mid_in_use_p = NULL;
2287 return deflated_count;
2288 }
2289
2290 class HandshakeForDeflation : public HandshakeClosure {
2291 public:
2292 HandshakeForDeflation() : HandshakeClosure("HandshakeForDeflation") {}
2293
2294 void do_thread(Thread* thread) {
2295 log_trace(monitorinflation)("HandshakeForDeflation::do_thread: thread="
2296 INTPTR_FORMAT, p2i(thread));
2297 }
2298 };
2299
2300 void ObjectSynchronizer::deflate_idle_monitors_using_JT() {
2301 // Deflate any global idle monitors.
2302 deflate_global_idle_monitors_using_JT();
2303
2304 int count = 0;
2305 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) {
2306 if (Atomic::load(&jt->om_in_use_count) > 0 && !jt->is_exiting()) {
2307 // This JavaThread is using ObjectMonitors so deflate any that
2308 // are idle unless this JavaThread is exiting; do not race with
2309 // ObjectSynchronizer::om_flush().
2310 deflate_per_thread_idle_monitors_using_JT(jt);
2311 count++;
2312 }
2313 }
2314 if (count > 0) {
2315 log_debug(monitorinflation)("did async deflation of idle monitors for %d thread(s).", count);
2316 }
2317
2318 log_info(monitorinflation)("async global_population=%d, global_in_use_count=%d, "
2319 "global_free_count=%d, global_wait_count=%d",
2320 Atomic::load(&om_list_globals._population),
2321 Atomic::load(&om_list_globals._in_use_count),
2322 Atomic::load(&om_list_globals._free_count),
2323 Atomic::load(&om_list_globals._wait_count));
2324
2325 // The ServiceThread's async deflation request has been processed.
2326 _last_async_deflation_time_ns = os::javaTimeNanos();
2327 set_is_async_deflation_requested(false);
2328
2329 if (Atomic::load(&om_list_globals._wait_count) > 0) {
2330 // There are deflated ObjectMonitors waiting for a handshake
2331 // (or a safepoint) for safety.
2332
2333 ObjectMonitor* list = Atomic::load(&om_list_globals._wait_list);
2334 assert(list != NULL, "om_list_globals._wait_list must not be NULL");
2335 int count = Atomic::load(&om_list_globals._wait_count);
2336 Atomic::store(&om_list_globals._wait_count, 0);
2337 OrderAccess::storestore(); // Make sure counter update is seen first.
2338 Atomic::store(&om_list_globals._wait_list, (ObjectMonitor*)NULL);
2339
2340 // Find the tail for prepend_list_to_common(). No need to mark
2341 // ObjectMonitors for this list walk since only the deflater
2342 // thread manages the wait list.
2343 #ifdef ASSERT
2344 int l_count = 0;
2345 #endif
2346 ObjectMonitor* tail = NULL;
2347 for (ObjectMonitor* n = list; n != NULL; n = unmarked_next(n)) {
2348 tail = n;
2349 #ifdef ASSERT
2350 l_count++;
2351 #endif
2352 }
2353 assert(count == l_count, "count=%d != l_count=%d", count, l_count);
2354
2355 // Will execute a safepoint if !ThreadLocalHandshakes:
2356 HandshakeForDeflation hfd_hc;
2357 Handshake::execute(&hfd_hc);
2358
2359 prepend_list_to_common(list, tail, count, &om_list_globals._free_list,
2360 &om_list_globals._free_count);
2361
2362 log_info(monitorinflation)("moved %d idle monitors from global waiting list to global free list", count);
2363 }
2364 }
2365
2366 // Deflate global idle ObjectMonitors using a JavaThread.
2367 //
2368 void ObjectSynchronizer::deflate_global_idle_monitors_using_JT() {
2369 assert(Thread::current()->is_Java_thread(), "precondition");
2370 JavaThread* self = JavaThread::current();
2371
2372 deflate_common_idle_monitors_using_JT(true /* is_global */, self);
2373 }
2374
2375 // Deflate the specified JavaThread's idle ObjectMonitors using a JavaThread.
2376 //
2377 void ObjectSynchronizer::deflate_per_thread_idle_monitors_using_JT(JavaThread* target) {
2378 assert(Thread::current()->is_Java_thread(), "precondition");
2379
2380 deflate_common_idle_monitors_using_JT(false /* !is_global */, target);
2381 }
2382
2383 // Deflate global or per-thread idle ObjectMonitors using a JavaThread.
2384 //
2385 void ObjectSynchronizer::deflate_common_idle_monitors_using_JT(bool is_global, JavaThread* target) {
2386 JavaThread* self = JavaThread::current();
2387
2388 int deflated_count = 0;
2389 ObjectMonitor* free_head_p = NULL; // Local SLL of scavenged ObjectMonitors
2390 ObjectMonitor* free_tail_p = NULL;
2391 ObjectMonitor* saved_mid_in_use_p = NULL;
2392 elapsedTimer timer;
2393
2394 if (log_is_enabled(Info, monitorinflation)) {
2395 timer.start();
2396 }
2397
2398 if (is_global) {
2399 OM_PERFDATA_OP(MonExtant, set_value(Atomic::load(&om_list_globals._in_use_count)));
2400 } else {
2401 OM_PERFDATA_OP(MonExtant, inc(Atomic::load(&target->om_in_use_count)));
2402 }
2403
2404 do {
2405 int local_deflated_count;
2406 if (is_global) {
2407 local_deflated_count =
2408 deflate_monitor_list_using_JT(&om_list_globals._in_use_list,
2409 &om_list_globals._in_use_count,
2410 &free_head_p, &free_tail_p,
2411 &saved_mid_in_use_p);
2412 } else {
2413 local_deflated_count =
2414 deflate_monitor_list_using_JT(&target->om_in_use_list,
2415 &target->om_in_use_count, &free_head_p,
2416 &free_tail_p, &saved_mid_in_use_p);
2417 }
2418 deflated_count += local_deflated_count;
2419
2420 if (free_head_p != NULL) {
2421 // Move the deflated ObjectMonitors to the global free list.
2422 guarantee(free_tail_p != NULL && local_deflated_count > 0, "free_tail_p=" INTPTR_FORMAT ", local_deflated_count=%d", p2i(free_tail_p), local_deflated_count);
2423 // Note: The target thread can be doing an om_alloc() that
2424 // is trying to prepend an ObjectMonitor on its in-use list
2425 // at the same time that we have deflated the current in-use
2426 // list head and put it on the local free list. prepend_to_common()
2427 // will detect the race and retry which avoids list corruption,
2428 // but the next field in free_tail_p can flicker to marked
2429 // and then unmarked while prepend_to_common() is sorting it
2430 // all out.
2431 #ifdef ASSERT
2432 ObjectMonitor* l_next_om = unmarked_next(free_tail_p);
2433 assert(l_next_om == NULL, "must be NULL: _next_om=" INTPTR_FORMAT, p2i(l_next_om));
2434 #endif
2435
2436 prepend_list_to_global_wait_list(free_head_p, free_tail_p, local_deflated_count);
2437
2438 OM_PERFDATA_OP(Deflations, inc(local_deflated_count));
2439 }
2440
2441 if (saved_mid_in_use_p != NULL) {
2442 // deflate_monitor_list_using_JT() detected a safepoint starting.
2443 timer.stop();
2444 {
2445 if (is_global) {
2446 log_debug(monitorinflation)("pausing deflation of global idle monitors for a safepoint.");
2447 } else {
2448 log_debug(monitorinflation)("jt=" INTPTR_FORMAT ": pausing deflation of per-thread idle monitors for a safepoint.", p2i(target));
2449 }
2450 assert(SafepointMechanism::should_block(self), "sanity check");
2451 ThreadBlockInVM blocker(self);
2452 }
2453 // Prepare for another loop after the safepoint.
2454 free_head_p = NULL;
2455 free_tail_p = NULL;
2456 if (log_is_enabled(Info, monitorinflation)) {
2457 timer.start();
2458 }
2459 }
2460 } while (saved_mid_in_use_p != NULL);
2461 timer.stop();
2462
2463 LogStreamHandle(Debug, monitorinflation) lsh_debug;
2464 LogStreamHandle(Info, monitorinflation) lsh_info;
2465 LogStream* ls = NULL;
2466 if (log_is_enabled(Debug, monitorinflation)) {
2467 ls = &lsh_debug;
2468 } else if (deflated_count != 0 && log_is_enabled(Info, monitorinflation)) {
2469 ls = &lsh_info;
2470 }
2471 if (ls != NULL) {
2472 if (is_global) {
2473 ls->print_cr("async-deflating global idle monitors, %3.7f secs, %d monitors", timer.seconds(), deflated_count);
2474 } else {
2475 ls->print_cr("jt=" INTPTR_FORMAT ": async-deflating per-thread idle monitors, %3.7f secs, %d monitors", p2i(target), timer.seconds(), deflated_count);
2476 }
2477 }
2478 }
2479
2480 // Monitor cleanup on JavaThread::exit
2481
2482 // Iterate through monitor cache and attempt to release thread's monitors
2483 // Gives up on a particular monitor if an exception occurs, but continues
2484 // the overall iteration, swallowing the exception.
2485 class ReleaseJavaMonitorsClosure: public MonitorClosure {
2486 private:
2487 TRAPS;
2488
2489 public:
2490 ReleaseJavaMonitorsClosure(Thread* thread) : THREAD(thread) {}
2491 void do_monitor(ObjectMonitor* mid) {
2492 if (mid->owner() == THREAD) {
2493 (void)mid->complete_exit(CHECK);
2494 }
2495 }
2496 };
2497
2498 // Release all inflated monitors owned by THREAD. Lightweight monitors are
2499 // ignored. This is meant to be called during JNI thread detach which assumes
2500 // all remaining monitors are heavyweight. All exceptions are swallowed.
2501 // Scanning the extant monitor list can be time consuming.
2502 // A simple optimization is to add a per-thread flag that indicates a thread
2503 // called jni_monitorenter() during its lifetime.
2504 //
2505 // Instead of NoSafepointVerifier it might be cheaper to
2506 // use an idiom of the form:
2507 // auto int tmp = SafepointSynchronize::_safepoint_counter ;
2508 // <code that must not run at safepoint>
2509 // guarantee (((tmp ^ _safepoint_counter) | (tmp & 1)) == 0) ;
2510 // Since the tests are extremely cheap we could leave them enabled
2511 // for normal product builds.
2512
2513 void ObjectSynchronizer::release_monitors_owned_by_thread(TRAPS) {
2514 assert(THREAD == JavaThread::current(), "must be current Java thread");
2515 NoSafepointVerifier nsv;
2516 ReleaseJavaMonitorsClosure rjmc(THREAD);
2517 ObjectSynchronizer::monitors_iterate(&rjmc);
2518 THREAD->clear_pending_exception();
2519 }
2520
2521 const char* ObjectSynchronizer::inflate_cause_name(const InflateCause cause) {
2522 switch (cause) {
2523 case inflate_cause_vm_internal: return "VM Internal";
2524 case inflate_cause_monitor_enter: return "Monitor Enter";
2525 case inflate_cause_wait: return "Monitor Wait";
2526 case inflate_cause_notify: return "Monitor Notify";
2527 case inflate_cause_hash_code: return "Monitor Hash Code";
2528 case inflate_cause_jni_enter: return "JNI Monitor Enter";
2529 case inflate_cause_jni_exit: return "JNI Monitor Exit";
2530 default:
2531 ShouldNotReachHere();
2532 }
2533 return "Unknown";
2534 }
2535
2536 //------------------------------------------------------------------------------
2537 // Debugging code
2538
2539 u_char* ObjectSynchronizer::get_gvars_addr() {
2540 return (u_char*)&GVars;
2541 }
2542
2543 u_char* ObjectSynchronizer::get_gvars_hc_sequence_addr() {
2544 return (u_char*)&GVars.hc_sequence;
2545 }
2546
2547 size_t ObjectSynchronizer::get_gvars_size() {
2548 return sizeof(SharedGlobals);
2549 }
2550
2551 u_char* ObjectSynchronizer::get_gvars_stw_random_addr() {
2552 return (u_char*)&GVars.stw_random;
2553 }
2554
2555 // This function can be called at a safepoint or it can be called when
2556 // we are trying to exit the VM. When we are trying to exit the VM, the
2557 // list walker functions can run in parallel with the other list
2558 // operations so spin-locking is used for safety.
2559 //
2560 // Calls to this function can be added in various places as a debugging
2561 // aid; pass 'true' for the 'on_exit' parameter to have in-use monitor
2562 // details logged at the Info level and 'false' for the 'on_exit'
2563 // parameter to have in-use monitor details logged at the Trace level.
2564 //
2565 void ObjectSynchronizer::audit_and_print_stats(bool on_exit) {
2566 assert(on_exit || SafepointSynchronize::is_at_safepoint(), "invariant");
2567
2568 LogStreamHandle(Debug, monitorinflation) lsh_debug;
2569 LogStreamHandle(Info, monitorinflation) lsh_info;
2570 LogStreamHandle(Trace, monitorinflation) lsh_trace;
2571 LogStream* ls = NULL;
2572 if (log_is_enabled(Trace, monitorinflation)) {
2573 ls = &lsh_trace;
2574 } else if (log_is_enabled(Debug, monitorinflation)) {
2575 ls = &lsh_debug;
2576 } else if (log_is_enabled(Info, monitorinflation)) {
2577 ls = &lsh_info;
2578 }
2579 assert(ls != NULL, "sanity check");
2580
2581 // Log counts for the global and per-thread monitor lists:
2582 int chk_om_population = log_monitor_list_counts(ls);
2583 int error_cnt = 0;
2584
2585 ls->print_cr("Checking global lists:");
2586
2587 // Check om_list_globals._population:
2588 if (Atomic::load(&om_list_globals._population) == chk_om_population) {
2589 ls->print_cr("global_population=%d equals chk_om_population=%d",
2590 Atomic::load(&om_list_globals._population), chk_om_population);
2591 } else {
2592 // With fine grained locks on the monitor lists, it is possible for
2593 // log_monitor_list_counts() to return a value that doesn't match
2594 // om_list_globals._population. So far a higher value has been
2595 // seen in testing so something is being double counted by
2596 // log_monitor_list_counts().
2597 ls->print_cr("WARNING: global_population=%d is not equal to "
2598 "chk_om_population=%d",
2599 Atomic::load(&om_list_globals._population), chk_om_population);
2600 }
2601
2602 // Check om_list_globals._in_use_list and om_list_globals._in_use_count:
2603 chk_global_in_use_list_and_count(ls, &error_cnt);
2604
2605 // Check om_list_globals._free_list and om_list_globals._free_count:
2606 chk_global_free_list_and_count(ls, &error_cnt);
2607
2608 // Check om_list_globals._wait_list and om_list_globals._wait_count:
2609 chk_global_wait_list_and_count(ls, &error_cnt);
2610
2611 ls->print_cr("Checking per-thread lists:");
2612
2613 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) {
2614 // Check om_in_use_list and om_in_use_count:
2615 chk_per_thread_in_use_list_and_count(jt, ls, &error_cnt);
2616
2617 // Check om_free_list and om_free_count:
2618 chk_per_thread_free_list_and_count(jt, ls, &error_cnt);
2619 }
2620
2621 if (error_cnt == 0) {
2622 ls->print_cr("No errors found in monitor list checks.");
2623 } else {
2624 log_error(monitorinflation)("found monitor list errors: error_cnt=%d", error_cnt);
2625 }
2626
2627 if ((on_exit && log_is_enabled(Info, monitorinflation)) ||
2628 (!on_exit && log_is_enabled(Trace, monitorinflation))) {
2629 // When exiting this log output is at the Info level. When called
2630 // at a safepoint, this log output is at the Trace level since
2631 // there can be a lot of it.
2632 log_in_use_monitor_details(ls);
2633 }
2634
2635 ls->flush();
2636
2637 guarantee(error_cnt == 0, "ERROR: found monitor list errors: error_cnt=%d", error_cnt);
2638 }
2639
2640 // Check a free monitor entry; log any errors.
2641 void ObjectSynchronizer::chk_free_entry(JavaThread* jt, ObjectMonitor* n,
2642 outputStream * out, int *error_cnt_p) {
2643 stringStream ss;
2644 if (n->is_busy()) {
2645 if (jt != NULL) {
2646 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
2647 ": free per-thread monitor must not be busy: %s", p2i(jt),
2648 p2i(n), n->is_busy_to_string(&ss));
2649 } else {
2650 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": free global monitor "
2651 "must not be busy: %s", p2i(n), n->is_busy_to_string(&ss));
2652 }
2653 *error_cnt_p = *error_cnt_p + 1;
2654 }
2655 if (n->header().value() != 0) {
2656 if (jt != NULL) {
2657 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
2658 ": free per-thread monitor must have NULL _header "
2659 "field: _header=" INTPTR_FORMAT, p2i(jt), p2i(n),
2660 n->header().value());
2661 *error_cnt_p = *error_cnt_p + 1;
2662 }
2663 }
2664 if (n->object() != NULL) {
2665 if (jt != NULL) {
2666 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
2667 ": free per-thread monitor must have NULL _object "
2668 "field: _object=" INTPTR_FORMAT, p2i(jt), p2i(n),
2669 p2i(n->object()));
2670 } else {
2671 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": free global monitor "
2672 "must have NULL _object field: _object=" INTPTR_FORMAT,
2673 p2i(n), p2i(n->object()));
2674 }
2675 *error_cnt_p = *error_cnt_p + 1;
2676 }
2677 }
2678
2679 // Lock the next ObjectMonitor for traversal and unlock the current
2680 // ObjectMonitor. Returns the next ObjectMonitor if there is one.
2681 // Otherwise returns NULL (after unlocking the current ObjectMonitor).
2682 // This function is used by the various list walker functions to
2683 // safely walk a list without allowing an ObjectMonitor to be moved
2684 // to another list in the middle of a walk.
2685 static ObjectMonitor* lock_next_for_traversal(ObjectMonitor* cur) {
2686 assert(is_locked(cur), "cur=" INTPTR_FORMAT " must be locked", p2i(cur));
2687 ObjectMonitor* next = unmarked_next(cur);
2688 if (next == NULL) { // Reached the end of the list.
2689 om_unlock(cur);
2690 return NULL;
2691 }
2692 om_lock(next); // Lock next before unlocking current to keep
2693 om_unlock(cur); // from being by-passed by another thread.
2694 return next;
2695 }
2696
2697 // Check the global free list and count; log the results of the checks.
2698 void ObjectSynchronizer::chk_global_free_list_and_count(outputStream * out,
2699 int *error_cnt_p) {
2700 int chk_om_free_count = 0;
2701 ObjectMonitor* cur = NULL;
2702 if ((cur = get_list_head_locked(&om_list_globals._free_list)) != NULL) {
2703 // Marked the global free list head so process the list.
2704 while (true) {
2705 chk_free_entry(NULL /* jt */, cur, out, error_cnt_p);
2706 chk_om_free_count++;
2707
2708 cur = lock_next_for_traversal(cur);
2709 if (cur == NULL) {
2710 break;
2711 }
2712 }
2713 }
2714 int l_free_count = Atomic::load(&om_list_globals._free_count);
2715 if (l_free_count == chk_om_free_count) {
2716 out->print_cr("global_free_count=%d equals chk_om_free_count=%d",
2717 l_free_count, chk_om_free_count);
2718 } else {
2719 // With fine grained locks on om_list_globals._free_list, it
2720 // is possible for an ObjectMonitor to be prepended to
2721 // om_list_globals._free_list after we started calculating
2722 // chk_om_free_count so om_list_globals._free_count may not
2723 // match anymore.
2724 out->print_cr("WARNING: global_free_count=%d is not equal to "
2725 "chk_om_free_count=%d", l_free_count, chk_om_free_count);
2726 }
2727 }
2728
2729 // Check the global wait list and count; log the results of the checks.
2730 void ObjectSynchronizer::chk_global_wait_list_and_count(outputStream * out,
2731 int *error_cnt_p) {
2732 int chk_om_wait_count = 0;
2733 ObjectMonitor* cur = NULL;
2734 if ((cur = get_list_head_locked(&om_list_globals._wait_list)) != NULL) {
2735 // Marked the global wait list head so process the list.
2736 while (true) {
2737 // Rules for om_list_globals._wait_list are the same as for
2738 // om_list_globals._free_list:
2739 chk_free_entry(NULL /* jt */, cur, out, error_cnt_p);
2740 chk_om_wait_count++;
2741
2742 cur = lock_next_for_traversal(cur);
2743 if (cur == NULL) {
2744 break;
2745 }
2746 }
2747 }
2748 if (Atomic::load(&om_list_globals._wait_count) == chk_om_wait_count) {
2749 out->print_cr("global_wait_count=%d equals chk_om_wait_count=%d",
2750 Atomic::load(&om_list_globals._wait_count), chk_om_wait_count);
2751 } else {
2752 out->print_cr("ERROR: global_wait_count=%d is not equal to "
2753 "chk_om_wait_count=%d",
2754 Atomic::load(&om_list_globals._wait_count), chk_om_wait_count);
2755 *error_cnt_p = *error_cnt_p + 1;
2756 }
2757 }
2758
2759 // Check the global in-use list and count; log the results of the checks.
2760 void ObjectSynchronizer::chk_global_in_use_list_and_count(outputStream * out,
2761 int *error_cnt_p) {
2762 int chk_om_in_use_count = 0;
2763 ObjectMonitor* cur = NULL;
2764 if ((cur = get_list_head_locked(&om_list_globals._in_use_list)) != NULL) {
2765 // Marked the global in-use list head so process the list.
2766 while (true) {
2767 chk_in_use_entry(NULL /* jt */, cur, out, error_cnt_p);
2768 chk_om_in_use_count++;
2769
2770 cur = lock_next_for_traversal(cur);
2771 if (cur == NULL) {
2772 break;
2773 }
2774 }
2775 }
2776 int l_in_use_count = Atomic::load(&om_list_globals._in_use_count);
2777 if (l_in_use_count == chk_om_in_use_count) {
2778 out->print_cr("global_in_use_count=%d equals chk_om_in_use_count=%d",
2779 l_in_use_count, chk_om_in_use_count);
2780 } else {
2781 // With fine grained locks on the monitor lists, it is possible for
2782 // an exiting JavaThread to put its in-use ObjectMonitors on the
2783 // global in-use list after chk_om_in_use_count is calculated above.
2784 out->print_cr("WARNING: global_in_use_count=%d is not equal to chk_om_in_use_count=%d",
2785 l_in_use_count, chk_om_in_use_count);
2786 }
2787 }
2788
2789 // Check an in-use monitor entry; log any errors.
2790 void ObjectSynchronizer::chk_in_use_entry(JavaThread* jt, ObjectMonitor* n,
2791 outputStream * out, int *error_cnt_p) {
2792 if (n->header().value() == 0) {
2793 if (jt != NULL) {
2794 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
2795 ": in-use per-thread monitor must have non-NULL _header "
2796 "field.", p2i(jt), p2i(n));
2797 } else {
2798 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": in-use global monitor "
2799 "must have non-NULL _header field.", p2i(n));
2800 }
2801 *error_cnt_p = *error_cnt_p + 1;
2802 }
2803 if (n->object() == NULL) {
2804 if (jt != NULL) {
2805 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
2806 ": in-use per-thread monitor must have non-NULL _object "
2807 "field.", p2i(jt), p2i(n));
2808 } else {
2809 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": in-use global monitor "
2810 "must have non-NULL _object field.", p2i(n));
2811 }
2812 *error_cnt_p = *error_cnt_p + 1;
2813 }
2814 const oop obj = (oop)n->object();
2815 const markWord mark = obj->mark();
2816 if (!mark.has_monitor()) {
2817 if (jt != NULL) {
2818 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
2819 ": in-use per-thread monitor's object does not think "
2820 "it has a monitor: obj=" INTPTR_FORMAT ", mark="
2821 INTPTR_FORMAT, p2i(jt), p2i(n), p2i(obj), mark.value());
2822 } else {
2823 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": in-use global "
2824 "monitor's object does not think it has a monitor: obj="
2825 INTPTR_FORMAT ", mark=" INTPTR_FORMAT, p2i(n),
2826 p2i(obj), mark.value());
2827 }
2828 *error_cnt_p = *error_cnt_p + 1;
2829 }
2830 ObjectMonitor* const obj_mon = mark.monitor();
2831 if (n != obj_mon) {
2832 if (jt != NULL) {
2833 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
2834 ": in-use per-thread monitor's object does not refer "
2835 "to the same monitor: obj=" INTPTR_FORMAT ", mark="
2836 INTPTR_FORMAT ", obj_mon=" INTPTR_FORMAT, p2i(jt),
2837 p2i(n), p2i(obj), mark.value(), p2i(obj_mon));
2838 } else {
2839 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": in-use global "
2840 "monitor's object does not refer to the same monitor: obj="
2841 INTPTR_FORMAT ", mark=" INTPTR_FORMAT ", obj_mon="
2842 INTPTR_FORMAT, p2i(n), p2i(obj), mark.value(), p2i(obj_mon));
2843 }
2844 *error_cnt_p = *error_cnt_p + 1;
2845 }
2846 }
2847
2848 // Check the thread's free list and count; log the results of the checks.
2849 void ObjectSynchronizer::chk_per_thread_free_list_and_count(JavaThread *jt,
2850 outputStream * out,
2851 int *error_cnt_p) {
2852 int chk_om_free_count = 0;
2853 ObjectMonitor* cur = NULL;
2854 if ((cur = get_list_head_locked(&jt->om_free_list)) != NULL) {
2855 // Marked the per-thread free list head so process the list.
2856 while (true) {
2857 chk_free_entry(jt, cur, out, error_cnt_p);
2858 chk_om_free_count++;
2859
2860 cur = lock_next_for_traversal(cur);
2861 if (cur == NULL) {
2862 break;
2863 }
2864 }
2865 }
2866 int l_om_free_count = Atomic::load(&jt->om_free_count);
2867 if (l_om_free_count == chk_om_free_count) {
2868 out->print_cr("jt=" INTPTR_FORMAT ": om_free_count=%d equals "
2869 "chk_om_free_count=%d", p2i(jt), l_om_free_count, chk_om_free_count);
2870 } else {
2871 out->print_cr("ERROR: jt=" INTPTR_FORMAT ": om_free_count=%d is not "
2872 "equal to chk_om_free_count=%d", p2i(jt), l_om_free_count,
2873 chk_om_free_count);
2874 *error_cnt_p = *error_cnt_p + 1;
2875 }
2876 }
2877
2878 // Check the thread's in-use list and count; log the results of the checks.
2879 void ObjectSynchronizer::chk_per_thread_in_use_list_and_count(JavaThread *jt,
2880 outputStream * out,
2881 int *error_cnt_p) {
2882 int chk_om_in_use_count = 0;
2883 ObjectMonitor* cur = NULL;
2884 if ((cur = get_list_head_locked(&jt->om_in_use_list)) != NULL) {
2885 // Marked the per-thread in-use list head so process the list.
2886 while (true) {
2887 chk_in_use_entry(jt, cur, out, error_cnt_p);
2888 chk_om_in_use_count++;
2889
2890 cur = lock_next_for_traversal(cur);
2891 if (cur == NULL) {
2892 break;
2893 }
2894 }
2895 }
2896 int l_om_in_use_count = Atomic::load(&jt->om_in_use_count);
2897 if (l_om_in_use_count == chk_om_in_use_count) {
2898 out->print_cr("jt=" INTPTR_FORMAT ": om_in_use_count=%d equals "
2899 "chk_om_in_use_count=%d", p2i(jt), l_om_in_use_count,
2900 chk_om_in_use_count);
2901 } else {
2902 out->print_cr("ERROR: jt=" INTPTR_FORMAT ": om_in_use_count=%d is not "
2903 "equal to chk_om_in_use_count=%d", p2i(jt), l_om_in_use_count,
2904 chk_om_in_use_count);
2905 *error_cnt_p = *error_cnt_p + 1;
2906 }
2907 }
2908
2909 // Log details about ObjectMonitors on the in-use lists. The 'BHL'
2910 // flags indicate why the entry is in-use, 'object' and 'object type'
2911 // indicate the associated object and its type.
2912 void ObjectSynchronizer::log_in_use_monitor_details(outputStream * out) {
2913 stringStream ss;
2914 if (Atomic::load(&om_list_globals._in_use_count) > 0) {
2915 out->print_cr("In-use global monitor info:");
2916 out->print_cr("(B -> is_busy, H -> has hash code, L -> lock status)");
2917 out->print_cr("%18s %s %18s %18s",
2918 "monitor", "BHL", "object", "object type");
2919 out->print_cr("================== === ================== ==================");
2920 ObjectMonitor* cur = NULL;
2921 if ((cur = get_list_head_locked(&om_list_globals._in_use_list)) != NULL) {
2922 // Marked the global in-use list head so process the list.
2923 while (true) {
2924 const oop obj = (oop) cur->object();
2925 const markWord mark = cur->header();
2926 RESOURCE_MARK;
2927 out->print(INTPTR_FORMAT " %d%d%d " INTPTR_FORMAT " %s", p2i(cur),
2928 cur->is_busy() != 0, mark.hash() != 0, cur->owner() != NULL,
2929 p2i(obj), obj->klass()->external_name());
2930 if (cur->is_busy() != 0) {
2931 out->print(" (%s)", cur->is_busy_to_string(&ss));
2932 ss.reset();
2933 }
2934 out->cr();
2935
2936 cur = lock_next_for_traversal(cur);
2937 if (cur == NULL) {
2938 break;
2939 }
2940 }
2941 }
2942 }
2943
2944 out->print_cr("In-use per-thread monitor info:");
2945 out->print_cr("(B -> is_busy, H -> has hash code, L -> lock status)");
2946 out->print_cr("%18s %18s %s %18s %18s",
2947 "jt", "monitor", "BHL", "object", "object type");
2948 out->print_cr("================== ================== === ================== ==================");
2949 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) {
2950 ObjectMonitor* cur = NULL;
2951 if ((cur = get_list_head_locked(&jt->om_in_use_list)) != NULL) {
2952 // Marked the global in-use list head so process the list.
2953 while (true) {
2954 const oop obj = (oop) cur->object();
2955 const markWord mark = cur->header();
2956 RESOURCE_MARK;
2957 out->print(INTPTR_FORMAT " " INTPTR_FORMAT " %d%d%d " INTPTR_FORMAT
2958 " %s", p2i(jt), p2i(cur), cur->is_busy() != 0,
2959 mark.hash() != 0, cur->owner() != NULL, p2i(obj),
2960 obj->klass()->external_name());
2961 if (cur->is_busy() != 0) {
2962 out->print(" (%s)", cur->is_busy_to_string(&ss));
2963 ss.reset();
2964 }
2965 out->cr();
2966
2967 cur = lock_next_for_traversal(cur);
2968 if (cur == NULL) {
2969 break;
2970 }
2971 }
2972 }
2973 }
2974
2975 out->flush();
2976 }
2977
2978 // Log counts for the global and per-thread monitor lists and return
2979 // the population count.
2980 int ObjectSynchronizer::log_monitor_list_counts(outputStream * out) {
2981 int pop_count = 0;
2982 out->print_cr("%18s %10s %10s %10s %10s",
2983 "Global Lists:", "InUse", "Free", "Wait", "Total");
2984 out->print_cr("================== ========== ========== ========== ==========");
2985 int l_in_use_count = Atomic::load(&om_list_globals._in_use_count);
2986 int l_free_count = Atomic::load(&om_list_globals._free_count);
2987 int l_wait_count = Atomic::load(&om_list_globals._wait_count);
2988 out->print_cr("%18s %10d %10d %10d %10d", "", l_in_use_count,
2989 l_free_count, l_wait_count,
2990 Atomic::load(&om_list_globals._population));
2991 pop_count += l_in_use_count + l_free_count + l_wait_count;
2992
2993 out->print_cr("%18s %10s %10s %10s",
2994 "Per-Thread Lists:", "InUse", "Free", "Provision");
2995 out->print_cr("================== ========== ========== ==========");
2996
2997 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) {
2998 int l_om_in_use_count = Atomic::load(&jt->om_in_use_count);
2999 int l_om_free_count = Atomic::load(&jt->om_free_count);
3000 out->print_cr(INTPTR_FORMAT " %10d %10d %10d", p2i(jt),
3001 l_om_in_use_count, l_om_free_count, jt->om_free_provision);
3002 pop_count += l_om_in_use_count + l_om_free_count;
3003 }
3004 return pop_count;
3005 }
3006
3007 #ifndef PRODUCT
3008
3009 // Check if monitor belongs to the monitor cache
3010 // The list is grow-only so it's *relatively* safe to traverse
3011 // the list of extant blocks without taking a lock.
3012
3013 int ObjectSynchronizer::verify_objmon_isinpool(ObjectMonitor *monitor) {
3014 PaddedObjectMonitor* block = Atomic::load(&g_block_list);
3015 while (block != NULL) {
3016 assert(block->object() == CHAINMARKER, "must be a block header");
3017 if (monitor > &block[0] && monitor < &block[_BLOCKSIZE]) {
3018 address mon = (address)monitor;
3019 address blk = (address)block;
3020 size_t diff = mon - blk;
3021 assert((diff % sizeof(PaddedObjectMonitor)) == 0, "must be aligned");
3022 return 1;
3023 }
3024 // unmarked_next() is not needed with g_block_list (no locking
3025 // used with block linkage _next_om fields).
3026 block = (PaddedObjectMonitor*)block->next_om();
3027 }
3028 return 0;
3029 }
3030
3031 #endif