1 /*
2 * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
35 #include "metaprogramming/isIntegral.hpp"
36 #include "metaprogramming/isPointer.hpp"
37 #include "metaprogramming/isSame.hpp"
38 #include "metaprogramming/isVolatile.hpp"
39 #include "oops/accessDecorators.hpp"
40 #include "oops/oopsHierarchy.hpp"
41 #include "runtime/globals.hpp"
42 #include "utilities/debug.hpp"
43 #include "utilities/globalDefinitions.hpp"
44
45
46 // This metafunction returns either oop or narrowOop depending on whether
47 // an access needs to use compressed oops or not.
48 template <DecoratorSet decorators>
49 struct HeapOopType: AllStatic {
50 static const bool needs_oop_compress = HasDecorator<decorators, INTERNAL_CONVERT_COMPRESSED_OOP>::value &&
51 HasDecorator<decorators, INTERNAL_RT_USE_COMPRESSED_OOPS>::value;
52 typedef typename Conditional<needs_oop_compress, narrowOop, oop>::type type;
53 };
54
55 namespace AccessInternal {
56 enum BarrierType {
57 BARRIER_STORE,
58 BARRIER_STORE_AT,
59 BARRIER_LOAD,
60 BARRIER_LOAD_AT,
61 BARRIER_ATOMIC_CMPXCHG,
62 BARRIER_ATOMIC_CMPXCHG_AT,
63 BARRIER_ATOMIC_XCHG,
64 BARRIER_ATOMIC_XCHG_AT,
65 BARRIER_ARRAYCOPY,
66 BARRIER_CLONE,
67 BARRIER_RESOLVE
68 };
69
70 template <DecoratorSet decorators, typename T>
71 struct MustConvertCompressedOop: public IntegralConstant<bool,
72 HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value &&
73 IsSame<typename HeapOopType<decorators>::type, narrowOop>::value &&
74 IsSame<T, oop>::value> {};
75
76 // This metafunction returns an appropriate oop type if the value is oop-like
77 // and otherwise returns the same type T.
78 template <DecoratorSet decorators, typename T>
79 struct EncodedType: AllStatic {
80 typedef typename Conditional<
81 HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value,
82 typename HeapOopType<decorators>::type, T>::type type;
83 };
84
85 template <DecoratorSet decorators>
86 inline typename HeapOopType<decorators>::type*
97 #else
98 struct PossiblyLockedAccess: public IntegralConstant<bool, (sizeof(T) > 4)> {};
99 #endif
100
101 template <DecoratorSet decorators, typename T>
102 struct AccessFunctionTypes {
103 typedef T (*load_at_func_t)(oop base, ptrdiff_t offset);
104 typedef void (*store_at_func_t)(oop base, ptrdiff_t offset, T value);
105 typedef T (*atomic_cmpxchg_at_func_t)(T new_value, oop base, ptrdiff_t offset, T compare_value);
106 typedef T (*atomic_xchg_at_func_t)(T new_value, oop base, ptrdiff_t offset);
107
108 typedef T (*load_func_t)(void* addr);
109 typedef void (*store_func_t)(void* addr, T value);
110 typedef T (*atomic_cmpxchg_func_t)(T new_value, void* addr, T compare_value);
111 typedef T (*atomic_xchg_func_t)(T new_value, void* addr);
112
113 typedef void (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
114 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
115 size_t length);
116 typedef void (*clone_func_t)(oop src, oop dst, size_t size);
117 typedef oop (*resolve_func_t)(oop obj);
118 };
119
120 template <DecoratorSet decorators>
121 struct AccessFunctionTypes<decorators, void> {
122 typedef void (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, void* src,
123 arrayOop dst_obj, size_t dst_offset_in_bytes, void* dst,
124 size_t length);
125 };
126
127 template <DecoratorSet decorators, typename T, BarrierType barrier> struct AccessFunction {};
128
129 #define ACCESS_GENERATE_ACCESS_FUNCTION(bt, func) \
130 template <DecoratorSet decorators, typename T> \
131 struct AccessFunction<decorators, T, bt>: AllStatic{ \
132 typedef typename AccessFunctionTypes<decorators, T>::func type; \
133 }
134 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE, store_func_t);
135 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE_AT, store_at_func_t);
136 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD, load_func_t);
137 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD_AT, load_at_func_t);
138 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG, atomic_cmpxchg_func_t);
139 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG_AT, atomic_cmpxchg_at_func_t);
140 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG, atomic_xchg_func_t);
141 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG_AT, atomic_xchg_at_func_t);
142 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ARRAYCOPY, arraycopy_func_t);
143 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_CLONE, clone_func_t);
144 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_RESOLVE, resolve_func_t);
145 #undef ACCESS_GENERATE_ACCESS_FUNCTION
146
147 template <DecoratorSet decorators, typename T, BarrierType barrier_type>
148 typename AccessFunction<decorators, T, barrier_type>::type resolve_barrier();
149
150 template <DecoratorSet decorators, typename T, BarrierType barrier_type>
151 typename AccessFunction<decorators, T, barrier_type>::type resolve_oop_barrier();
152
153 class AccessLocker {
154 public:
155 AccessLocker();
156 ~AccessLocker();
157 };
158 bool wide_atomic_needs_locking();
159
160 void* field_addr(oop base, ptrdiff_t offset);
161
162 // Forward calls to Copy:: in the cpp file to reduce dependencies and allow
163 // faster build times, given how frequently included access is.
389 return load<T>(field_addr(base, offset));
390 }
391
392 template <typename T>
393 static T atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) {
394 return atomic_cmpxchg(new_value, field_addr(base, offset), compare_value);
395 }
396
397 template <typename T>
398 static T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
399 return atomic_xchg(new_value, field_addr(base, offset));
400 }
401
402 template <typename T>
403 static void oop_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
404 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
405 size_t length);
406
407 static void clone(oop src, oop dst, size_t size);
408
409 static oop resolve(oop obj) { return obj; }
410 };
411
412 // Below is the implementation of the first 4 steps of the template pipeline:
413 // * Step 1: Set default decorators and decay types. This step gets rid of CV qualifiers
414 // and sets default decorators to sensible values.
415 // * Step 2: Reduce types. This step makes sure there is only a single T type and not
416 // multiple types. The P type of the address and T type of the value must
417 // match.
418 // * Step 3: Pre-runtime dispatch. This step checks whether a runtime call can be
419 // avoided, and in that case avoids it (calling raw accesses or
420 // primitive accesses in a build that does not require primitive GC barriers)
421 // * Step 4: Runtime-dispatch. This step performs a runtime dispatch to the corresponding
422 // BarrierSet::AccessBarrier accessor that attaches GC-required barriers
423 // to the access.
424
425 namespace AccessInternal {
426 template <typename T>
427 struct OopOrNarrowOopInternal: AllStatic {
428 typedef oop type;
572 size_t length) {
573 return _arraycopy_func(src_obj, src_offset_in_bytes, src_raw,
574 dst_obj, dst_offset_in_bytes, dst_raw,
575 length);
576 }
577 };
578
579 template <DecoratorSet decorators, typename T>
580 struct RuntimeDispatch<decorators, T, BARRIER_CLONE>: AllStatic {
581 typedef typename AccessFunction<decorators, T, BARRIER_CLONE>::type func_t;
582 static func_t _clone_func;
583
584 static void clone_init(oop src, oop dst, size_t size);
585
586 static inline void clone(oop src, oop dst, size_t size) {
587 _clone_func(src, dst, size);
588 }
589 };
590
591 template <DecoratorSet decorators, typename T>
592 struct RuntimeDispatch<decorators, T, BARRIER_RESOLVE>: AllStatic {
593 typedef typename AccessFunction<decorators, T, BARRIER_RESOLVE>::type func_t;
594 static func_t _resolve_func;
595
596 static oop resolve_init(oop obj);
597
598 static inline oop resolve(oop obj) {
599 return _resolve_func(obj);
600 }
601 };
602
603 // Initialize the function pointers to point to the resolving function.
604 template <DecoratorSet decorators, typename T>
605 typename AccessFunction<decorators, T, BARRIER_STORE>::type
606 RuntimeDispatch<decorators, T, BARRIER_STORE>::_store_func = &store_init;
607
608 template <DecoratorSet decorators, typename T>
609 typename AccessFunction<decorators, T, BARRIER_STORE_AT>::type
610 RuntimeDispatch<decorators, T, BARRIER_STORE_AT>::_store_at_func = &store_at_init;
611
625 typename AccessFunction<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>::type
626 RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>::_atomic_cmpxchg_at_func = &atomic_cmpxchg_at_init;
627
628 template <DecoratorSet decorators, typename T>
629 typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG>::type
630 RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG>::_atomic_xchg_func = &atomic_xchg_init;
631
632 template <DecoratorSet decorators, typename T>
633 typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG_AT>::type
634 RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>::_atomic_xchg_at_func = &atomic_xchg_at_init;
635
636 template <DecoratorSet decorators, typename T>
637 typename AccessFunction<decorators, T, BARRIER_ARRAYCOPY>::type
638 RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>::_arraycopy_func = &arraycopy_init;
639
640 template <DecoratorSet decorators, typename T>
641 typename AccessFunction<decorators, T, BARRIER_CLONE>::type
642 RuntimeDispatch<decorators, T, BARRIER_CLONE>::_clone_func = &clone_init;
643
644 template <DecoratorSet decorators, typename T>
645 typename AccessFunction<decorators, T, BARRIER_RESOLVE>::type
646 RuntimeDispatch<decorators, T, BARRIER_RESOLVE>::_resolve_func = &resolve_init;
647
648 // Step 3: Pre-runtime dispatching.
649 // The PreRuntimeDispatch class is responsible for filtering the barrier strength
650 // decorators. That is, for AS_RAW, it hardwires the accesses without a runtime
651 // dispatch point. Otherwise it goes through a runtime check if hardwiring was
652 // not possible.
653 struct PreRuntimeDispatch: AllStatic {
654 template<DecoratorSet decorators>
655 struct CanHardwireRaw: public IntegralConstant<
656 bool,
657 !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value || // primitive access
658 !HasDecorator<decorators, INTERNAL_CONVERT_COMPRESSED_OOP>::value || // don't care about compressed oops (oop* address)
659 HasDecorator<decorators, INTERNAL_RT_USE_COMPRESSED_OOPS>::value> // we can infer we use compressed oops (narrowOop* address)
660 {};
661
662 static const DecoratorSet convert_compressed_oops = INTERNAL_RT_USE_COMPRESSED_OOPS | INTERNAL_CONVERT_COMPRESSED_OOP;
663
664 template<DecoratorSet decorators>
946 }
947 }
948
949 template <DecoratorSet decorators>
950 inline static typename EnableIf<
951 HasDecorator<decorators, AS_RAW>::value>::type
952 clone(oop src, oop dst, size_t size) {
953 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
954 Raw::clone(src, dst, size);
955 }
956
957 template <DecoratorSet decorators>
958 inline static typename EnableIf<
959 !HasDecorator<decorators, AS_RAW>::value>::type
960 clone(oop src, oop dst, size_t size) {
961 RuntimeDispatch<decorators, oop, BARRIER_CLONE>::clone(src, dst, size);
962 }
963
964 template <DecoratorSet decorators>
965 inline static typename EnableIf<
966 HasDecorator<decorators, INTERNAL_BT_TO_SPACE_INVARIANT>::value, oop>::type
967 resolve(oop obj) {
968 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
969 return Raw::resolve(obj);
970 }
971
972 template <DecoratorSet decorators>
973 inline static typename EnableIf<
974 !HasDecorator<decorators, INTERNAL_BT_TO_SPACE_INVARIANT>::value, oop>::type
975 resolve(oop obj) {
976 return RuntimeDispatch<decorators, oop, BARRIER_RESOLVE>::resolve(obj);
977 }
978 };
979
980 // Step 2: Reduce types.
981 // Enforce that for non-oop types, T and P have to be strictly the same.
982 // P is the type of the address and T is the type of the values.
983 // As for oop types, it is allow to send T in {narrowOop, oop} and
984 // P in {narrowOop, oop, HeapWord*}. The following rules apply according to
985 // the subsequent table. (columns are P, rows are T)
1248 }
1249
1250 template <DecoratorSet decorators, typename T>
1251 inline void arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw,
1252 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
1253 size_t length) {
1254 STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ||
1255 (IsSame<T, void>::value || IsIntegral<T>::value) ||
1256 IsFloatingPoint<T>::value)); // arraycopy allows type erased void elements
1257 typedef typename Decay<T>::type DecayedT;
1258 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | IS_ARRAY | IN_HEAP>::value;
1259 arraycopy_reduce_types<expanded_decorators>(src_obj, src_offset_in_bytes, const_cast<DecayedT*>(src_raw),
1260 dst_obj, dst_offset_in_bytes, const_cast<DecayedT*>(dst_raw),
1261 length);
1262 }
1263
1264 template <DecoratorSet decorators>
1265 inline void clone(oop src, oop dst, size_t size) {
1266 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1267 PreRuntimeDispatch::clone<expanded_decorators>(src, dst, size);
1268 }
1269
1270 template <DecoratorSet decorators>
1271 inline oop resolve(oop obj) {
1272 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1273 return PreRuntimeDispatch::resolve<expanded_decorators>(obj);
1274 }
1275
1276 // Infer the type that should be returned from an Access::oop_load.
1277 template <typename P, DecoratorSet decorators>
1278 class OopLoadProxy: public StackObj {
1279 private:
1280 P *const _addr;
1281 public:
1282 OopLoadProxy(P* addr) : _addr(addr) {}
1283
1284 inline operator oop() {
1285 return load<decorators | INTERNAL_VALUE_IS_OOP, P, oop>(_addr);
1286 }
1287
|
1 /*
2 * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
35 #include "metaprogramming/isIntegral.hpp"
36 #include "metaprogramming/isPointer.hpp"
37 #include "metaprogramming/isSame.hpp"
38 #include "metaprogramming/isVolatile.hpp"
39 #include "oops/accessDecorators.hpp"
40 #include "oops/oopsHierarchy.hpp"
41 #include "runtime/globals.hpp"
42 #include "utilities/debug.hpp"
43 #include "utilities/globalDefinitions.hpp"
44
45
46 // This metafunction returns either oop or narrowOop depending on whether
47 // an access needs to use compressed oops or not.
48 template <DecoratorSet decorators>
49 struct HeapOopType: AllStatic {
50 static const bool needs_oop_compress = HasDecorator<decorators, INTERNAL_CONVERT_COMPRESSED_OOP>::value &&
51 HasDecorator<decorators, INTERNAL_RT_USE_COMPRESSED_OOPS>::value;
52 typedef typename Conditional<needs_oop_compress, narrowOop, oop>::type type;
53 };
54
55 // This meta-function returns either oop or narrowOop depending on whether
56 // a back-end needs to consider compressed oops types or not.
57 template <DecoratorSet decorators>
58 struct ValueOopType: AllStatic {
59 static const bool needs_oop_compress = HasDecorator<decorators, INTERNAL_RT_USE_COMPRESSED_OOPS>::value;
60 typedef typename Conditional<needs_oop_compress, narrowOop, oop>::type type;
61 };
62
63 namespace AccessInternal {
64 enum BarrierType {
65 BARRIER_STORE,
66 BARRIER_STORE_AT,
67 BARRIER_LOAD,
68 BARRIER_LOAD_AT,
69 BARRIER_ATOMIC_CMPXCHG,
70 BARRIER_ATOMIC_CMPXCHG_AT,
71 BARRIER_ATOMIC_XCHG,
72 BARRIER_ATOMIC_XCHG_AT,
73 BARRIER_ARRAYCOPY,
74 BARRIER_CLONE,
75 BARRIER_VALUE_COPY,
76 BARRIER_RESOLVE
77 };
78
79 template <DecoratorSet decorators, typename T>
80 struct MustConvertCompressedOop: public IntegralConstant<bool,
81 HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value &&
82 IsSame<typename HeapOopType<decorators>::type, narrowOop>::value &&
83 IsSame<T, oop>::value> {};
84
85 // This metafunction returns an appropriate oop type if the value is oop-like
86 // and otherwise returns the same type T.
87 template <DecoratorSet decorators, typename T>
88 struct EncodedType: AllStatic {
89 typedef typename Conditional<
90 HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value,
91 typename HeapOopType<decorators>::type, T>::type type;
92 };
93
94 template <DecoratorSet decorators>
95 inline typename HeapOopType<decorators>::type*
106 #else
107 struct PossiblyLockedAccess: public IntegralConstant<bool, (sizeof(T) > 4)> {};
108 #endif
109
110 template <DecoratorSet decorators, typename T>
111 struct AccessFunctionTypes {
112 typedef T (*load_at_func_t)(oop base, ptrdiff_t offset);
113 typedef void (*store_at_func_t)(oop base, ptrdiff_t offset, T value);
114 typedef T (*atomic_cmpxchg_at_func_t)(T new_value, oop base, ptrdiff_t offset, T compare_value);
115 typedef T (*atomic_xchg_at_func_t)(T new_value, oop base, ptrdiff_t offset);
116
117 typedef T (*load_func_t)(void* addr);
118 typedef void (*store_func_t)(void* addr, T value);
119 typedef T (*atomic_cmpxchg_func_t)(T new_value, void* addr, T compare_value);
120 typedef T (*atomic_xchg_func_t)(T new_value, void* addr);
121
122 typedef void (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
123 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
124 size_t length);
125 typedef void (*clone_func_t)(oop src, oop dst, size_t size);
126 typedef void (*value_copy_func_t)(void* src, void* dst, ValueKlass* md);
127 typedef oop (*resolve_func_t)(oop obj);
128 };
129
130 template <DecoratorSet decorators>
131 struct AccessFunctionTypes<decorators, void> {
132 typedef void (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, void* src,
133 arrayOop dst_obj, size_t dst_offset_in_bytes, void* dst,
134 size_t length);
135 };
136
137 template <DecoratorSet decorators, typename T, BarrierType barrier> struct AccessFunction {};
138
139 #define ACCESS_GENERATE_ACCESS_FUNCTION(bt, func) \
140 template <DecoratorSet decorators, typename T> \
141 struct AccessFunction<decorators, T, bt>: AllStatic{ \
142 typedef typename AccessFunctionTypes<decorators, T>::func type; \
143 }
144 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE, store_func_t);
145 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE_AT, store_at_func_t);
146 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD, load_func_t);
147 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD_AT, load_at_func_t);
148 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG, atomic_cmpxchg_func_t);
149 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG_AT, atomic_cmpxchg_at_func_t);
150 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG, atomic_xchg_func_t);
151 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG_AT, atomic_xchg_at_func_t);
152 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ARRAYCOPY, arraycopy_func_t);
153 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_CLONE, clone_func_t);
154 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_VALUE_COPY, value_copy_func_t);
155 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_RESOLVE, resolve_func_t);
156 #undef ACCESS_GENERATE_ACCESS_FUNCTION
157
158 template <DecoratorSet decorators, typename T, BarrierType barrier_type>
159 typename AccessFunction<decorators, T, barrier_type>::type resolve_barrier();
160
161 template <DecoratorSet decorators, typename T, BarrierType barrier_type>
162 typename AccessFunction<decorators, T, barrier_type>::type resolve_oop_barrier();
163
164 class AccessLocker {
165 public:
166 AccessLocker();
167 ~AccessLocker();
168 };
169 bool wide_atomic_needs_locking();
170
171 void* field_addr(oop base, ptrdiff_t offset);
172
173 // Forward calls to Copy:: in the cpp file to reduce dependencies and allow
174 // faster build times, given how frequently included access is.
400 return load<T>(field_addr(base, offset));
401 }
402
403 template <typename T>
404 static T atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) {
405 return atomic_cmpxchg(new_value, field_addr(base, offset), compare_value);
406 }
407
408 template <typename T>
409 static T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
410 return atomic_xchg(new_value, field_addr(base, offset));
411 }
412
413 template <typename T>
414 static void oop_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
415 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
416 size_t length);
417
418 static void clone(oop src, oop dst, size_t size);
419
420 static void value_copy(void* src, void* dst, ValueKlass* md);
421
422 static oop resolve(oop obj) { return obj; }
423 };
424
425 // Below is the implementation of the first 4 steps of the template pipeline:
426 // * Step 1: Set default decorators and decay types. This step gets rid of CV qualifiers
427 // and sets default decorators to sensible values.
428 // * Step 2: Reduce types. This step makes sure there is only a single T type and not
429 // multiple types. The P type of the address and T type of the value must
430 // match.
431 // * Step 3: Pre-runtime dispatch. This step checks whether a runtime call can be
432 // avoided, and in that case avoids it (calling raw accesses or
433 // primitive accesses in a build that does not require primitive GC barriers)
434 // * Step 4: Runtime-dispatch. This step performs a runtime dispatch to the corresponding
435 // BarrierSet::AccessBarrier accessor that attaches GC-required barriers
436 // to the access.
437
438 namespace AccessInternal {
439 template <typename T>
440 struct OopOrNarrowOopInternal: AllStatic {
441 typedef oop type;
585 size_t length) {
586 return _arraycopy_func(src_obj, src_offset_in_bytes, src_raw,
587 dst_obj, dst_offset_in_bytes, dst_raw,
588 length);
589 }
590 };
591
592 template <DecoratorSet decorators, typename T>
593 struct RuntimeDispatch<decorators, T, BARRIER_CLONE>: AllStatic {
594 typedef typename AccessFunction<decorators, T, BARRIER_CLONE>::type func_t;
595 static func_t _clone_func;
596
597 static void clone_init(oop src, oop dst, size_t size);
598
599 static inline void clone(oop src, oop dst, size_t size) {
600 _clone_func(src, dst, size);
601 }
602 };
603
604 template <DecoratorSet decorators, typename T>
605 struct RuntimeDispatch<decorators, T, BARRIER_VALUE_COPY>: AllStatic {
606 typedef typename AccessFunction<decorators, T, BARRIER_VALUE_COPY>::type func_t;
607 static func_t _value_copy_func;
608
609 static void value_copy_init(void* src, void* dst, ValueKlass* md);
610
611 static inline void value_copy(void* src, void* dst, ValueKlass* md) {
612 _value_copy_func(src, dst, md);
613 }
614 };
615
616 template <DecoratorSet decorators, typename T>
617 struct RuntimeDispatch<decorators, T, BARRIER_RESOLVE>: AllStatic {
618 typedef typename AccessFunction<decorators, T, BARRIER_RESOLVE>::type func_t;
619 static func_t _resolve_func;
620
621 static oop resolve_init(oop obj);
622
623 static inline oop resolve(oop obj) {
624 return _resolve_func(obj);
625 }
626 };
627
628 // Initialize the function pointers to point to the resolving function.
629 template <DecoratorSet decorators, typename T>
630 typename AccessFunction<decorators, T, BARRIER_STORE>::type
631 RuntimeDispatch<decorators, T, BARRIER_STORE>::_store_func = &store_init;
632
633 template <DecoratorSet decorators, typename T>
634 typename AccessFunction<decorators, T, BARRIER_STORE_AT>::type
635 RuntimeDispatch<decorators, T, BARRIER_STORE_AT>::_store_at_func = &store_at_init;
636
650 typename AccessFunction<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>::type
651 RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>::_atomic_cmpxchg_at_func = &atomic_cmpxchg_at_init;
652
653 template <DecoratorSet decorators, typename T>
654 typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG>::type
655 RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG>::_atomic_xchg_func = &atomic_xchg_init;
656
657 template <DecoratorSet decorators, typename T>
658 typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG_AT>::type
659 RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>::_atomic_xchg_at_func = &atomic_xchg_at_init;
660
661 template <DecoratorSet decorators, typename T>
662 typename AccessFunction<decorators, T, BARRIER_ARRAYCOPY>::type
663 RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>::_arraycopy_func = &arraycopy_init;
664
665 template <DecoratorSet decorators, typename T>
666 typename AccessFunction<decorators, T, BARRIER_CLONE>::type
667 RuntimeDispatch<decorators, T, BARRIER_CLONE>::_clone_func = &clone_init;
668
669 template <DecoratorSet decorators, typename T>
670 typename AccessFunction<decorators, T, BARRIER_VALUE_COPY>::type
671 RuntimeDispatch<decorators, T, BARRIER_VALUE_COPY>::_value_copy_func = &value_copy_init;
672
673 template <DecoratorSet decorators, typename T>
674 typename AccessFunction<decorators, T, BARRIER_RESOLVE>::type
675 RuntimeDispatch<decorators, T, BARRIER_RESOLVE>::_resolve_func = &resolve_init;
676
677 // Step 3: Pre-runtime dispatching.
678 // The PreRuntimeDispatch class is responsible for filtering the barrier strength
679 // decorators. That is, for AS_RAW, it hardwires the accesses without a runtime
680 // dispatch point. Otherwise it goes through a runtime check if hardwiring was
681 // not possible.
682 struct PreRuntimeDispatch: AllStatic {
683 template<DecoratorSet decorators>
684 struct CanHardwireRaw: public IntegralConstant<
685 bool,
686 !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value || // primitive access
687 !HasDecorator<decorators, INTERNAL_CONVERT_COMPRESSED_OOP>::value || // don't care about compressed oops (oop* address)
688 HasDecorator<decorators, INTERNAL_RT_USE_COMPRESSED_OOPS>::value> // we can infer we use compressed oops (narrowOop* address)
689 {};
690
691 static const DecoratorSet convert_compressed_oops = INTERNAL_RT_USE_COMPRESSED_OOPS | INTERNAL_CONVERT_COMPRESSED_OOP;
692
693 template<DecoratorSet decorators>
975 }
976 }
977
978 template <DecoratorSet decorators>
979 inline static typename EnableIf<
980 HasDecorator<decorators, AS_RAW>::value>::type
981 clone(oop src, oop dst, size_t size) {
982 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
983 Raw::clone(src, dst, size);
984 }
985
986 template <DecoratorSet decorators>
987 inline static typename EnableIf<
988 !HasDecorator<decorators, AS_RAW>::value>::type
989 clone(oop src, oop dst, size_t size) {
990 RuntimeDispatch<decorators, oop, BARRIER_CLONE>::clone(src, dst, size);
991 }
992
993 template <DecoratorSet decorators>
994 inline static typename EnableIf<
995 HasDecorator<decorators, AS_RAW>::value>::type
996 value_copy(void* src, void* dst, ValueKlass* md) {
997 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
998 Raw::value_copy(src, dst, md);
999 }
1000
1001 template <DecoratorSet decorators>
1002 inline static typename EnableIf<
1003 !HasDecorator<decorators, AS_RAW>::value>::type
1004 value_copy(void* src, void* dst, ValueKlass* md) {
1005 const DecoratorSet expanded_decorators = decorators;
1006 RuntimeDispatch<expanded_decorators, void*, BARRIER_VALUE_COPY>::value_copy(src, dst, md);
1007 }
1008
1009
1010 template <DecoratorSet decorators>
1011 inline static typename EnableIf<
1012 HasDecorator<decorators, INTERNAL_BT_TO_SPACE_INVARIANT>::value, oop>::type
1013 resolve(oop obj) {
1014 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
1015 return Raw::resolve(obj);
1016 }
1017
1018 template <DecoratorSet decorators>
1019 inline static typename EnableIf<
1020 !HasDecorator<decorators, INTERNAL_BT_TO_SPACE_INVARIANT>::value, oop>::type
1021 resolve(oop obj) {
1022 return RuntimeDispatch<decorators, oop, BARRIER_RESOLVE>::resolve(obj);
1023 }
1024 };
1025
1026 // Step 2: Reduce types.
1027 // Enforce that for non-oop types, T and P have to be strictly the same.
1028 // P is the type of the address and T is the type of the values.
1029 // As for oop types, it is allow to send T in {narrowOop, oop} and
1030 // P in {narrowOop, oop, HeapWord*}. The following rules apply according to
1031 // the subsequent table. (columns are P, rows are T)
1294 }
1295
1296 template <DecoratorSet decorators, typename T>
1297 inline void arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw,
1298 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
1299 size_t length) {
1300 STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ||
1301 (IsSame<T, void>::value || IsIntegral<T>::value) ||
1302 IsFloatingPoint<T>::value)); // arraycopy allows type erased void elements
1303 typedef typename Decay<T>::type DecayedT;
1304 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | IS_ARRAY | IN_HEAP>::value;
1305 arraycopy_reduce_types<expanded_decorators>(src_obj, src_offset_in_bytes, const_cast<DecayedT*>(src_raw),
1306 dst_obj, dst_offset_in_bytes, const_cast<DecayedT*>(dst_raw),
1307 length);
1308 }
1309
1310 template <DecoratorSet decorators>
1311 inline void clone(oop src, oop dst, size_t size) {
1312 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1313 PreRuntimeDispatch::clone<expanded_decorators>(src, dst, size);
1314 }
1315
1316 template <DecoratorSet decorators>
1317 inline void value_copy(void* src, void* dst, ValueKlass* md) {
1318 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1319 PreRuntimeDispatch::value_copy<expanded_decorators>(src, dst, md);
1320 }
1321
1322 template <DecoratorSet decorators>
1323 inline oop resolve(oop obj) {
1324 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1325 return PreRuntimeDispatch::resolve<expanded_decorators>(obj);
1326 }
1327
1328 // Infer the type that should be returned from an Access::oop_load.
1329 template <typename P, DecoratorSet decorators>
1330 class OopLoadProxy: public StackObj {
1331 private:
1332 P *const _addr;
1333 public:
1334 OopLoadProxy(P* addr) : _addr(addr) {}
1335
1336 inline operator oop() {
1337 return load<decorators | INTERNAL_VALUE_IS_OOP, P, oop>(_addr);
1338 }
1339
|