1 /* 2 * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. 3 * Copyright (c) 2016 SAP SE. All rights reserved. 4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 * 6 * This code is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 only, as 8 * published by the Free Software Foundation. 9 * 10 * This code is distributed in the hope that it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 * version 2 for more details (a copy is included in the LICENSE file that 14 * accompanied this code). 15 * 16 * You should have received a copy of the GNU General Public License version 17 * 2 along with this work; if not, write to the Free Software Foundation, 18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19 * 20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 * or visit www.oracle.com if you need additional information or have any 22 * questions. 23 * 24 */ 25 26 #ifndef OS_CPU_LINUX_S390_VM_ATOMIC_LINUX_S390_INLINE_HPP 27 #define OS_CPU_LINUX_S390_VM_ATOMIC_LINUX_S390_INLINE_HPP 28 29 #include "runtime/atomic.hpp" 30 #include "runtime/os.hpp" 31 #include "vm_version_s390.hpp" 32 33 // Note that the compare-and-swap instructions on System z perform 34 // a serialization function before the storage operand is fetched 35 // and again after the operation is completed. 36 // 37 // Used constraint modifiers: 38 // = write-only access: Value on entry to inline-assembler code irrelevant. 39 // + read/write access: Value on entry is used; on exit value is changed. 40 // read-only access: Value on entry is used and never changed. 41 // & early-clobber access: Might be modified before all read-only operands 42 // have been used. 43 // a address register operand (not GR0). 44 // d general register operand (including GR0) 45 // Q memory operand w/o index register. 46 // 0..9 operand reference (by operand position). 47 // Used for operands that fill multiple roles. One example would be a 48 // write-only operand receiving its initial value from a read-only operand. 49 // Refer to cmpxchg(..) operand #0 and variable cmp_val for a real-life example. 50 // 51 52 // On System z, all store operations are atomic if the address where the data is stored into 53 // is an integer multiple of the data length. Furthermore, all stores are ordered: 54 // a store which occurs conceptually before another store becomes visible to other CPUs 55 // before the other store becomes visible. 56 57 //------------ 58 // Atomic::add 59 //------------ 60 // These methods force the value in memory to be augmented by the passed increment. 61 // Both, memory value and increment, are treated as 32bit signed binary integers. 62 // No overflow exceptions are recognized, and the condition code does not hold 63 // information about the value in memory. 64 // 65 // The value in memory is updated by using a compare-and-swap instruction. The 66 // instruction is retried as often as required. 67 // 68 // The return value of the method is the value that was successfully stored. At the 69 // time the caller receives back control, the value in memory may have changed already. 70 71 template<size_t byte_size> 72 struct Atomic::PlatformAdd 73 : Atomic::AddAndFetch<Atomic::PlatformAdd<byte_size> > 74 { 75 template<typename I, typename D> 76 D add_and_fetch(I add_value, D volatile* dest, cmpxchg_memory_order order) const; 77 }; 78 79 template<> 80 template<typename I, typename D> 81 inline D Atomic::PlatformAdd<4>::add_and_fetch(I inc, D volatile* dest, 82 cmpxchg_memory_order order) const { 83 STATIC_ASSERT(4 == sizeof(I)); 84 STATIC_ASSERT(4 == sizeof(D)); 85 86 D old, upd; 87 88 if (VM_Version::has_LoadAndALUAtomicV1()) { 89 __asm__ __volatile__ ( 90 " LGFR 0,%[inc] \n\t" // save increment 91 " LA 3,%[mem] \n\t" // force data address into ARG2 92 // " LAA %[upd],%[inc],%[mem] \n\t" // increment and get old value 93 // " LAA 2,0,0(3) \n\t" // actually coded instruction 94 " .byte 0xeb \n\t" // LAA main opcode 95 " .byte 0x20 \n\t" // R1,R3 96 " .byte 0x30 \n\t" // R2,disp1 97 " .byte 0x00 \n\t" // disp2,disp3 98 " .byte 0x00 \n\t" // disp4,disp5 99 " .byte 0xf8 \n\t" // LAA minor opcode 100 " AR 2,0 \n\t" // calc new value in register 101 " LR %[upd],2 \n\t" // move to result register 102 //---< outputs >--- 103 : [upd] "=&d" (upd) // write-only, updated counter value 104 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 105 //---< inputs >--- 106 : [inc] "a" (inc) // read-only. 107 //---< clobbered >--- 108 : "cc", "r0", "r2", "r3", "memory" 109 ); 110 if (order == memory_order_conservative) { 111 __asm__ __volatile__ ("bcr 14, 0" : : : "memory"); 112 } 113 } else { 114 __asm__ __volatile__ ( 115 " LLGF %[old],%[mem] \n\t" // get old value 116 "0: LA %[upd],0(%[inc],%[old]) \n\t" // calc result 117 " CS %[old],%[upd],%[mem] \n\t" // try to xchg res with mem 118 " JNE 0b \n\t" // no success? -> retry 119 //---< outputs >--- 120 : [old] "=&a" (old) // write-only, old counter value 121 , [upd] "=&d" (upd) // write-only, updated counter value 122 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 123 //---< inputs >--- 124 : [inc] "a" (inc) // read-only. 125 //---< clobbered >--- 126 : "cc", "memory" 127 ); 128 } 129 130 return upd; 131 } 132 133 134 template<> 135 template<typename I, typename D> 136 inline D Atomic::PlatformAdd<8>::add_and_fetch(I inc, D volatile* dest, 137 cmpxchg_memory_order order) const { 138 STATIC_ASSERT(8 == sizeof(I)); 139 STATIC_ASSERT(8 == sizeof(D)); 140 141 D old, upd; 142 143 if (VM_Version::has_LoadAndALUAtomicV1()) { 144 __asm__ __volatile__ ( 145 " LGR 0,%[inc] \n\t" // save increment 146 " LA 3,%[mem] \n\t" // force data address into ARG2 147 // " LAAG %[upd],%[inc],%[mem] \n\t" // increment and get old value 148 // " LAAG 2,0,0(3) \n\t" // actually coded instruction 149 " .byte 0xeb \n\t" // LAA main opcode 150 " .byte 0x20 \n\t" // R1,R3 151 " .byte 0x30 \n\t" // R2,disp1 152 " .byte 0x00 \n\t" // disp2,disp3 153 " .byte 0x00 \n\t" // disp4,disp5 154 " .byte 0xe8 \n\t" // LAA minor opcode 155 " AGR 2,0 \n\t" // calc new value in register 156 " LGR %[upd],2 \n\t" // move to result register 157 //---< outputs >--- 158 : [upd] "=&d" (upd) // write-only, updated counter value 159 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 160 //---< inputs >--- 161 : [inc] "a" (inc) // read-only. 162 //---< clobbered >--- 163 : "cc", "r0", "r2", "r3", "memory" 164 ); 165 if (order == memory_order_conservative) { 166 __asm__ __volatile__ ("bcr 14, 0" : : : "memory"); 167 } 168 } else { 169 __asm__ __volatile__ ( 170 " LG %[old],%[mem] \n\t" // get old value 171 "0: LA %[upd],0(%[inc],%[old]) \n\t" // calc result 172 " CSG %[old],%[upd],%[mem] \n\t" // try to xchg res with mem 173 " JNE 0b \n\t" // no success? -> retry 174 //---< outputs >--- 175 : [old] "=&a" (old) // write-only, old counter value 176 , [upd] "=&d" (upd) // write-only, updated counter value 177 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 178 //---< inputs >--- 179 : [inc] "a" (inc) // read-only. 180 //---< clobbered >--- 181 : "cc", "memory" 182 ); 183 } 184 185 return upd; 186 } 187 188 189 //------------- 190 // Atomic::xchg 191 //------------- 192 // These methods force the value in memory to be replaced by the new value passed 193 // in as argument. 194 // 195 // The value in memory is replaced by using a compare-and-swap instruction. The 196 // instruction is retried as often as required. This makes sure that the new 197 // value can be seen, at least for a very short period of time, by other CPUs. 198 // 199 // If we would use a normal "load(old value) store(new value)" sequence, 200 // the new value could be lost unnoticed, due to a store(new value) from 201 // another thread. 202 // 203 // The return value is the (unchanged) value from memory as it was when the 204 // replacement succeeded. 205 template<> 206 template<typename T> 207 inline T Atomic::PlatformXchg<4>::operator()(T exchange_value, 208 T volatile* dest) const { 209 STATIC_ASSERT(4 == sizeof(T)); 210 T old; 211 212 __asm__ __volatile__ ( 213 " LLGF %[old],%[mem] \n\t" // get old value 214 "0: CS %[old],%[upd],%[mem] \n\t" // try to xchg upd with mem 215 " JNE 0b \n\t" // no success? -> retry 216 //---< outputs >--- 217 : [old] "=&d" (old) // write-only, prev value irrelevant 218 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 219 //---< inputs >--- 220 : [upd] "d" (exchange_value) // read-only, value to be written to memory 221 //---< clobbered >--- 222 : "cc", "memory" 223 ); 224 225 return old; 226 } 227 228 template<> 229 template<typename T> 230 inline T Atomic::PlatformXchg<8>::operator()(T exchange_value, 231 T volatile* dest) const { 232 STATIC_ASSERT(8 == sizeof(T)); 233 T old; 234 235 __asm__ __volatile__ ( 236 " LG %[old],%[mem] \n\t" // get old value 237 "0: CSG %[old],%[upd],%[mem] \n\t" // try to xchg upd with mem 238 " JNE 0b \n\t" // no success? -> retry 239 //---< outputs >--- 240 : [old] "=&d" (old) // write-only, init from memory 241 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 242 //---< inputs >--- 243 : [upd] "d" (exchange_value) // read-only, value to be written to memory 244 //---< clobbered >--- 245 : "cc", "memory" 246 ); 247 248 return old; 249 } 250 251 //---------------- 252 // Atomic::cmpxchg 253 //---------------- 254 // These methods compare the value in memory with a given compare value. 255 // If both values compare equal, the value in memory is replaced with 256 // the exchange value. 257 // 258 // The value in memory is compared and replaced by using a compare-and-swap 259 // instruction. The instruction is NOT retried (one shot only). 260 // 261 // The return value is the (unchanged) value from memory as it was when the 262 // compare-and-swap instruction completed. A successful exchange operation 263 // is indicated by (return value == compare_value). If unsuccessful, a new 264 // exchange value can be calculated based on the return value which is the 265 // latest contents of the memory location. 266 // 267 // Inspecting the return value is the only way for the caller to determine 268 // if the compare-and-swap instruction was successful: 269 // - If return value and compare value compare equal, the compare-and-swap 270 // instruction was successful and the value in memory was replaced by the 271 // exchange value. 272 // - If return value and compare value compare unequal, the compare-and-swap 273 // instruction was not successful. The value in memory was left unchanged. 274 // 275 // The s390 processors always fence before and after the csg instructions. 276 // Thus we ignore the memory ordering argument. The docu says: "A serialization 277 // function is performed before the operand is fetched and again after the 278 // operation is completed." 279 280 // No direct support for cmpxchg of bytes; emulate using int. 281 template<> 282 struct Atomic::PlatformCmpxchg<1> : Atomic::CmpxchgByteUsingInt {}; 283 284 template<> 285 template<typename T> 286 inline T Atomic::PlatformCmpxchg<4>::operator()(T xchg_val, 287 T volatile* dest, 288 T cmp_val, 289 cmpxchg_memory_order unused) const { 290 STATIC_ASSERT(4 == sizeof(T)); 291 T old; 292 293 __asm__ __volatile__ ( 294 " CS %[old],%[upd],%[mem] \n\t" // Try to xchg upd with mem. 295 // outputs 296 : [old] "=&d" (old) // Write-only, prev value irrelevant. 297 , [mem] "+Q" (*dest) // Read/write, memory to be updated atomically. 298 // inputs 299 : [upd] "d" (xchg_val) 300 , "0" (cmp_val) // Read-only, initial value for [old] (operand #0). 301 // clobbered 302 : "cc", "memory" 303 ); 304 305 return old; 306 } 307 308 template<> 309 template<typename T> 310 inline T Atomic::PlatformCmpxchg<8>::operator()(T xchg_val, 311 T volatile* dest, 312 T cmp_val, 313 cmpxchg_memory_order unused) const { 314 STATIC_ASSERT(8 == sizeof(T)); 315 T old; 316 317 __asm__ __volatile__ ( 318 " CSG %[old],%[upd],%[mem] \n\t" // Try to xchg upd with mem. 319 // outputs 320 : [old] "=&d" (old) // Write-only, prev value irrelevant. 321 , [mem] "+Q" (*dest) // Read/write, memory to be updated atomically. 322 // inputs 323 : [upd] "d" (xchg_val) 324 , "0" (cmp_val) // Read-only, initial value for [old] (operand #0). 325 // clobbered 326 : "cc", "memory" 327 ); 328 329 return old; 330 } 331 332 #endif // OS_CPU_LINUX_S390_VM_ATOMIC_LINUX_S390_INLINE_HPP