1 /* 2 * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. 3 * Copyright (c) 2016 SAP SE. All rights reserved. 4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 * 6 * This code is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 only, as 8 * published by the Free Software Foundation. 9 * 10 * This code is distributed in the hope that it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 * version 2 for more details (a copy is included in the LICENSE file that 14 * accompanied this code). 15 * 16 * You should have received a copy of the GNU General Public License version 17 * 2 along with this work; if not, write to the Free Software Foundation, 18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19 * 20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 * or visit www.oracle.com if you need additional information or have any 22 * questions. 23 * 24 */ 25 26 #ifndef OS_CPU_LINUX_S390_VM_ATOMIC_LINUX_S390_INLINE_HPP 27 #define OS_CPU_LINUX_S390_VM_ATOMIC_LINUX_S390_INLINE_HPP 28 29 #include "runtime/atomic.hpp" 30 #include "runtime/os.hpp" 31 #include "vm_version_s390.hpp" 32 33 // Note that the compare-and-swap instructions on System z perform 34 // a serialization function before the storage operand is fetched 35 // and again after the operation is completed. 36 // 37 // Used constraint modifiers: 38 // = write-only access: Value on entry to inline-assembler code irrelevant. 39 // + read/write access: Value on entry is used; on exit value is changed. 40 // read-only access: Value on entry is used and never changed. 41 // & early-clobber access: Might be modified before all read-only operands 42 // have been used. 43 // a address register operand (not GR0). 44 // d general register operand (including GR0) 45 // Q memory operand w/o index register. 46 // 0..9 operand reference (by operand position). 47 // Used for operands that fill multiple roles. One example would be a 48 // write-only operand receiving its initial value from a read-only operand. 49 // Refer to cmpxchg(..) operand #0 and variable cmp_val for a real-life example. 50 // 51 52 // On System z, all store operations are atomic if the address where the data is stored into 53 // is an integer multiple of the data length. Furthermore, all stores are ordered: 54 // a store which occurs conceptually before another store becomes visible to other CPUs 55 // before the other store becomes visible. 56 57 //------------ 58 // Atomic::add 59 //------------ 60 // These methods force the value in memory to be augmented by the passed increment. 61 // Both, memory value and increment, are treated as 32bit signed binary integers. 62 // No overflow exceptions are recognized, and the condition code does not hold 63 // information about the value in memory. 64 // 65 // The value in memory is updated by using a compare-and-swap instruction. The 66 // instruction is retried as often as required. 67 // 68 // The return value of the method is the value that was successfully stored. At the 69 // time the caller receives back control, the value in memory may have changed already. 70 71 template<size_t byte_size> 72 struct Atomic::PlatformAdd 73 : Atomic::AddAndFetch<Atomic::PlatformAdd<byte_size> > 74 { 75 template<typename I, typename D> 76 D add_and_fetch(I add_value, D volatile* dest) const; 77 }; 78 79 template<> 80 template<typename I, typename D> 81 inline D Atomic::PlatformAdd<4>::add_and_fetch(I inc, D volatile* dest) const { 82 STATIC_ASSERT(4 == sizeof(I)); 83 STATIC_ASSERT(4 == sizeof(D)); 84 85 D old, upd; 86 87 if (VM_Version::has_LoadAndALUAtomicV1()) { 88 __asm__ __volatile__ ( 89 " LGFR 0,%[inc] \n\t" // save increment 90 " LA 3,%[mem] \n\t" // force data address into ARG2 91 // " LAA %[upd],%[inc],%[mem] \n\t" // increment and get old value 92 // " LAA 2,0,0(3) \n\t" // actually coded instruction 93 " .byte 0xeb \n\t" // LAA main opcode 94 " .byte 0x20 \n\t" // R1,R3 95 " .byte 0x30 \n\t" // R2,disp1 96 " .byte 0x00 \n\t" // disp2,disp3 97 " .byte 0x00 \n\t" // disp4,disp5 98 " .byte 0xf8 \n\t" // LAA minor opcode 99 " AR 2,0 \n\t" // calc new value in register 100 " LR %[upd],2 \n\t" // move to result register 101 //---< outputs >--- 102 : [upd] "=&d" (upd) // write-only, updated counter value 103 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 104 //---< inputs >--- 105 : [inc] "a" (inc) // read-only. 106 //---< clobbered >--- 107 : "cc", "r0", "r2", "r3", "memory" 108 ); 109 } else { 110 __asm__ __volatile__ ( 111 " LLGF %[old],%[mem] \n\t" // get old value 112 "0: LA %[upd],0(%[inc],%[old]) \n\t" // calc result 113 " CS %[old],%[upd],%[mem] \n\t" // try to xchg res with mem 114 " JNE 0b \n\t" // no success? -> retry 115 //---< outputs >--- 116 : [old] "=&a" (old) // write-only, old counter value 117 , [upd] "=&d" (upd) // write-only, updated counter value 118 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 119 //---< inputs >--- 120 : [inc] "a" (inc) // read-only. 121 //---< clobbered >--- 122 : "cc", "memory" 123 ); 124 } 125 126 return upd; 127 } 128 129 130 template<> 131 template<typename I, typename D> 132 inline D Atomic::PlatformAdd<8>::add_and_fetch(I inc, D volatile* dest) const { 133 STATIC_ASSERT(8 == sizeof(I)); 134 STATIC_ASSERT(8 == sizeof(D)); 135 136 D old, upd; 137 138 if (VM_Version::has_LoadAndALUAtomicV1()) { 139 __asm__ __volatile__ ( 140 " LGR 0,%[inc] \n\t" // save increment 141 " LA 3,%[mem] \n\t" // force data address into ARG2 142 // " LAAG %[upd],%[inc],%[mem] \n\t" // increment and get old value 143 // " LAAG 2,0,0(3) \n\t" // actually coded instruction 144 " .byte 0xeb \n\t" // LAA main opcode 145 " .byte 0x20 \n\t" // R1,R3 146 " .byte 0x30 \n\t" // R2,disp1 147 " .byte 0x00 \n\t" // disp2,disp3 148 " .byte 0x00 \n\t" // disp4,disp5 149 " .byte 0xe8 \n\t" // LAA minor opcode 150 " AGR 2,0 \n\t" // calc new value in register 151 " LGR %[upd],2 \n\t" // move to result register 152 //---< outputs >--- 153 : [upd] "=&d" (upd) // write-only, updated counter value 154 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 155 //---< inputs >--- 156 : [inc] "a" (inc) // read-only. 157 //---< clobbered >--- 158 : "cc", "r0", "r2", "r3", "memory" 159 ); 160 } else { 161 __asm__ __volatile__ ( 162 " LG %[old],%[mem] \n\t" // get old value 163 "0: LA %[upd],0(%[inc],%[old]) \n\t" // calc result 164 " CSG %[old],%[upd],%[mem] \n\t" // try to xchg res with mem 165 " JNE 0b \n\t" // no success? -> retry 166 //---< outputs >--- 167 : [old] "=&a" (old) // write-only, old counter value 168 , [upd] "=&d" (upd) // write-only, updated counter value 169 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 170 //---< inputs >--- 171 : [inc] "a" (inc) // read-only. 172 //---< clobbered >--- 173 : "cc", "memory" 174 ); 175 } 176 177 return upd; 178 } 179 180 181 //------------- 182 // Atomic::xchg 183 //------------- 184 // These methods force the value in memory to be replaced by the new value passed 185 // in as argument. 186 // 187 // The value in memory is replaced by using a compare-and-swap instruction. The 188 // instruction is retried as often as required. This makes sure that the new 189 // value can be seen, at least for a very short period of time, by other CPUs. 190 // 191 // If we would use a normal "load(old value) store(new value)" sequence, 192 // the new value could be lost unnoticed, due to a store(new value) from 193 // another thread. 194 // 195 // The return value is the (unchanged) value from memory as it was when the 196 // replacement succeeded. 197 template<> 198 template<typename T> 199 inline T Atomic::PlatformXchg<4>::operator()(T exchange_value, 200 T volatile* dest) const { 201 STATIC_ASSERT(4 == sizeof(T)); 202 T old; 203 204 __asm__ __volatile__ ( 205 " LLGF %[old],%[mem] \n\t" // get old value 206 "0: CS %[old],%[upd],%[mem] \n\t" // try to xchg upd with mem 207 " JNE 0b \n\t" // no success? -> retry 208 //---< outputs >--- 209 : [old] "=&d" (old) // write-only, prev value irrelevant 210 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 211 //---< inputs >--- 212 : [upd] "d" (exchange_value) // read-only, value to be written to memory 213 //---< clobbered >--- 214 : "cc", "memory" 215 ); 216 217 return old; 218 } 219 220 template<> 221 template<typename T> 222 inline T Atomic::PlatformXchg<8>::operator()(T exchange_value, 223 T volatile* dest) const { 224 STATIC_ASSERT(8 == sizeof(T)); 225 T old; 226 227 __asm__ __volatile__ ( 228 " LG %[old],%[mem] \n\t" // get old value 229 "0: CSG %[old],%[upd],%[mem] \n\t" // try to xchg upd with mem 230 " JNE 0b \n\t" // no success? -> retry 231 //---< outputs >--- 232 : [old] "=&d" (old) // write-only, init from memory 233 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 234 //---< inputs >--- 235 : [upd] "d" (exchange_value) // read-only, value to be written to memory 236 //---< clobbered >--- 237 : "cc", "memory" 238 ); 239 240 return old; 241 } 242 243 //---------------- 244 // Atomic::cmpxchg 245 //---------------- 246 // These methods compare the value in memory with a given compare value. 247 // If both values compare equal, the value in memory is replaced with 248 // the exchange value. 249 // 250 // The value in memory is compared and replaced by using a compare-and-swap 251 // instruction. The instruction is NOT retried (one shot only). 252 // 253 // The return value is the (unchanged) value from memory as it was when the 254 // compare-and-swap instruction completed. A successful exchange operation 255 // is indicated by (return value == compare_value). If unsuccessful, a new 256 // exchange value can be calculated based on the return value which is the 257 // latest contents of the memory location. 258 // 259 // Inspecting the return value is the only way for the caller to determine 260 // if the compare-and-swap instruction was successful: 261 // - If return value and compare value compare equal, the compare-and-swap 262 // instruction was successful and the value in memory was replaced by the 263 // exchange value. 264 // - If return value and compare value compare unequal, the compare-and-swap 265 // instruction was not successful. The value in memory was left unchanged. 266 // 267 // The s390 processors always fence before and after the csg instructions. 268 // Thus we ignore the memory ordering argument. The docu says: "A serialization 269 // function is performed before the operand is fetched and again after the 270 // operation is completed." 271 272 // No direct support for cmpxchg of bytes; emulate using int. 273 template<> 274 struct Atomic::PlatformCmpxchg<1> : Atomic::CmpxchgByteUsingInt {}; 275 276 template<> 277 template<typename T> 278 inline T Atomic::PlatformCmpxchg<4>::operator()(T xchg_val, 279 T volatile* dest, 280 T cmp_val, 281 cmpxchg_memory_order unused) const { 282 STATIC_ASSERT(4 == sizeof(T)); 283 T old; 284 285 __asm__ __volatile__ ( 286 " CS %[old],%[upd],%[mem] \n\t" // Try to xchg upd with mem. 287 // outputs 288 : [old] "=&d" (old) // Write-only, prev value irrelevant. 289 , [mem] "+Q" (*dest) // Read/write, memory to be updated atomically. 290 // inputs 291 : [upd] "d" (xchg_val) 292 , "0" (cmp_val) // Read-only, initial value for [old] (operand #0). 293 // clobbered 294 : "cc", "memory" 295 ); 296 297 return old; 298 } 299 300 template<> 301 template<typename T> 302 inline T Atomic::PlatformCmpxchg<8>::operator()(T xchg_val, 303 T volatile* dest, 304 T cmp_val, 305 cmpxchg_memory_order unused) const { 306 STATIC_ASSERT(8 == sizeof(T)); 307 T old; 308 309 __asm__ __volatile__ ( 310 " CSG %[old],%[upd],%[mem] \n\t" // Try to xchg upd with mem. 311 // outputs 312 : [old] "=&d" (old) // Write-only, prev value irrelevant. 313 , [mem] "+Q" (*dest) // Read/write, memory to be updated atomically. 314 // inputs 315 : [upd] "d" (xchg_val) 316 , "0" (cmp_val) // Read-only, initial value for [old] (operand #0). 317 // clobbered 318 : "cc", "memory" 319 ); 320 321 return old; 322 } 323 324 #endif // OS_CPU_LINUX_S390_VM_ATOMIC_LINUX_S390_INLINE_HPP