1 /* 2 * Copyright (c) 2020, Red Hat Inc. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. Oracle designates this 8 * particular file as subject to the "Classpath" exception as provided 9 * by Oracle in the LICENSE file that accompanied this code. 10 * 11 * This code is distributed in the hope that it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 * version 2 for more details (a copy is included in the LICENSE file that 15 * accompanied this code). 16 * 17 * You should have received a copy of the GNU General Public License version 18 * 2 along with this work; if not, write to the Free Software Foundation, 19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 20 * 21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 22 * or visit www.oracle.com if you need additional information or have any 23 * questions. 24 */ 25 26 package jdk.test.lib.containers.cgroup; 27 28 import java.io.IOException; 29 import java.nio.file.Files; 30 import java.nio.file.Path; 31 import java.nio.file.Paths; 32 import java.util.Arrays; 33 import java.util.List; 34 import java.util.concurrent.TimeUnit; 35 import java.util.stream.Collectors; 36 37 import jdk.internal.platform.Metrics; 38 import jdk.test.lib.Asserts; 39 40 public class MetricsTesterCgroupV2 implements CgroupMetricsTester { 41 42 private static final long UNLIMITED = -1; 43 private static final long NOT_SUPPORTED = -2; 44 private static final UnifiedController UNIFIED = new UnifiedController(); 45 private static final String MAX = "max"; 46 private static final int PER_CPU_SHARES = 1024; 47 48 private final long startSysVal; 49 private final long startUserVal; 50 private final long startUsage; 51 52 static class UnifiedController { 53 54 private static final String NAME = "unified"; 55 private final String path; 56 57 UnifiedController() { 58 path = constructPath(); 59 } 60 61 String getPath() { 62 return path; 63 } 64 65 private static String constructPath() { 66 String mountPath; 67 String cgroupPath; 68 try { 69 List<String> fifthTokens = Files.lines(Paths.get("/proc/self/mountinfo")) 70 .filter( l -> l.contains("- cgroup2")) 71 .map(UnifiedController::splitAndMountPath) 72 .collect(Collectors.toList()); 73 if (fifthTokens.size() != 1) { 74 throw new AssertionError("Expected only one cgroup2 line"); 75 } 76 mountPath = fifthTokens.get(0); 77 78 List<String> cgroupPaths = Files.lines(Paths.get("/proc/self/cgroup")) 79 .filter( l -> l.startsWith("0:")) 80 .map(UnifiedController::splitAndCgroupPath) 81 .collect(Collectors.toList()); 82 if (cgroupPaths.size() != 1) { 83 throw new AssertionError("Expected only one unified controller line"); 84 } 85 cgroupPath = cgroupPaths.get(0); 86 return Paths.get(mountPath, cgroupPath).toString(); 87 } catch (IOException e) { 88 return null; 89 } 90 } 91 92 public static String splitAndMountPath(String input) { 93 String[] tokens = input.split("\\s+"); 94 return tokens[4]; // fifth entry is the mount path 95 } 96 97 public static String splitAndCgroupPath(String input) { 98 String[] tokens = input.split(":"); 99 return tokens[2]; 100 } 101 } 102 103 private long getLongLimitValueFromFile(String file) { 104 String strVal = getStringVal(file); 105 if (MAX.equals(strVal)) { 106 return UNLIMITED; 107 } 108 return convertStringToLong(strVal); 109 } 110 111 public MetricsTesterCgroupV2() { 112 Metrics metrics = Metrics.systemMetrics(); 113 // Initialize CPU usage metrics before we do any testing. 114 startSysVal = metrics.getCpuSystemUsage(); 115 startUserVal = metrics.getCpuUserUsage(); 116 startUsage = metrics.getCpuUsage(); 117 } 118 119 private long getLongValueFromFile(String file) { 120 return convertStringToLong(getStringVal(file)); 121 } 122 123 private long getLongValueEntryFromFile(String file, String metric) { 124 Path filePath = Paths.get(UNIFIED.getPath(), file); 125 try { 126 String strVal = Files.lines(filePath).filter(l -> l.startsWith(metric)).collect(Collectors.joining()); 127 String[] keyValues = strVal.split("\\s+"); 128 String value = keyValues[1]; 129 return convertStringToLong(value); 130 } catch (IOException e) { 131 return 0; 132 } 133 } 134 135 private String getStringVal(String file) { 136 Path filePath = Paths.get(UNIFIED.getPath(), file); 137 try { 138 return Files.lines(filePath).collect(Collectors.joining()); 139 } catch (IOException e) { 140 return null; 141 } 142 } 143 144 private void fail(String metric, long oldVal, long newVal) { 145 CgroupMetricsTester.fail(UnifiedController.NAME, metric, oldVal, newVal); 146 } 147 148 private void fail(String metric, String oldVal, String newVal) { 149 CgroupMetricsTester.fail(UnifiedController.NAME, metric, oldVal, newVal); 150 } 151 152 private void warn(String metric, long oldVal, long newVal) { 153 CgroupMetricsTester.warn(UnifiedController.NAME, metric, oldVal, newVal); 154 } 155 156 private void verifyNotSupported(long metricVal) { 157 Asserts.assertEquals(metricVal, NOT_SUPPORTED, "Expected metric to be not supported!"); 158 } 159 160 private void verifyNotSupported(double metricVal) { 161 if (!CgroupMetricsTester.compareWithErrorMargin(NOT_SUPPORTED, metricVal)) { 162 throw new RuntimeException("Metric not supported, got: " + metricVal + 163 " expected: " + NOT_SUPPORTED); 164 } 165 } 166 167 private void verifyPerCpuNotSupported(long[] perCpuUsage) { 168 Asserts.assertNull(perCpuUsage, "perCpuUsage expected to be not supported"); 169 } 170 171 private long getCpuShares(String file) { 172 long rawVal = getLongValueFromFile(file); 173 if (rawVal == 0 || rawVal == 100) { 174 return UNLIMITED; 175 } 176 int shares = (int)rawVal; 177 // CPU shares (OCI) value needs to get translated into 178 // a proper Cgroups v2 value. See: 179 // https://github.com/containers/crun/blob/master/crun.1.md#cpu-controller 180 // 181 // Use the inverse of (x == OCI value, y == cgroupsv2 value): 182 // ((262142 * y - 1)/9999) + 2 = x 183 // 184 int x = 262142 * shares - 1; 185 double frac = x/9999.0; 186 x = ((int)frac) + 2; 187 if ( x <= PER_CPU_SHARES ) { 188 return PER_CPU_SHARES; // mimic cgroups v1 189 } 190 int f = x/PER_CPU_SHARES; 191 int lower_multiple = f * PER_CPU_SHARES; 192 int upper_multiple = (f + 1) * PER_CPU_SHARES; 193 int distance_lower = Math.max(lower_multiple, x) - Math.min(lower_multiple, x); 194 int distance_upper = Math.max(upper_multiple, x) - Math.min(upper_multiple, x); 195 x = distance_lower <= distance_upper ? lower_multiple : upper_multiple; 196 return x; 197 } 198 199 private long getCpuMaxValueFromFile(String file) { 200 return getCpuValueFromFile(file, 0 /* $MAX index */); 201 } 202 203 private long getCpuPeriodValueFromFile(String file) { 204 return getCpuValueFromFile(file, 1 /* $PERIOD index */); 205 } 206 207 private long getCpuValueFromFile(String file, int index) { 208 String maxPeriod = getStringVal(file); 209 if (maxPeriod == null) { 210 return UNLIMITED; 211 } 212 String[] tokens = maxPeriod.split("\\s+"); 213 String val = tokens[index]; 214 if (MAX.equals(val)) { 215 return UNLIMITED; 216 } 217 return convertStringToLong(val); 218 } 219 220 private long convertStringToLong(String val) { 221 return CgroupMetricsTester.convertStringToLong(val, UNLIMITED); 222 } 223 224 @Override 225 public void testMemorySubsystem() { 226 Metrics metrics = Metrics.systemMetrics(); 227 228 // User Memory 229 long oldVal = metrics.getMemoryFailCount(); 230 long newVal = getLongValueEntryFromFile("memory.events", "max"); 231 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) { 232 fail("memory.events[max]", oldVal, newVal); 233 } 234 235 oldVal = metrics.getMemoryLimit(); 236 newVal = getLongLimitValueFromFile("memory.max"); 237 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) { 238 fail("memory.max", oldVal, newVal); 239 } 240 241 oldVal = metrics.getMemoryUsage(); 242 newVal = getLongValueFromFile("memory.current"); 243 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) { 244 fail("memory.current", oldVal, newVal); 245 } 246 verifyNotSupported(metrics.getMemoryMaxUsage()); 247 248 // Kernel memory 249 verifyNotSupported(metrics.getKernelMemoryFailCount()); 250 verifyNotSupported(metrics.getKernelMemoryLimit()); 251 verifyNotSupported(metrics.getKernelMemoryMaxUsage()); 252 verifyNotSupported(metrics.getKernelMemoryUsage()); 253 254 //TCP Memory 255 verifyNotSupported(metrics.getTcpMemoryFailCount()); 256 verifyNotSupported(metrics.getTcpMemoryLimit()); 257 verifyNotSupported(metrics.getTcpMemoryMaxUsage()); 258 259 oldVal = metrics.getTcpMemoryUsage(); 260 newVal = getLongValueEntryFromFile("memory.stat", "sock"); 261 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) { 262 fail("memory.stat[sock]", oldVal, newVal); 263 } 264 265 // Memory and Swap 266 verifyNotSupported(metrics.getMemoryAndSwapFailCount()); 267 verifyNotSupported(metrics.getMemoryAndSwapMaxUsage()); 268 269 oldVal = metrics.getMemoryAndSwapLimit(); 270 newVal = getLongLimitValueFromFile("memory.swap.max"); 271 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) { 272 fail("memory.swap.max", oldVal, newVal); 273 } 274 275 oldVal = metrics.getMemoryAndSwapUsage(); 276 newVal = getLongValueFromFile("memory.swap.current"); 277 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) { 278 fail("memory.swap.current", oldVal, newVal); 279 } 280 281 oldVal = metrics.getMemorySoftLimit(); 282 newVal = getLongLimitValueFromFile("memory.high"); 283 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) { 284 fail("memory.high", oldVal, newVal); 285 } 286 287 Asserts.assertNull(metrics.isMemoryOOMKillEnabled(), "Not supported"); 288 } 289 290 @Override 291 public void testCpuAccounting() { 292 Metrics metrics = Metrics.systemMetrics(); 293 long oldVal = metrics.getCpuUsage(); 294 long newVal = TimeUnit.MICROSECONDS.toNanos(getLongValueEntryFromFile("cpu.stat", "usage_usec")); 295 296 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) { 297 warn("cpu.stat[usage_usec]", oldVal, newVal); 298 } 299 300 verifyPerCpuNotSupported(metrics.getPerCpuUsage()); 301 302 oldVal = metrics.getCpuUserUsage(); 303 newVal = TimeUnit.MICROSECONDS.toNanos(getLongValueEntryFromFile("cpu.stat", "user_usec")); 304 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) { 305 warn("cpu.stat[user_usec]", oldVal, newVal); 306 } 307 308 oldVal = metrics.getCpuSystemUsage(); 309 newVal = TimeUnit.MICROSECONDS.toNanos(getLongValueEntryFromFile("cpu.stat", "system_usec")); 310 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) { 311 warn("cpu.stat[system_usec]", oldVal, newVal); 312 } 313 } 314 315 @Override 316 public void testCpuSchedulingMetrics() { 317 Metrics metrics = Metrics.systemMetrics(); 318 long oldVal = metrics.getCpuPeriod(); 319 long newVal = getCpuPeriodValueFromFile("cpu.max"); 320 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) { 321 fail("cpu.max[$PERIOD]", oldVal, newVal); 322 } 323 324 oldVal = metrics.getCpuQuota(); 325 newVal = getCpuMaxValueFromFile("cpu.max"); 326 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) { 327 fail("cpu.max[$MAX]", oldVal, newVal); 328 } 329 330 oldVal = metrics.getCpuShares(); 331 newVal = getCpuShares("cpu.weight"); 332 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) { 333 fail("cpu.weight", oldVal, newVal); 334 } 335 336 oldVal = metrics.getCpuNumPeriods(); 337 newVal = getLongValueEntryFromFile("cpu.stat", "nr_periods"); 338 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) { 339 fail("cpu.stat[nr_periods]", oldVal, newVal); 340 } 341 342 oldVal = metrics.getCpuNumThrottled(); 343 newVal = getLongValueEntryFromFile("cpu.stat", "nr_throttled"); 344 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) { 345 fail("cpu.stat[nr_throttled]", oldVal, newVal); 346 } 347 348 oldVal = metrics.getCpuThrottledTime(); 349 newVal = TimeUnit.MICROSECONDS.toNanos(getLongValueEntryFromFile("cpu.stat", "throttled_usec")); 350 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) { 351 fail("cpu.stat[throttled_usec]", oldVal, newVal); 352 } 353 } 354 355 @Override 356 public void testCpuSets() { 357 Metrics metrics = Metrics.systemMetrics(); 358 int[] cpus = mapNullToEmpty(metrics.getCpuSetCpus()); 359 Integer[] oldVal = Arrays.stream(cpus).boxed().toArray(Integer[]::new); 360 Arrays.sort(oldVal); 361 362 String cpusstr = getStringVal("cpuset.cpus"); 363 // Parse range string in the format 1,2-6,7 364 Integer[] newVal = CgroupMetricsTester.convertCpuSetsToArray(cpusstr); 365 Arrays.sort(newVal); 366 if (Arrays.compare(oldVal, newVal) != 0) { 367 fail("cpuset.cpus", Arrays.toString(oldVal), 368 Arrays.toString(newVal)); 369 } 370 371 cpus = mapNullToEmpty(metrics.getEffectiveCpuSetCpus()); 372 oldVal = Arrays.stream(cpus).boxed().toArray(Integer[]::new); 373 Arrays.sort(oldVal); 374 cpusstr = getStringVal("cpuset.cpus.effective"); 375 newVal = CgroupMetricsTester.convertCpuSetsToArray(cpusstr); 376 Arrays.sort(newVal); 377 if (Arrays.compare(oldVal, newVal) != 0) { 378 fail("cpuset.cpus.effective", Arrays.toString(oldVal), 379 Arrays.toString(newVal)); 380 } 381 382 cpus = mapNullToEmpty(metrics.getCpuSetMems()); 383 oldVal = Arrays.stream(cpus).boxed().toArray(Integer[]::new); 384 Arrays.sort(oldVal); 385 cpusstr = getStringVal("cpuset.mems"); 386 newVal = CgroupMetricsTester.convertCpuSetsToArray(cpusstr); 387 Arrays.sort(newVal); 388 if (Arrays.compare(oldVal, newVal) != 0) { 389 fail("cpuset.mems", Arrays.toString(oldVal), 390 Arrays.toString(newVal)); 391 } 392 393 cpus = mapNullToEmpty(metrics.getEffectiveCpuSetMems()); 394 oldVal = Arrays.stream(cpus).boxed().toArray(Integer[]::new); 395 Arrays.sort(oldVal); 396 cpusstr = getStringVal("cpuset.mems.effective"); 397 newVal = CgroupMetricsTester.convertCpuSetsToArray(cpusstr); 398 Arrays.sort(newVal); 399 if (Arrays.compare(oldVal, newVal) != 0) { 400 fail("cpuset.mems.effective", Arrays.toString(oldVal), 401 Arrays.toString(newVal)); 402 } 403 404 verifyNotSupported(metrics.getCpuSetMemoryPressure()); 405 Asserts.assertNull(metrics.isCpuSetMemoryPressureEnabled(), "Should be not supported"); 406 } 407 408 private int[] mapNullToEmpty(int[] cpus) { 409 if (cpus == null) { 410 // Not available. For sake of testing continue with an 411 // empty array. 412 cpus = new int[0]; 413 } 414 return cpus; 415 } 416 417 @Override 418 public void testCpuConsumption() { 419 Metrics metrics = Metrics.systemMetrics(); 420 // make system call 421 long newSysVal = metrics.getCpuSystemUsage(); 422 long newUserVal = metrics.getCpuUserUsage(); 423 long newUsage = metrics.getCpuUsage(); 424 425 verifyPerCpuNotSupported(metrics.getPerCpuUsage()); 426 427 // system/user CPU usage counters may be slowly increasing. 428 // allow for equal values for a pass 429 if (newSysVal < startSysVal) { 430 fail("getCpuSystemUsage", newSysVal, startSysVal); 431 } 432 433 // system/user CPU usage counters may be slowly increasing. 434 // allow for equal values for a pass 435 if (newUserVal < startUserVal) { 436 fail("getCpuUserUsage", newUserVal, startUserVal); 437 } 438 439 if (newUsage <= startUsage) { 440 fail("getCpuUsage", newUsage, startUsage); 441 } 442 } 443 444 @Override 445 public void testMemoryUsage() { 446 Metrics metrics = Metrics.systemMetrics(); 447 long memoryMaxUsage = metrics.getMemoryMaxUsage(); 448 long memoryUsage = metrics.getMemoryUsage(); 449 long newMemoryMaxUsage = 0, newMemoryUsage = 0; 450 451 // allocate memory in a loop and check more than once for new values 452 // otherwise we might occasionally see the effect of decreasing new memory 453 // values. For example because the system could free up memory 454 byte[][] bytes = new byte[32][]; 455 for (int i = 0; i < 32; i++) { 456 bytes[i] = new byte[8*1024*1024]; 457 newMemoryUsage = metrics.getMemoryUsage(); 458 if (newMemoryUsage > memoryUsage) { 459 break; 460 } 461 } 462 newMemoryMaxUsage = metrics.getMemoryMaxUsage(); 463 464 if (newMemoryMaxUsage < memoryMaxUsage) { 465 fail("getMemoryMaxUsage", memoryMaxUsage, 466 newMemoryMaxUsage); 467 } 468 469 if (newMemoryUsage < memoryUsage) { 470 fail("getMemoryUsage", memoryUsage, newMemoryUsage); 471 } 472 } 473 474 @Override 475 public void testMisc() { 476 testIOStat(); 477 } 478 479 private void testIOStat() { 480 Metrics metrics = Metrics.systemMetrics(); 481 long oldVal = metrics.getBlkIOServiceCount(); 482 long newVal = getIoStatAccumulate(new String[] { "rios", "wios" }); 483 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) { 484 fail("io.stat->rios/wios: ", oldVal, newVal); 485 } 486 487 oldVal = metrics.getBlkIOServiced(); 488 newVal = getIoStatAccumulate(new String[] { "rbytes", "wbytes" }); 489 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) { 490 fail("io.stat->rbytes/wbytes: ", oldVal, newVal); 491 } 492 } 493 494 private long getIoStatAccumulate(String[] matchNames) { 495 try { 496 return Files.lines(Paths.get(UNIFIED.getPath(), "io.stat")) 497 .map(line -> { 498 long accumulator = 0; 499 String[] tokens = line.split("\\s+"); 500 for (String t: tokens) { 501 String[] keyVal = t.split("="); 502 if (keyVal.length != 2) { 503 continue; 504 } 505 for (String match: matchNames) { 506 if (match.equals(keyVal[0])) { 507 accumulator += Long.parseLong(keyVal[1]); 508 } 509 } 510 } 511 return accumulator; 512 }).collect(Collectors.summingLong(e -> e)); 513 } catch (IOException e) { 514 return Metrics.LONG_RETVAL_NOT_SUPPORTED; 515 } 516 } 517 }