1 /*
   2  * Copyright (c) 2020, Red Hat Inc.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 
  26 package jdk.test.lib.containers.cgroup;
  27 
  28 import java.io.IOException;
  29 import java.nio.file.Files;
  30 import java.nio.file.Path;
  31 import java.nio.file.Paths;
  32 import java.util.Arrays;
  33 import java.util.List;
  34 import java.util.concurrent.TimeUnit;
  35 import java.util.stream.Collectors;
  36 
  37 import jdk.internal.platform.Metrics;
  38 import jdk.test.lib.Asserts;
  39 
  40 public class MetricsTesterCgroupV2 implements CgroupMetricsTester {
  41 
  42     private static final long UNLIMITED = -1;
  43     private static final long NOT_SUPPORTED = -2;
  44     private static final UnifiedController UNIFIED = new UnifiedController();
  45     private static final String MAX = "max";
  46     private static final int PER_CPU_SHARES = 1024;
  47 
  48     private final long startSysVal;
  49     private final long startUserVal;
  50     private final long startUsage;
  51 
  52     static class UnifiedController {
  53 
  54         private static final String NAME = "unified";
  55         private final String path;
  56 
  57         UnifiedController() {
  58             path = constructPath();
  59         }
  60 
  61         String getPath() {
  62             return path;
  63         }
  64 
  65         private static String constructPath() {
  66             String mountPath;
  67             String cgroupPath;
  68             try {
  69                 List<String> fifthTokens = Files.lines(Paths.get("/proc/self/mountinfo"))
  70                         .filter( l -> l.contains("- cgroup2"))
  71                         .map(UnifiedController::splitAndMountPath)
  72                         .collect(Collectors.toList());
  73                 if (fifthTokens.size() != 1) {
  74                     throw new AssertionError("Expected only one cgroup2 line");
  75                 }
  76                 mountPath = fifthTokens.get(0);
  77 
  78                 List<String> cgroupPaths = Files.lines(Paths.get("/proc/self/cgroup"))
  79                         .filter( l -> l.startsWith("0:"))
  80                         .map(UnifiedController::splitAndCgroupPath)
  81                         .collect(Collectors.toList());
  82                 if (cgroupPaths.size() != 1) {
  83                     throw new AssertionError("Expected only one unified controller line");
  84                 }
  85                 cgroupPath = cgroupPaths.get(0);
  86                 return Paths.get(mountPath, cgroupPath).toString();
  87             } catch (IOException e) {
  88                 return null;
  89             }
  90         }
  91 
  92         public static String splitAndMountPath(String input) {
  93             String[] tokens = input.split("\\s+");
  94             return tokens[4]; // fifth entry is the mount path
  95         }
  96 
  97         public static String splitAndCgroupPath(String input) {
  98             String[] tokens = input.split(":");
  99             return tokens[2];
 100         }
 101     }
 102 
 103     private long getLongLimitValueFromFile(String file) {
 104         String strVal = getStringVal(file);
 105         if (MAX.equals(strVal)) {
 106             return UNLIMITED;
 107         }
 108         return convertStringToLong(strVal);
 109     }
 110 
 111     public MetricsTesterCgroupV2() {
 112         Metrics metrics = Metrics.systemMetrics();
 113         // Initialize CPU usage metrics before we do any testing.
 114         startSysVal = metrics.getCpuSystemUsage();
 115         startUserVal = metrics.getCpuUserUsage();
 116         startUsage = metrics.getCpuUsage();
 117     }
 118 
 119     private long getLongValueFromFile(String file) {
 120         return convertStringToLong(getStringVal(file));
 121     }
 122 
 123     private long getLongValueEntryFromFile(String file, String metric) {
 124         Path filePath = Paths.get(UNIFIED.getPath(), file);
 125         try {
 126             String strVal = Files.lines(filePath).filter(l -> l.startsWith(metric)).collect(Collectors.joining());
 127             String[] keyValues = strVal.split("\\s+");
 128             String value = keyValues[1];
 129             return convertStringToLong(value);
 130         } catch (IOException e) {
 131             return 0;
 132         }
 133     }
 134 
 135     private String getStringVal(String file) {
 136         Path filePath = Paths.get(UNIFIED.getPath(), file);
 137         try {
 138             return Files.lines(filePath).collect(Collectors.joining());
 139         } catch (IOException e) {
 140             return null;
 141         }
 142     }
 143 
 144     private void fail(String metric, long oldVal, long newVal) {
 145         CgroupMetricsTester.fail(UnifiedController.NAME, metric, oldVal, newVal);
 146     }
 147 
 148     private void fail(String metric, String oldVal, String newVal) {
 149         CgroupMetricsTester.fail(UnifiedController.NAME, metric, oldVal, newVal);
 150     }
 151 
 152     private void warn(String metric, long oldVal, long newVal) {
 153         CgroupMetricsTester.warn(UnifiedController.NAME, metric, oldVal, newVal);
 154     }
 155 
 156     private void verifyNotSupported(long metricVal) {
 157         Asserts.assertEquals(metricVal, NOT_SUPPORTED, "Expected metric to be not supported!");
 158     }
 159 
 160     private void verifyNotSupported(double metricVal) {
 161         if (!CgroupMetricsTester.compareWithErrorMargin(NOT_SUPPORTED, metricVal)) {
 162             throw new RuntimeException("Metric not supported, got: " + metricVal +
 163                                        " expected: " + NOT_SUPPORTED);
 164         }
 165     }
 166 
 167     private void verifyPerCpuNotSupported(long[] perCpuUsage) {
 168         Asserts.assertNull(perCpuUsage, "perCpuUsage expected to be not supported");
 169     }
 170 
 171     private long getCpuShares(String file) {
 172         long rawVal = getLongValueFromFile(file);
 173         if (rawVal == 0 || rawVal == 100) {
 174             return UNLIMITED;
 175         }
 176         int shares = (int)rawVal;
 177         // CPU shares (OCI) value needs to get translated into
 178         // a proper Cgroups v2 value. See:
 179         // https://github.com/containers/crun/blob/master/crun.1.md#cpu-controller
 180         //
 181         // Use the inverse of (x == OCI value, y == cgroupsv2 value):
 182         // ((262142 * y - 1)/9999) + 2 = x
 183         //
 184         int x = 262142 * shares - 1;
 185         double frac = x/9999.0;
 186         x = ((int)frac) + 2;
 187         if ( x <= PER_CPU_SHARES ) {
 188             return PER_CPU_SHARES; // mimic cgroups v1
 189         }
 190         int f = x/PER_CPU_SHARES;
 191         int lower_multiple = f * PER_CPU_SHARES;
 192         int upper_multiple = (f + 1) * PER_CPU_SHARES;
 193         int distance_lower = Math.max(lower_multiple, x) - Math.min(lower_multiple, x);
 194         int distance_upper = Math.max(upper_multiple, x) - Math.min(upper_multiple, x);
 195         x = distance_lower <= distance_upper ? lower_multiple : upper_multiple;
 196         return x;
 197     }
 198 
 199     private long getCpuMaxValueFromFile(String file) {
 200         return getCpuValueFromFile(file, 0 /* $MAX index */);
 201     }
 202 
 203     private long getCpuPeriodValueFromFile(String file) {
 204         return getCpuValueFromFile(file, 1 /* $PERIOD index */);
 205     }
 206 
 207     private long getCpuValueFromFile(String file, int index) {
 208         String maxPeriod = getStringVal(file);
 209         if (maxPeriod == null) {
 210             return UNLIMITED;
 211         }
 212         String[] tokens = maxPeriod.split("\\s+");
 213         String val = tokens[index];
 214         if (MAX.equals(val)) {
 215             return UNLIMITED;
 216         }
 217         return convertStringToLong(val);
 218     }
 219 
 220     private long convertStringToLong(String val) {
 221         return CgroupMetricsTester.convertStringToLong(val, UNLIMITED);
 222     }
 223 
 224     @Override
 225     public void testMemorySubsystem() {
 226         Metrics metrics = Metrics.systemMetrics();
 227 
 228         // User Memory
 229         long oldVal = metrics.getMemoryFailCount();
 230         long newVal = getLongValueEntryFromFile("memory.events", "max");
 231         if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
 232             fail("memory.events[max]", oldVal, newVal);
 233         }
 234 
 235         oldVal = metrics.getMemoryLimit();
 236         newVal = getLongLimitValueFromFile("memory.max");
 237         if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
 238             fail("memory.max", oldVal, newVal);
 239         }
 240 
 241         oldVal = metrics.getMemoryUsage();
 242         newVal = getLongValueFromFile("memory.current");
 243         if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
 244             fail("memory.current", oldVal, newVal);
 245         }
 246         verifyNotSupported(metrics.getMemoryMaxUsage());
 247 
 248         // Kernel memory
 249         verifyNotSupported(metrics.getKernelMemoryFailCount());
 250         verifyNotSupported(metrics.getKernelMemoryLimit());
 251         verifyNotSupported(metrics.getKernelMemoryMaxUsage());
 252         verifyNotSupported(metrics.getKernelMemoryUsage());
 253 
 254         //TCP Memory
 255         verifyNotSupported(metrics.getTcpMemoryFailCount());
 256         verifyNotSupported(metrics.getTcpMemoryLimit());
 257         verifyNotSupported(metrics.getTcpMemoryMaxUsage());
 258 
 259         oldVal = metrics.getTcpMemoryUsage();
 260         newVal = getLongValueEntryFromFile("memory.stat", "sock");
 261         if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
 262             fail("memory.stat[sock]", oldVal, newVal);
 263         }
 264 
 265         //  Memory and Swap
 266         verifyNotSupported(metrics.getMemoryAndSwapFailCount());
 267         verifyNotSupported(metrics.getMemoryAndSwapMaxUsage());
 268 
 269         oldVal = metrics.getMemoryAndSwapLimit();
 270         newVal = getLongLimitValueFromFile("memory.swap.max");
 271         if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
 272             fail("memory.swap.max", oldVal, newVal);
 273         }
 274 
 275         oldVal = metrics.getMemoryAndSwapUsage();
 276         newVal = getLongValueFromFile("memory.swap.current");
 277         if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
 278             fail("memory.swap.current", oldVal, newVal);
 279         }
 280 
 281         oldVal = metrics.getMemorySoftLimit();
 282         newVal = getLongLimitValueFromFile("memory.high");
 283         if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
 284             fail("memory.high", oldVal, newVal);
 285         }
 286 
 287         Asserts.assertNull(metrics.isMemoryOOMKillEnabled(), "Not supported");
 288     }
 289 
 290     @Override
 291     public void testCpuAccounting() {
 292         Metrics metrics = Metrics.systemMetrics();
 293         long oldVal = metrics.getCpuUsage();
 294         long newVal = TimeUnit.MICROSECONDS.toNanos(getLongValueEntryFromFile("cpu.stat", "usage_usec"));
 295 
 296         if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
 297             warn("cpu.stat[usage_usec]", oldVal, newVal);
 298         }
 299 
 300         verifyPerCpuNotSupported(metrics.getPerCpuUsage());
 301 
 302         oldVal = metrics.getCpuUserUsage();
 303         newVal = TimeUnit.MICROSECONDS.toNanos(getLongValueEntryFromFile("cpu.stat", "user_usec"));
 304         if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
 305             warn("cpu.stat[user_usec]", oldVal, newVal);
 306         }
 307 
 308         oldVal = metrics.getCpuSystemUsage();
 309         newVal = TimeUnit.MICROSECONDS.toNanos(getLongValueEntryFromFile("cpu.stat", "system_usec"));
 310         if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
 311             warn("cpu.stat[system_usec]", oldVal, newVal);
 312         }
 313     }
 314 
 315     @Override
 316     public void testCpuSchedulingMetrics() {
 317         Metrics metrics = Metrics.systemMetrics();
 318         long oldVal = metrics.getCpuPeriod();
 319         long newVal = getCpuPeriodValueFromFile("cpu.max");
 320         if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
 321             fail("cpu.max[$PERIOD]", oldVal, newVal);
 322         }
 323 
 324         oldVal = metrics.getCpuQuota();
 325         newVal = getCpuMaxValueFromFile("cpu.max");
 326         if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
 327             fail("cpu.max[$MAX]", oldVal, newVal);
 328         }
 329 
 330         oldVal = metrics.getCpuShares();
 331         newVal = getCpuShares("cpu.weight");
 332         if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
 333             fail("cpu.weight", oldVal, newVal);
 334         }
 335 
 336         oldVal = metrics.getCpuNumPeriods();
 337         newVal = getLongValueEntryFromFile("cpu.stat", "nr_periods");
 338         if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
 339             fail("cpu.stat[nr_periods]", oldVal, newVal);
 340         }
 341 
 342         oldVal = metrics.getCpuNumThrottled();
 343         newVal = getLongValueEntryFromFile("cpu.stat", "nr_throttled");
 344         if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
 345             fail("cpu.stat[nr_throttled]", oldVal, newVal);
 346         }
 347 
 348         oldVal = metrics.getCpuThrottledTime();
 349         newVal = TimeUnit.MICROSECONDS.toNanos(getLongValueEntryFromFile("cpu.stat", "throttled_usec"));
 350         if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
 351             fail("cpu.stat[throttled_usec]", oldVal, newVal);
 352         }
 353     }
 354 
 355     @Override
 356     public void testCpuSets() {
 357         Metrics metrics = Metrics.systemMetrics();
 358         int[] cpus = mapNullToEmpty(metrics.getCpuSetCpus());
 359         Integer[] oldVal = Arrays.stream(cpus).boxed().toArray(Integer[]::new);
 360         Arrays.sort(oldVal);
 361 
 362         String cpusstr = getStringVal("cpuset.cpus");
 363         // Parse range string in the format 1,2-6,7
 364         Integer[] newVal = CgroupMetricsTester.convertCpuSetsToArray(cpusstr);
 365         Arrays.sort(newVal);
 366         if (Arrays.compare(oldVal, newVal) != 0) {
 367             fail("cpuset.cpus", Arrays.toString(oldVal),
 368                                 Arrays.toString(newVal));
 369         }
 370 
 371         cpus = mapNullToEmpty(metrics.getEffectiveCpuSetCpus());
 372         oldVal = Arrays.stream(cpus).boxed().toArray(Integer[]::new);
 373         Arrays.sort(oldVal);
 374         cpusstr = getStringVal("cpuset.cpus.effective");
 375         newVal = CgroupMetricsTester.convertCpuSetsToArray(cpusstr);
 376         Arrays.sort(newVal);
 377         if (Arrays.compare(oldVal, newVal) != 0) {
 378             fail("cpuset.cpus.effective", Arrays.toString(oldVal),
 379                                           Arrays.toString(newVal));
 380         }
 381 
 382         cpus = mapNullToEmpty(metrics.getCpuSetMems());
 383         oldVal = Arrays.stream(cpus).boxed().toArray(Integer[]::new);
 384         Arrays.sort(oldVal);
 385         cpusstr = getStringVal("cpuset.mems");
 386         newVal = CgroupMetricsTester.convertCpuSetsToArray(cpusstr);
 387         Arrays.sort(newVal);
 388         if (Arrays.compare(oldVal, newVal) != 0) {
 389             fail("cpuset.mems", Arrays.toString(oldVal),
 390                                 Arrays.toString(newVal));
 391         }
 392 
 393         cpus = mapNullToEmpty(metrics.getEffectiveCpuSetMems());
 394         oldVal = Arrays.stream(cpus).boxed().toArray(Integer[]::new);
 395         Arrays.sort(oldVal);
 396         cpusstr = getStringVal("cpuset.mems.effective");
 397         newVal = CgroupMetricsTester.convertCpuSetsToArray(cpusstr);
 398         Arrays.sort(newVal);
 399         if (Arrays.compare(oldVal, newVal) != 0) {
 400             fail("cpuset.mems.effective", Arrays.toString(oldVal),
 401                                           Arrays.toString(newVal));
 402         }
 403 
 404         verifyNotSupported(metrics.getCpuSetMemoryPressure());
 405         Asserts.assertNull(metrics.isCpuSetMemoryPressureEnabled(), "Should be not supported");
 406     }
 407 
 408     private int[] mapNullToEmpty(int[] cpus) {
 409         if (cpus == null) {
 410             // Not available. For sake of testing continue with an
 411             // empty array.
 412             cpus = new int[0];
 413         }
 414         return cpus;
 415     }
 416 
 417     @Override
 418     public void testCpuConsumption() {
 419         Metrics metrics = Metrics.systemMetrics();
 420         // make system call
 421         long newSysVal = metrics.getCpuSystemUsage();
 422         long newUserVal = metrics.getCpuUserUsage();
 423         long newUsage = metrics.getCpuUsage();
 424 
 425         verifyPerCpuNotSupported(metrics.getPerCpuUsage());
 426 
 427         // system/user CPU usage counters may be slowly increasing.
 428         // allow for equal values for a pass
 429         if (newSysVal < startSysVal) {
 430             fail("getCpuSystemUsage", newSysVal, startSysVal);
 431         }
 432 
 433         // system/user CPU usage counters may be slowly increasing.
 434         // allow for equal values for a pass
 435         if (newUserVal < startUserVal) {
 436             fail("getCpuUserUsage", newUserVal, startUserVal);
 437         }
 438 
 439         if (newUsage <= startUsage) {
 440             fail("getCpuUsage", newUsage, startUsage);
 441         }
 442     }
 443 
 444     @Override
 445     public void testMemoryUsage() {
 446         Metrics metrics = Metrics.systemMetrics();
 447         long memoryMaxUsage = metrics.getMemoryMaxUsage();
 448         long memoryUsage = metrics.getMemoryUsage();
 449         long newMemoryMaxUsage = 0, newMemoryUsage = 0;
 450 
 451         // allocate memory in a loop and check more than once for new values
 452         // otherwise we might occasionally see the effect of decreasing new memory
 453         // values. For example because the system could free up memory
 454         byte[][] bytes = new byte[32][];
 455         for (int i = 0; i < 32; i++) {
 456             bytes[i] = new byte[8*1024*1024];
 457             newMemoryUsage = metrics.getMemoryUsage();
 458             if (newMemoryUsage > memoryUsage) {
 459                 break;
 460             }
 461         }
 462         newMemoryMaxUsage = metrics.getMemoryMaxUsage();
 463 
 464         if (newMemoryMaxUsage < memoryMaxUsage) {
 465             fail("getMemoryMaxUsage", memoryMaxUsage,
 466                                       newMemoryMaxUsage);
 467         }
 468 
 469         if (newMemoryUsage < memoryUsage) {
 470             fail("getMemoryUsage", memoryUsage, newMemoryUsage);
 471         }
 472     }
 473 
 474     @Override
 475     public void testMisc() {
 476         testIOStat();
 477     }
 478 
 479     private void testIOStat() {
 480         Metrics metrics = Metrics.systemMetrics();
 481         long oldVal = metrics.getBlkIOServiceCount();
 482         long newVal = getIoStatAccumulate(new String[] { "rios", "wios" });
 483         if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
 484             fail("io.stat->rios/wios: ", oldVal, newVal);
 485         }
 486 
 487         oldVal = metrics.getBlkIOServiced();
 488         newVal = getIoStatAccumulate(new String[] { "rbytes", "wbytes" });
 489         if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
 490             fail("io.stat->rbytes/wbytes: ", oldVal, newVal);
 491         }
 492     }
 493 
 494     private long getIoStatAccumulate(String[] matchNames) {
 495         try {
 496             return Files.lines(Paths.get(UNIFIED.getPath(), "io.stat"))
 497                     .map(line -> {
 498                         long accumulator = 0;
 499                         String[] tokens = line.split("\\s+");
 500                         for (String t: tokens) {
 501                             String[] keyVal = t.split("=");
 502                             if (keyVal.length != 2) {
 503                                 continue;
 504                             }
 505                             for (String match: matchNames) {
 506                                 if (match.equals(keyVal[0])) {
 507                                     accumulator += Long.parseLong(keyVal[1]);
 508                                 }
 509                             }
 510                         }
 511                         return accumulator;
 512                     }).collect(Collectors.summingLong(e -> e));
 513         } catch (IOException e) {
 514             return Metrics.LONG_RETVAL_NOT_SUPPORTED;
 515         }
 516     }
 517 }