/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.util;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.Random;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.fs.Path;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
/**
* A JUnit test to test {@link SysInfoLinux}
* Create the fake /proc/ information and verify the parsing and calculation
*/
public class TestSysInfoLinux {
/**
* LinuxResourceCalculatorPlugin with a fake timer
*/
static class FakeLinuxResourceCalculatorPlugin extends SysInfoLinux {
static final int SECTORSIZE = 4096;
long currentTime = 0;
public FakeLinuxResourceCalculatorPlugin(String procfsMemFile,
String procfsCpuFile,
String procfsStatFile,
String procfsNetFile,
String procfsDisksFile,
long jiffyLengthInMillis) {
super(procfsMemFile, procfsCpuFile, procfsStatFile, procfsNetFile,
procfsDisksFile, jiffyLengthInMillis);
}
@Override
long getCurrentTime() {
return currentTime;
}
public void advanceTime(long adv) {
currentTime += adv * this.getJiffyLengthInMillis();
}
@Override
int readDiskBlockInformation(String diskName, int defSector) {
return SECTORSIZE;
}
}
private static final FakeLinuxResourceCalculatorPlugin plugin;
private static String TEST_ROOT_DIR = new Path(System.getProperty(
"test.build.data", "/tmp")).toString().replace(' ', '+');
private static final String FAKE_MEMFILE;
private static final String FAKE_CPUFILE;
private static final String FAKE_STATFILE;
private static final String FAKE_NETFILE;
private static final String FAKE_DISKSFILE;
private static final long FAKE_JIFFY_LENGTH = 10L;
static {
int randomNum = (new Random()).nextInt(1000000000);
FAKE_MEMFILE = TEST_ROOT_DIR + File.separator + "MEMINFO_" + randomNum;
FAKE_CPUFILE = TEST_ROOT_DIR + File.separator + "CPUINFO_" + randomNum;
FAKE_STATFILE = TEST_ROOT_DIR + File.separator + "STATINFO_" + randomNum;
FAKE_NETFILE = TEST_ROOT_DIR + File.separator + "NETINFO_" + randomNum;
FAKE_DISKSFILE = TEST_ROOT_DIR + File.separator + "DISKSINFO_" + randomNum;
plugin = new FakeLinuxResourceCalculatorPlugin(FAKE_MEMFILE, FAKE_CPUFILE,
FAKE_STATFILE,
FAKE_NETFILE,
FAKE_DISKSFILE,
FAKE_JIFFY_LENGTH);
}
static final String MEMINFO_FORMAT =
"MemTotal: %d kB\n" +
"MemFree: %d kB\n" +
"Buffers: 138244 kB\n" +
"Cached: 947780 kB\n" +
"SwapCached: 142880 kB\n" +
"Active: 3229888 kB\n" +
"Inactive: %d kB\n" +
"SwapTotal: %d kB\n" +
"SwapFree: %d kB\n" +
"Dirty: 122012 kB\n" +
"Writeback: 0 kB\n" +
"AnonPages: 2710792 kB\n" +
"Mapped: 24740 kB\n" +
"Slab: 132528 kB\n" +
"SReclaimable: 105096 kB\n" +
"SUnreclaim: 27432 kB\n" +
"PageTables: 11448 kB\n" +
"NFS_Unstable: 0 kB\n" +
"Bounce: 0 kB\n" +
"CommitLimit: 4125904 kB\n" +
"Committed_AS: 4143556 kB\n" +
"VmallocTotal: 34359738367 kB\n" +
"VmallocUsed: 1632 kB\n" +
"VmallocChunk: 34359736375 kB\n" +
"HugePages_Total: 0\n" +
"HugePages_Free: 0\n" +
"HugePages_Rsvd: 0\n" +
"Hugepagesize: 2048 kB";
static final String CPUINFO_FORMAT =
"processor : %s\n" +
"vendor_id : AuthenticAMD\n" +
"cpu family : 15\n" +
"model : 33\n" +
"model name : Dual Core AMD Opteron(tm) Processor 280\n" +
"stepping : 2\n" +
"cpu MHz : %f\n" +
"cache size : 1024 KB\n" +
"physical id : %s\n" +
"siblings : 2\n" +
"core id : %s\n" +
"cpu cores : 2\n" +
"fpu : yes\n" +
"fpu_exception : yes\n" +
"cpuid level : 1\n" +
"wp : yes\n" +
"flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov " +
"pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt lm " +
"3dnowext 3dnow pni lahf_lm cmp_legacy\n" +
"bogomips : 4792.41\n" +
"TLB size : 1024 4K pages\n" +
"clflush size : 64\n" +
"cache_alignment : 64\n" +
"address sizes : 40 bits physical, 48 bits virtual\n" +
"power management: ts fid vid ttp";
static final String STAT_FILE_FORMAT =
"cpu %d %d %d 1646495089 831319 48713 164346 0\n" +
"cpu0 15096055 30805 3823005 411456015 206027 13 14269 0\n" +
"cpu1 14760561 89890 6432036 408707910 456857 48074 130857 0\n" +
"cpu2 12761169 20842 3758639 413976772 98028 411 10288 0\n" +
"cpu3 12355207 47322 5789691 412354390 70406 213 8931 0\n" +
"intr 114648668 20010764 2 0 945665 2 0 0 0 0 0 0 0 4 0 0 0 0 0 0\n" +
"ctxt 242017731764\n" +
"btime 1257808753\n" +
"processes 26414943\n" +
"procs_running 1\n" +
"procs_blocked 0\n";
static final String NETINFO_FORMAT =
"Inter-| Receive | Transmit\n"+
"face |bytes packets errs drop fifo frame compressed multicast|bytes packets"+
"errs drop fifo colls carrier compressed\n"+
" lo: 42236310 563003 0 0 0 0 0 0 42236310 563003 " +
"0 0 0 0 0 0\n"+
" eth0: %d 3452527 0 0 0 0 0 299787 %d 1866280 0 0 " +
"0 0 0 0\n"+
" eth1: %d 3152521 0 0 0 0 0 219781 %d 1866290 0 0 " +
"0 0 0 0\n";
static final String DISKSINFO_FORMAT =
"1 0 ram0 0 0 0 0 0 0 0 0 0 0 0\n"+
"1 1 ram1 0 0 0 0 0 0 0 0 0 0 0\n"+
"1 2 ram2 0 0 0 0 0 0 0 0 0 0 0\n"+
"1 3 ram3 0 0 0 0 0 0 0 0 0 0 0\n"+
"1 4 ram4 0 0 0 0 0 0 0 0 0 0 0\n"+
"1 5 ram5 0 0 0 0 0 0 0 0 0 0 0\n"+
"1 6 ram6 0 0 0 0 0 0 0 0 0 0 0\n"+
"7 0 loop0 0 0 0 0 0 0 0 0 0 0 0\n"+
"7 1 loop1 0 0 0 0 0 0 0 0 0 0 0\n"+
"8 0 sda 82575678 2486518 %d 59876600 3225402 19761924 %d " +
"6407705 4 48803346 66227952\n"+
"8 1 sda1 732 289 21354 787 7 3 32 4 0 769 791"+
"8 2 sda2 744272 2206315 23605200 6742762 336830 2979630 " +
"26539520 1424776 4 1820130 8165444\n"+
"8 3 sda3 81830497 279914 17881852954 53132969 2888558 16782291 " +
"157367552 4982925 0 47077660 58061635\n"+
"8 32 sdc 10148118 693255 %d 122125461 6090515 401630172 %d 2696685590 " +
"0 26848216 2818793840\n"+
"8 33 sdc1 10147917 693230 2054138426 122125426 6090506 401630172 " +
"3261765880 2696685589 0 26848181 2818793804\n"+
"8 64 sde 9989771 553047 %d 93407551 5978572 391997273 %d 2388274325 " +
"0 24396646 2481664818\n"+
"8 65 sde1 9989570 553022 1943973346 93407489 5978563 391997273 3183807264 " +
"2388274325 0 24396584 2481666274\n"+
"8 80 sdf 10197163 693995 %d 144374395 6216644 408395438 %d 2669389056 0 " +
"26164759 2813746348\n"+
"8 81 sdf1 10196962 693970 2033452794 144374355 6216635 408395438 3316897064 " +
"2669389056 0 26164719 2813746308\n"+
"8 129 sdi1 10078602 657936 2056552626 108362198 6134036 403851153 3279882064 " +
"2639256086 0 26260432 2747601085\n";
/**
* Test parsing /proc/stat and /proc/cpuinfo
* @throws IOException
*/
@Test
public void parsingProcStatAndCpuFile() throws IOException {
// Write fake /proc/cpuinfo file.
long numProcessors = 8;
long cpuFrequencyKHz = 2392781;
String fileContent = "";
for (int i = 0; i < numProcessors; i++) {
fileContent +=
String.format(CPUINFO_FORMAT, i, cpuFrequencyKHz / 1000D, 0, 0)
+ "\n";
}
File tempFile = new File(FAKE_CPUFILE);
tempFile.deleteOnExit();
FileWriter fWriter = new FileWriter(FAKE_CPUFILE);
fWriter.write(fileContent);
fWriter.close();
assertEquals(plugin.getNumProcessors(), numProcessors);
assertEquals(plugin.getCpuFrequency(), cpuFrequencyKHz);
// Write fake /proc/stat file.
long uTime = 54972994;
long nTime = 188860;
long sTime = 19803373;
tempFile = new File(FAKE_STATFILE);
tempFile.deleteOnExit();
updateStatFile(uTime, nTime, sTime);
assertEquals(plugin.getCumulativeCpuTime(),
FAKE_JIFFY_LENGTH * (uTime + nTime + sTime));
assertEquals(plugin.getCpuUsage(), (float)(CpuTimeTracker.UNAVAILABLE),0.0);
// Advance the time and sample again to test the CPU usage calculation
uTime += 100L;
plugin.advanceTime(200L);
updateStatFile(uTime, nTime, sTime);
assertEquals(plugin.getCumulativeCpuTime(),
FAKE_JIFFY_LENGTH * (uTime + nTime + sTime));
assertEquals(plugin.getCpuUsage(), 6.25F, 0.0);
// Advance the time and sample again. This time, we call getCpuUsage() only.
uTime += 600L;
plugin.advanceTime(300L);
updateStatFile(uTime, nTime, sTime);
assertEquals(plugin.getCpuUsage(), 25F, 0.0);
// Advance very short period of time (one jiffy length).
// In this case, CPU usage should not be updated.
uTime += 1L;
plugin.advanceTime(1L);
updateStatFile(uTime, nTime, sTime);
assertEquals(plugin.getCumulativeCpuTime(),
FAKE_JIFFY_LENGTH * (uTime + nTime + sTime));
assertEquals(plugin.getCpuUsage(), 25F, 0.0); // CPU usage is not updated.
}
/**
* Write information to fake /proc/stat file
*/
private void updateStatFile(long uTime, long nTime, long sTime)
throws IOException {
FileWriter fWriter = new FileWriter(FAKE_STATFILE);
fWriter.write(String.format(STAT_FILE_FORMAT, uTime, nTime, sTime));
fWriter.close();
}
/**
* Test parsing /proc/meminfo
* @throws IOException
*/
@Test
public void parsingProcMemFile() throws IOException {
long memTotal = 4058864L;
long memFree = 99632L;
long inactive = 567732L;
long swapTotal = 2096472L;
long swapFree = 1818480L;
File tempFile = new File(FAKE_MEMFILE);
tempFile.deleteOnExit();
FileWriter fWriter = new FileWriter(FAKE_MEMFILE);
fWriter.write(String.format(MEMINFO_FORMAT,
memTotal, memFree, inactive, swapTotal, swapFree));
fWriter.close();
assertEquals(plugin.getAvailablePhysicalMemorySize(),
1024L * (memFree + inactive));
assertEquals(plugin.getAvailableVirtualMemorySize(),
1024L * (memFree + inactive + swapFree));
assertEquals(plugin.getPhysicalMemorySize(), 1024L * memTotal);
assertEquals(plugin.getVirtualMemorySize(), 1024L * (memTotal + swapTotal));
}
@Test
public void testCoreCounts() throws IOException {
String fileContent = "";
// single core, hyper threading
long numProcessors = 2;
long cpuFrequencyKHz = 2392781;
for (int i = 0; i < numProcessors; i++) {
fileContent =
fileContent.concat(String.format(CPUINFO_FORMAT, i,
cpuFrequencyKHz / 1000D, 0, 0));
fileContent = fileContent.concat("\n");
}
writeFakeCPUInfoFile(fileContent);
plugin.setReadCpuInfoFile(false);
assertEquals(numProcessors, plugin.getNumProcessors());
assertEquals(1, plugin.getNumCores());
// single socket quad core, no hyper threading
fileContent = "";
numProcessors = 4;
for (int i = 0; i < numProcessors; i++) {
fileContent =
fileContent.concat(String.format(CPUINFO_FORMAT, i,
cpuFrequencyKHz / 1000D, 0, i));
fileContent = fileContent.concat("\n");
}
writeFakeCPUInfoFile(fileContent);
plugin.setReadCpuInfoFile(false);
assertEquals(numProcessors, plugin.getNumProcessors());
assertEquals(4, plugin.getNumCores());
// dual socket single core, hyper threading
fileContent = "";
numProcessors = 4;
for (int i = 0; i < numProcessors; i++) {
fileContent =
fileContent.concat(String.format(CPUINFO_FORMAT, i,
cpuFrequencyKHz / 1000D, i / 2, 0));
fileContent = fileContent.concat("\n");
}
writeFakeCPUInfoFile(fileContent);
plugin.setReadCpuInfoFile(false);
assertEquals(numProcessors, plugin.getNumProcessors());
assertEquals(2, plugin.getNumCores());
// dual socket, dual core, no hyper threading
fileContent = "";
numProcessors = 4;
for (int i = 0; i < numProcessors; i++) {
fileContent =
fileContent.concat(String.format(CPUINFO_FORMAT, i,
cpuFrequencyKHz / 1000D, i / 2, i % 2));
fileContent = fileContent.concat("\n");
}
writeFakeCPUInfoFile(fileContent);
plugin.setReadCpuInfoFile(false);
assertEquals(numProcessors, plugin.getNumProcessors());
assertEquals(4, plugin.getNumCores());
// dual socket, dual core, hyper threading
fileContent = "";
numProcessors = 8;
for (int i = 0; i < numProcessors; i++) {
fileContent =
fileContent.concat(String.format(CPUINFO_FORMAT, i,
cpuFrequencyKHz / 1000D, i / 4, (i % 4) / 2));
fileContent = fileContent.concat("\n");
}
writeFakeCPUInfoFile(fileContent);
plugin.setReadCpuInfoFile(false);
assertEquals(numProcessors, plugin.getNumProcessors());
assertEquals(4, plugin.getNumCores());
}
private void writeFakeCPUInfoFile(String content) throws IOException {
File tempFile = new File(FAKE_CPUFILE);
FileWriter fWriter = new FileWriter(FAKE_CPUFILE);
tempFile.deleteOnExit();
try {
fWriter.write(content);
} finally {
IOUtils.closeQuietly(fWriter);
}
}
/**
* Test parsing /proc/net/dev
* @throws IOException
*/
@Test
public void parsingProcNetFile() throws IOException {
long numBytesReadIntf1 = 2097172468L;
long numBytesWrittenIntf1 = 1355620114L;
long numBytesReadIntf2 = 1097172460L;
long numBytesWrittenIntf2 = 1055620110L;
File tempFile = new File(FAKE_NETFILE);
tempFile.deleteOnExit();
FileWriter fWriter = new FileWriter(FAKE_NETFILE);
fWriter.write(String.format(NETINFO_FORMAT,
numBytesReadIntf1, numBytesWrittenIntf1,
numBytesReadIntf2, numBytesWrittenIntf2));
fWriter.close();
assertEquals(plugin.getNetworkBytesRead(), numBytesReadIntf1 + numBytesReadIntf2);
assertEquals(plugin.getNetworkBytesWritten(), numBytesWrittenIntf1 + numBytesWrittenIntf2);
}
/**
* Test parsing /proc/diskstats
* @throws IOException
*/
@Test
public void parsingProcDisksFile() throws IOException {
long numSectorsReadsda = 1790549L; long numSectorsWrittensda = 1839071L;
long numSectorsReadsdc = 20541402L; long numSectorsWrittensdc = 32617658L;
long numSectorsReadsde = 19439751L; long numSectorsWrittensde = 31838072L;
long numSectorsReadsdf = 20334546L; long numSectorsWrittensdf = 33168970L;
File tempFile = new File(FAKE_DISKSFILE);
tempFile.deleteOnExit();
FileWriter fWriter = new FileWriter(FAKE_DISKSFILE);
fWriter.write(String.format(DISKSINFO_FORMAT,
numSectorsReadsda, numSectorsWrittensda,
numSectorsReadsdc, numSectorsWrittensdc,
numSectorsReadsde, numSectorsWrittensde,
numSectorsReadsdf, numSectorsWrittensdf));
fWriter.close();
long expectedNumSectorsRead = numSectorsReadsda + numSectorsReadsdc +
numSectorsReadsde + numSectorsReadsdf;
long expectedNumSectorsWritten = numSectorsWrittensda + numSectorsWrittensdc +
numSectorsWrittensde + numSectorsWrittensdf;
// use non-default sector size
int diskSectorSize = FakeLinuxResourceCalculatorPlugin.SECTORSIZE;
assertEquals(expectedNumSectorsRead * diskSectorSize,
plugin.getStorageBytesRead());
assertEquals(expectedNumSectorsWritten * diskSectorSize,
plugin.getStorageBytesWritten());
}
}