Skip to content

MAPREDUCE-6729. Accurately compute the test execute time in DFSIO #112

New issue

Have a question about this project? # for a free GitHub account to open an issue and contact its maintainers and the community.

#

By clicking “#”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? # to your account

Closed
wants to merge 2 commits into from
Closed

MAPREDUCE-6729. Accurately compute the test execute time in DFSIO #112

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2a295d0
MAPREDUCE-6729. Accurately compute the test execute time in DFSIO
Jul 8, 2016
b8d6332
Merge branch 'trunk' of https://github.com/zhangminglei/hadoop into t…
Jul 8, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 27 additions & 27 deletions ...lient/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -228,67 +228,53 @@ public static void afterClass() throws Exception {

public static void testWrite() throws Exception {
FileSystem fs = cluster.getFileSystem();
long tStart = System.currentTimeMillis();
bench.writeTest(fs);
long execTime = System.currentTimeMillis() - tStart;
long execTime = bench.writeTest(fs);
bench.analyzeResult(fs, TestType.TEST_TYPE_WRITE, execTime);
}

@Test (timeout = 3000)
public void testRead() throws Exception {
FileSystem fs = cluster.getFileSystem();
long tStart = System.currentTimeMillis();
bench.readTest(fs);
long execTime = System.currentTimeMillis() - tStart;
long execTime = bench.readTest(fs);
bench.analyzeResult(fs, TestType.TEST_TYPE_READ, execTime);
}

@Test (timeout = 3000)
public void testReadRandom() throws Exception {
FileSystem fs = cluster.getFileSystem();
long tStart = System.currentTimeMillis();
bench.getConf().setLong("test.io.skip.size", 0);
bench.randomReadTest(fs);
long execTime = System.currentTimeMillis() - tStart;
long execTime = bench.randomReadTest(fs);
bench.analyzeResult(fs, TestType.TEST_TYPE_READ_RANDOM, execTime);
}

@Test (timeout = 3000)
public void testReadBackward() throws Exception {
FileSystem fs = cluster.getFileSystem();
long tStart = System.currentTimeMillis();
bench.getConf().setLong("test.io.skip.size", -DEFAULT_BUFFER_SIZE);
bench.randomReadTest(fs);
long execTime = System.currentTimeMillis() - tStart;
long execTime = bench.randomReadTest(fs);
bench.analyzeResult(fs, TestType.TEST_TYPE_READ_BACKWARD, execTime);
}

@Test (timeout = 3000)
public void testReadSkip() throws Exception {
FileSystem fs = cluster.getFileSystem();
long tStart = System.currentTimeMillis();
bench.getConf().setLong("test.io.skip.size", 1);
bench.randomReadTest(fs);
long execTime = System.currentTimeMillis() - tStart;
long execTime = bench.randomReadTest(fs);
bench.analyzeResult(fs, TestType.TEST_TYPE_READ_SKIP, execTime);
}

@Test (timeout = 6000)
public void testAppend() throws Exception {
FileSystem fs = cluster.getFileSystem();
long tStart = System.currentTimeMillis();
bench.appendTest(fs);
long execTime = System.currentTimeMillis() - tStart;
long execTime = bench.appendTest(fs);
bench.analyzeResult(fs, TestType.TEST_TYPE_APPEND, execTime);
}

@Test (timeout = 60000)
public void testTruncate() throws Exception {
FileSystem fs = cluster.getFileSystem();
bench.createControlFile(fs, DEFAULT_NR_BYTES / 2, DEFAULT_NR_FILES);
long tStart = System.currentTimeMillis();
bench.truncateTest(fs);
long execTime = System.currentTimeMillis() - tStart;
long execTime = bench.truncateTest(fs);
bench.analyzeResult(fs, TestType.TEST_TYPE_TRUNCATE, execTime);
}

Expand Down Expand Up @@ -430,12 +416,14 @@ public Long doIO(Reporter reporter,
}
}

private void writeTest(FileSystem fs) throws IOException {
private long writeTest(FileSystem fs) throws IOException {
Path writeDir = getWriteDir(config);
fs.delete(getDataDir(config), true);
fs.delete(writeDir, true);

long tStart = System.currentTimeMillis();
runIOTest(WriteMapper.class, writeDir);
long execTime = System.currentTimeMillis() - tStart;
return execTime;
}

private void runIOTest(
Expand Down Expand Up @@ -496,10 +484,13 @@ public Long doIO(Reporter reporter,
}
}

private void appendTest(FileSystem fs) throws IOException {
private long appendTest(FileSystem fs) throws IOException {
Path appendDir = getAppendDir(config);
fs.delete(appendDir, true);
long tStart = System.currentTimeMillis();
runIOTest(AppendMapper.class, appendDir);
long execTime = System.currentTimeMillis() - tStart;
return execTime;
}

/**
Expand Down Expand Up @@ -539,10 +530,13 @@ public Long doIO(Reporter reporter,
}
}

private void readTest(FileSystem fs) throws IOException {
private long readTest(FileSystem fs) throws IOException {
Path readDir = getReadDir(config);
fs.delete(readDir, true);
long tStart = System.currentTimeMillis();
runIOTest(ReadMapper.class, readDir);
long execTime = System.currentTimeMillis() - tStart;
return execTime;
}

/**
Expand Down Expand Up @@ -620,10 +614,13 @@ private long nextOffset(long current) {
}
}

private void randomReadTest(FileSystem fs) throws IOException {
private long randomReadTest(FileSystem fs) throws IOException {
Path readDir = getRandomReadDir(config);
fs.delete(readDir, true);
long tStart = System.currentTimeMillis();
runIOTest(RandomReadMapper.class, readDir);
long execTime = System.currentTimeMillis() - tStart;
return execTime;
}

/**
Expand Down Expand Up @@ -665,10 +662,13 @@ public Long doIO(Reporter reporter,
}
}

private void truncateTest(FileSystem fs) throws IOException {
private long truncateTest(FileSystem fs) throws IOException {
Path TruncateDir = getTruncateDir(config);
fs.delete(TruncateDir, true);
long tStart = System.currentTimeMillis();
runIOTest(TruncateMapper.class, TruncateDir);
long execTime = System.currentTimeMillis() - tStart;
return execTime;
}

private void sequentialTest(FileSystem fs,
Expand Down