1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.mapreduce;
20
21 import static org.mockito.Mockito.mock;
22 import static org.mockito.Mockito.when;
23
24 import java.io.IOException;
25 import java.util.List;
26
27 import org.apache.hadoop.conf.Configuration;
28 import org.apache.hadoop.fs.Path;
29 import org.apache.hadoop.hbase.HBaseTestingUtility;
30 import org.apache.hadoop.hbase.HConstants;
31 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
32 import org.apache.hadoop.hbase.testclassification.LargeTests;
33 import org.apache.hadoop.hbase.TableName;
34 import org.apache.hadoop.hbase.client.Result;
35 import org.apache.hadoop.hbase.client.Scan;
36 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
37 import org.apache.hadoop.hbase.mapreduce.TableSnapshotInputFormat.TableSnapshotRegionSplit;
38 import org.apache.hadoop.hbase.util.Bytes;
39 import org.apache.hadoop.io.NullWritable;
40 import org.apache.hadoop.mapreduce.InputSplit;
41 import org.apache.hadoop.mapreduce.Job;
42 import org.apache.hadoop.mapreduce.RecordReader;
43 import org.apache.hadoop.mapreduce.Reducer;
44 import org.apache.hadoop.mapreduce.TaskAttemptContext;
45 import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
46 import org.junit.After;
47 import org.junit.Assert;
48 import org.junit.Test;
49 import org.junit.experimental.categories.Category;
50
51 import com.google.common.collect.Lists;
52
53 @Category(LargeTests.class)
54 public class TestTableSnapshotInputFormat extends TableSnapshotInputFormatTestBase {
55
56 private static final byte[] bbb = Bytes.toBytes("bbb");
57 private static final byte[] yyy = Bytes.toBytes("yyy");
58
59 @Override
60 protected byte[] getStartRow() {
61 return bbb;
62 }
63
64 @Override
65 protected byte[] getEndRow() {
66 return yyy;
67 }
68
69 @After
70 public void tearDown() throws Exception {
71 }
72
73 @Test
74 public void testGetBestLocations() throws IOException {
75 TableSnapshotInputFormatImpl tsif = new TableSnapshotInputFormatImpl();
76 Configuration conf = UTIL.getConfiguration();
77
78 HDFSBlocksDistribution blockDistribution = new HDFSBlocksDistribution();
79 Assert.assertEquals(Lists.newArrayList(), tsif.getBestLocations(conf, blockDistribution));
80
81 blockDistribution.addHostsAndBlockWeight(new String[] {"h1"}, 1);
82 Assert.assertEquals(Lists.newArrayList("h1"), tsif.getBestLocations(conf, blockDistribution));
83
84 blockDistribution.addHostsAndBlockWeight(new String[] {"h1"}, 1);
85 Assert.assertEquals(Lists.newArrayList("h1"), tsif.getBestLocations(conf, blockDistribution));
86
87 blockDistribution.addHostsAndBlockWeight(new String[] {"h2"}, 1);
88 Assert.assertEquals(Lists.newArrayList("h1"), tsif.getBestLocations(conf, blockDistribution));
89
90 blockDistribution = new HDFSBlocksDistribution();
91 blockDistribution.addHostsAndBlockWeight(new String[] {"h1"}, 10);
92 blockDistribution.addHostsAndBlockWeight(new String[] {"h2"}, 7);
93 blockDistribution.addHostsAndBlockWeight(new String[] {"h3"}, 5);
94 blockDistribution.addHostsAndBlockWeight(new String[] {"h4"}, 1);
95 Assert.assertEquals(Lists.newArrayList("h1"), tsif.getBestLocations(conf, blockDistribution));
96
97 blockDistribution.addHostsAndBlockWeight(new String[] {"h2"}, 2);
98 Assert.assertEquals(Lists.newArrayList("h1", "h2"),
99 tsif.getBestLocations(conf, blockDistribution));
100
101 blockDistribution.addHostsAndBlockWeight(new String[] {"h2"}, 3);
102 Assert.assertEquals(Lists.newArrayList("h2", "h1"),
103 tsif.getBestLocations(conf, blockDistribution));
104
105 blockDistribution.addHostsAndBlockWeight(new String[] {"h3"}, 6);
106 blockDistribution.addHostsAndBlockWeight(new String[] {"h4"}, 9);
107
108 Assert.assertEquals(Lists.newArrayList("h2", "h3", "h4", "h1"),
109 tsif.getBestLocations(conf, blockDistribution));
110 }
111
112 public static enum TestTableSnapshotCounters {
113 VALIDATION_ERROR
114 }
115
116 public static class TestTableSnapshotMapper
117 extends TableMapper<ImmutableBytesWritable, NullWritable> {
118 @Override
119 protected void map(ImmutableBytesWritable key, Result value,
120 Context context) throws IOException, InterruptedException {
121
122 verifyRowFromMap(key, value);
123 context.write(key, NullWritable.get());
124 }
125 }
126
127 public static class TestTableSnapshotReducer
128 extends Reducer<ImmutableBytesWritable, NullWritable, NullWritable, NullWritable> {
129 HBaseTestingUtility.SeenRowTracker rowTracker =
130 new HBaseTestingUtility.SeenRowTracker(bbb, yyy);
131 @Override
132 protected void reduce(ImmutableBytesWritable key, Iterable<NullWritable> values,
133 Context context) throws IOException, InterruptedException {
134 rowTracker.addRow(key.get());
135 }
136
137 @Override
138 protected void cleanup(Context context) throws IOException,
139 InterruptedException {
140 rowTracker.validate();
141 }
142 }
143
144 @Test
145 public void testInitTableSnapshotMapperJobConfig() throws Exception {
146 setupCluster();
147 TableName tableName = TableName.valueOf("testInitTableSnapshotMapperJobConfig");
148 String snapshotName = "foo";
149
150 try {
151 createTableAndSnapshot(UTIL, tableName, snapshotName, getStartRow(), getEndRow(), 1);
152 Job job = new Job(UTIL.getConfiguration());
153 Path tmpTableDir = UTIL.getDataTestDirOnTestFS(snapshotName);
154
155 TableMapReduceUtil.initTableSnapshotMapperJob(snapshotName,
156 new Scan(), TestTableSnapshotMapper.class, ImmutableBytesWritable.class,
157 NullWritable.class, job, false, tmpTableDir);
158
159
160
161 Assert.assertEquals(
162 "Snapshot job should be configured for default LruBlockCache.",
163 HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT,
164 job.getConfiguration().getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, -1), 0.01);
165 Assert.assertEquals(
166 "Snapshot job should not use BucketCache.",
167 0, job.getConfiguration().getFloat("hbase.bucketcache.size", -1), 0.01);
168 } finally {
169 UTIL.getHBaseAdmin().deleteSnapshot(snapshotName);
170 UTIL.deleteTable(tableName);
171 tearDownCluster();
172 }
173 }
174
175 public void testWithMockedMapReduce(HBaseTestingUtility util, String snapshotName,
176 int numRegions, int expectedNumSplits) throws Exception {
177 setupCluster();
178 TableName tableName = TableName.valueOf("testWithMockedMapReduce");
179 try {
180 createTableAndSnapshot(
181 util, tableName, snapshotName, getStartRow(), getEndRow(), numRegions);
182
183 Job job = new Job(util.getConfiguration());
184 Path tmpTableDir = util.getDataTestDirOnTestFS(snapshotName);
185 Scan scan = new Scan(getStartRow(), getEndRow());
186
187 TableMapReduceUtil.initTableSnapshotMapperJob(snapshotName,
188 scan, TestTableSnapshotMapper.class, ImmutableBytesWritable.class,
189 NullWritable.class, job, false, tmpTableDir);
190
191 verifyWithMockedMapReduce(job, numRegions, expectedNumSplits, getStartRow(), getEndRow());
192
193 } finally {
194 util.getHBaseAdmin().deleteSnapshot(snapshotName);
195 util.deleteTable(tableName);
196 tearDownCluster();
197 }
198 }
199
200 private void verifyWithMockedMapReduce(Job job, int numRegions, int expectedNumSplits,
201 byte[] startRow, byte[] stopRow)
202 throws IOException, InterruptedException {
203 TableSnapshotInputFormat tsif = new TableSnapshotInputFormat();
204 List<InputSplit> splits = tsif.getSplits(job);
205
206 Assert.assertEquals(expectedNumSplits, splits.size());
207
208 HBaseTestingUtility.SeenRowTracker rowTracker =
209 new HBaseTestingUtility.SeenRowTracker(startRow, stopRow);
210
211 for (int i = 0; i < splits.size(); i++) {
212
213 InputSplit split = splits.get(i);
214 Assert.assertTrue(split instanceof TableSnapshotRegionSplit);
215
216
217 TaskAttemptContext taskAttemptContext = mock(TaskAttemptContext.class);
218 when(taskAttemptContext.getConfiguration()).thenReturn(job.getConfiguration());
219 RecordReader<ImmutableBytesWritable, Result> rr =
220 tsif.createRecordReader(split, taskAttemptContext);
221 rr.initialize(split, taskAttemptContext);
222
223
224 while (rr.nextKeyValue()) {
225 byte[] row = rr.getCurrentKey().get();
226 verifyRowFromMap(rr.getCurrentKey(), rr.getCurrentValue());
227 rowTracker.addRow(row);
228 }
229
230 rr.close();
231 }
232
233
234 rowTracker.validate();
235 }
236
237 @Override
238 protected void testWithMapReduceImpl(HBaseTestingUtility util, TableName tableName,
239 String snapshotName, Path tableDir, int numRegions, int expectedNumSplits,
240 boolean shutdownCluster) throws Exception {
241 doTestWithMapReduce(util, tableName, snapshotName, getStartRow(), getEndRow(), tableDir,
242 numRegions, expectedNumSplits, shutdownCluster);
243 }
244
245
246 public static void doTestWithMapReduce(HBaseTestingUtility util, TableName tableName,
247 String snapshotName, byte[] startRow, byte[] endRow, Path tableDir, int numRegions,
248 int expectedNumSplits, boolean shutdownCluster) throws Exception {
249
250
251 createTableAndSnapshot(util, tableName, snapshotName, startRow, endRow, numRegions);
252
253 if (shutdownCluster) {
254 util.shutdownMiniHBaseCluster();
255 }
256
257 try {
258
259 Job job = new Job(util.getConfiguration());
260 Scan scan = new Scan(startRow, endRow);
261
262 job.setJarByClass(util.getClass());
263 TableMapReduceUtil.addDependencyJars(job.getConfiguration(),
264 TestTableSnapshotInputFormat.class);
265
266 TableMapReduceUtil.initTableSnapshotMapperJob(snapshotName,
267 scan, TestTableSnapshotMapper.class, ImmutableBytesWritable.class,
268 NullWritable.class, job, true, tableDir);
269
270 job.setReducerClass(TestTableSnapshotInputFormat.TestTableSnapshotReducer.class);
271 job.setNumReduceTasks(1);
272 job.setOutputFormatClass(NullOutputFormat.class);
273
274 Assert.assertTrue(job.waitForCompletion(true));
275 } finally {
276 if (!shutdownCluster) {
277 util.getHBaseAdmin().deleteSnapshot(snapshotName);
278 util.deleteTable(tableName);
279 }
280 }
281 }
282 }