1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.mapreduce;
20
21 import static org.mockito.Mockito.mock;
22 import static org.mockito.Mockito.when;
23
24 import java.io.IOException;
25 import java.util.List;
26
27 import org.apache.hadoop.conf.Configuration;
28 import org.apache.hadoop.fs.Path;
29 import org.apache.hadoop.hbase.HBaseTestingUtility;
30 import org.apache.hadoop.hbase.HConstants;
31 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
32 import org.apache.hadoop.hbase.LargeTests;
33 import org.apache.hadoop.hbase.TableName;
34 import org.apache.hadoop.hbase.client.Result;
35 import org.apache.hadoop.hbase.client.Scan;
36 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
37 import org.apache.hadoop.hbase.mapreduce.TableSnapshotInputFormat.TableSnapshotRegionSplit;
38 import org.apache.hadoop.hbase.util.Bytes;
39 import org.apache.hadoop.hbase.util.FSUtils;
40 import org.apache.hadoop.io.NullWritable;
41 import org.apache.hadoop.mapreduce.InputSplit;
42 import org.apache.hadoop.mapreduce.Job;
43 import org.apache.hadoop.mapreduce.RecordReader;
44 import org.apache.hadoop.mapreduce.Reducer;
45 import org.apache.hadoop.mapreduce.TaskAttemptContext;
46 import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
47 import org.junit.Assert;
48 import org.junit.Test;
49 import org.junit.experimental.categories.Category;
50
51 import com.google.common.collect.Lists;
52
53 @Category(LargeTests.class)
54 public class TestTableSnapshotInputFormat extends TableSnapshotInputFormatTestBase {
55
56 public static byte[] bbb = Bytes.toBytes("bbb");
57 public static byte[] yyy = Bytes.toBytes("yyy");
58
59 @Override
60 protected byte[] getStartRow() {
61 return bbb;
62 }
63
64 @Override
65 protected byte[] getEndRow() {
66 return yyy;
67 }
68
69 @Test
70 public void testGetBestLocations() throws IOException {
71 TableSnapshotInputFormatImpl tsif = new TableSnapshotInputFormatImpl();
72 Configuration conf = UTIL.getConfiguration();
73
74 HDFSBlocksDistribution blockDistribution = new HDFSBlocksDistribution();
75 Assert.assertEquals(Lists.newArrayList(), tsif.getBestLocations(conf, blockDistribution));
76
77 blockDistribution.addHostsAndBlockWeight(new String[] {"h1"}, 1);
78 Assert.assertEquals(Lists.newArrayList("h1"), tsif.getBestLocations(conf, blockDistribution));
79
80 blockDistribution.addHostsAndBlockWeight(new String[] {"h1"}, 1);
81 Assert.assertEquals(Lists.newArrayList("h1"), tsif.getBestLocations(conf, blockDistribution));
82
83 blockDistribution.addHostsAndBlockWeight(new String[] {"h2"}, 1);
84 Assert.assertEquals(Lists.newArrayList("h1"), tsif.getBestLocations(conf, blockDistribution));
85
86 blockDistribution = new HDFSBlocksDistribution();
87 blockDistribution.addHostsAndBlockWeight(new String[] {"h1"}, 10);
88 blockDistribution.addHostsAndBlockWeight(new String[] {"h2"}, 7);
89 blockDistribution.addHostsAndBlockWeight(new String[] {"h3"}, 5);
90 blockDistribution.addHostsAndBlockWeight(new String[] {"h4"}, 1);
91 Assert.assertEquals(Lists.newArrayList("h1"), tsif.getBestLocations(conf, blockDistribution));
92
93 blockDistribution.addHostsAndBlockWeight(new String[] {"h2"}, 2);
94 Assert.assertEquals(Lists.newArrayList("h1", "h2"),
95 tsif.getBestLocations(conf, blockDistribution));
96
97 blockDistribution.addHostsAndBlockWeight(new String[] {"h2"}, 3);
98 Assert.assertEquals(Lists.newArrayList("h2", "h1"),
99 tsif.getBestLocations(conf, blockDistribution));
100
101 blockDistribution.addHostsAndBlockWeight(new String[] {"h3"}, 6);
102 blockDistribution.addHostsAndBlockWeight(new String[] {"h4"}, 9);
103
104 Assert.assertEquals(Lists.newArrayList("h2", "h3", "h4", "h1"),
105 tsif.getBestLocations(conf, blockDistribution));
106 }
107
108 public static enum TestTableSnapshotCounters {
109 VALIDATION_ERROR
110 }
111
112 public static class TestTableSnapshotMapper
113 extends TableMapper<ImmutableBytesWritable, NullWritable> {
114 @Override
115 protected void map(ImmutableBytesWritable key, Result value,
116 Context context) throws IOException, InterruptedException {
117
118 verifyRowFromMap(key, value);
119 context.write(key, NullWritable.get());
120 }
121 }
122
123 public static class TestTableSnapshotReducer
124 extends Reducer<ImmutableBytesWritable, NullWritable, NullWritable, NullWritable> {
125 HBaseTestingUtility.SeenRowTracker rowTracker =
126 new HBaseTestingUtility.SeenRowTracker(bbb, yyy);
127 @Override
128 protected void reduce(ImmutableBytesWritable key, Iterable<NullWritable> values,
129 Context context) throws IOException, InterruptedException {
130 rowTracker.addRow(key.get());
131 }
132
133 @Override
134 protected void cleanup(Context context) throws IOException,
135 InterruptedException {
136 rowTracker.validate();
137 }
138 }
139
140 @Test
141 public void testInitTableSnapshotMapperJobConfig() throws Exception {
142 setupCluster();
143 TableName tableName = TableName.valueOf("testInitTableSnapshotMapperJobConfig");
144 String snapshotName = "foo";
145
146 try {
147 createTableAndSnapshot(UTIL, tableName, snapshotName, getStartRow(), getEndRow(), 1);
148 Job job = new Job(UTIL.getConfiguration());
149 Path tmpTableDir = UTIL.getDataTestDirOnTestFS(snapshotName);
150
151 TableMapReduceUtil.initTableSnapshotMapperJob(snapshotName,
152 new Scan(), TestTableSnapshotMapper.class, ImmutableBytesWritable.class,
153 NullWritable.class, job, false, tmpTableDir);
154
155
156
157 Assert.assertEquals(
158 "Snapshot job should be configured for default LruBlockCache.",
159 HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT,
160 job.getConfiguration().getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, -1), 0.01);
161 Assert.assertEquals(
162 "Snapshot job should not use SlabCache.",
163 0, job.getConfiguration().getFloat("hbase.offheapcache.percentage", -1), 0.01);
164 Assert.assertEquals(
165 "Snapshot job should not use BucketCache.",
166 0, job.getConfiguration().getFloat("hbase.bucketcache.size", -1), 0.01);
167 } finally {
168 UTIL.getHBaseAdmin().deleteSnapshot(snapshotName);
169 UTIL.deleteTable(tableName);
170 tearDownCluster();
171 }
172 }
173
174 public void testWithMockedMapReduce(HBaseTestingUtility util, String snapshotName,
175 int numRegions, int expectedNumSplits) throws Exception {
176 setupCluster();
177 TableName tableName = TableName.valueOf("testWithMockedMapReduce");
178 try {
179 createTableAndSnapshot(
180 util, tableName, snapshotName, getStartRow(), getEndRow(), numRegions);
181
182 Job job = new Job(util.getConfiguration());
183 Path tmpTableDir = util.getDataTestDirOnTestFS(snapshotName);
184 Scan scan = new Scan(getStartRow(), getEndRow());
185
186 TableMapReduceUtil.initTableSnapshotMapperJob(snapshotName,
187 scan, TestTableSnapshotMapper.class, ImmutableBytesWritable.class,
188 NullWritable.class, job, false, tmpTableDir);
189
190 verifyWithMockedMapReduce(job, numRegions, expectedNumSplits, getStartRow(), getEndRow());
191
192 } finally {
193 util.getHBaseAdmin().deleteSnapshot(snapshotName);
194 util.deleteTable(tableName);
195 tearDownCluster();
196 }
197 }
198
199 private void verifyWithMockedMapReduce(Job job, int numRegions, int expectedNumSplits,
200 byte[] startRow, byte[] stopRow)
201 throws IOException, InterruptedException {
202 TableSnapshotInputFormat tsif = new TableSnapshotInputFormat();
203 List<InputSplit> splits = tsif.getSplits(job);
204
205 Assert.assertEquals(expectedNumSplits, splits.size());
206
207 HBaseTestingUtility.SeenRowTracker rowTracker =
208 new HBaseTestingUtility.SeenRowTracker(startRow, stopRow);
209
210 for (int i = 0; i < splits.size(); i++) {
211
212 InputSplit split = splits.get(i);
213 Assert.assertTrue(split instanceof TableSnapshotRegionSplit);
214
215
216 TaskAttemptContext taskAttemptContext = mock(TaskAttemptContext.class);
217 when(taskAttemptContext.getConfiguration()).thenReturn(job.getConfiguration());
218 RecordReader<ImmutableBytesWritable, Result> rr =
219 tsif.createRecordReader(split, taskAttemptContext);
220 rr.initialize(split, taskAttemptContext);
221
222
223 while (rr.nextKeyValue()) {
224 byte[] row = rr.getCurrentKey().get();
225 verifyRowFromMap(rr.getCurrentKey(), rr.getCurrentValue());
226 rowTracker.addRow(row);
227 }
228
229 rr.close();
230 }
231
232
233 rowTracker.validate();
234 }
235
236 @Override
237 protected void testWithMapReduceImpl(HBaseTestingUtility util, TableName tableName,
238 String snapshotName, Path tableDir, int numRegions, int expectedNumSplits,
239 boolean shutdownCluster) throws Exception {
240 doTestWithMapReduce(util, tableName, snapshotName, getStartRow(), getEndRow(), tableDir,
241 numRegions, expectedNumSplits, shutdownCluster);
242 }
243
244
245 public static void doTestWithMapReduce(HBaseTestingUtility util, TableName tableName,
246 String snapshotName, byte[] startRow, byte[] endRow, Path tableDir, int numRegions,
247 int expectedNumSplits, boolean shutdownCluster) throws Exception {
248
249
250 createTableAndSnapshot(util, tableName, snapshotName, startRow, endRow, numRegions);
251
252 if (shutdownCluster) {
253 util.shutdownMiniHBaseCluster();
254 }
255
256 try {
257
258 Job job = new Job(util.getConfiguration());
259 Scan scan = new Scan(startRow, endRow);
260
261 job.setJarByClass(util.getClass());
262 TableMapReduceUtil.addDependencyJars(job.getConfiguration(),
263 TestTableSnapshotInputFormat.class);
264
265 TableMapReduceUtil.initTableSnapshotMapperJob(snapshotName,
266 scan, TestTableSnapshotMapper.class, ImmutableBytesWritable.class,
267 NullWritable.class, job, true, tableDir);
268
269 job.setReducerClass(TestTableSnapshotInputFormat.TestTableSnapshotReducer.class);
270 job.setNumReduceTasks(1);
271 job.setOutputFormatClass(NullOutputFormat.class);
272
273 Assert.assertTrue(job.waitForCompletion(true));
274 } finally {
275 if (!shutdownCluster) {
276 util.getHBaseAdmin().deleteSnapshot(snapshotName);
277 util.deleteTable(tableName);
278 }
279 }
280 }
281 }