1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.mapred;
21
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertFalse;
24 import static org.junit.Assert.assertTrue;
25 import static org.mockito.Matchers.anyObject;
26 import static org.mockito.Mockito.doAnswer;
27 import static org.mockito.Mockito.doReturn;
28 import static org.mockito.Mockito.doThrow;
29 import static org.mockito.Mockito.mock;
30 import static org.mockito.Mockito.spy;
31
32 import java.io.IOException;
33 import java.util.Arrays;
34 import java.util.Map;
35
36 import org.apache.commons.logging.Log;
37 import org.apache.commons.logging.LogFactory;
38 import org.apache.hadoop.hbase.*;
39 import org.apache.hadoop.hbase.client.HTable;
40 import org.apache.hadoop.hbase.client.Put;
41 import org.apache.hadoop.hbase.client.Result;
42 import org.apache.hadoop.hbase.client.ResultScanner;
43 import org.apache.hadoop.hbase.client.Scan;
44 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
45 import org.apache.hadoop.hbase.util.Bytes;
46 import org.junit.AfterClass;
47 import org.junit.Before;
48 import org.junit.BeforeClass;
49 import org.junit.Test;
50 import org.junit.experimental.categories.Category;
51 import org.mockito.invocation.InvocationOnMock;
52 import org.mockito.stubbing.Answer;
53
54
55
56
57
58 @Category(LargeTests.class)
59 public class TestTableInputFormat {
60
61 private static final Log LOG = LogFactory.getLog(TestTableInputFormat.class);
62
63 private final static HBaseTestingUtility UTIL = new HBaseTestingUtility();
64 static final byte[] FAMILY = Bytes.toBytes("family");
65
66 private static final byte[][] columns = new byte[][] { FAMILY };
67
68 @BeforeClass
69 public static void beforeClass() throws Exception {
70 UTIL.startMiniCluster();
71 }
72
73 @AfterClass
74 public static void afterClass() throws Exception {
75 UTIL.shutdownMiniCluster();
76 }
77
78 @Before
79 public void before() throws IOException {
80 LOG.info("before");
81 UTIL.ensureSomeRegionServersAvailable(1);
82 LOG.info("before done");
83 }
84
85
86
87
88
89
90
91
92 public static HTable createTable(byte[] tableName) throws IOException {
93 HTable table = UTIL.createTable(tableName, FAMILY);
94 Put p = new Put("aaa".getBytes());
95 p.add(FAMILY, null, "value aaa".getBytes());
96 table.put(p);
97 p = new Put("bbb".getBytes());
98 p.add(FAMILY, null, "value bbb".getBytes());
99 table.put(p);
100 return table;
101 }
102
103
104
105
106
107
108
109
110
111
112 static boolean checkResult(Result r, ImmutableBytesWritable key,
113 byte[] expectedKey, byte[] expectedValue) {
114 assertEquals(0, key.compareTo(expectedKey));
115 Map<byte[], byte[]> vals = r.getFamilyMap(FAMILY);
116 byte[] value = vals.values().iterator().next();
117 assertTrue(Arrays.equals(value, expectedValue));
118 return true;
119 }
120
121
122
123
124
125
126
127
128 static void runTestMapred(HTable table) throws IOException {
129 org.apache.hadoop.hbase.mapred.TableRecordReader trr =
130 new org.apache.hadoop.hbase.mapred.TableRecordReader();
131 trr.setStartRow("aaa".getBytes());
132 trr.setEndRow("zzz".getBytes());
133 trr.setHTable(table);
134 trr.setInputColumns(columns);
135
136 trr.init();
137 Result r = new Result();
138 ImmutableBytesWritable key = new ImmutableBytesWritable();
139
140 boolean more = trr.next(key, r);
141 assertTrue(more);
142 checkResult(r, key, "aaa".getBytes(), "value aaa".getBytes());
143
144 more = trr.next(key, r);
145 assertTrue(more);
146 checkResult(r, key, "bbb".getBytes(), "value bbb".getBytes());
147
148
149 more = trr.next(key, r);
150 assertFalse(more);
151 }
152
153
154
155
156
157
158
159
160
161 static void runTestMapreduce(HTable table) throws IOException,
162 InterruptedException {
163 org.apache.hadoop.hbase.mapreduce.TableRecordReaderImpl trr =
164 new org.apache.hadoop.hbase.mapreduce.TableRecordReaderImpl();
165 Scan s = new Scan();
166 s.setStartRow("aaa".getBytes());
167 s.setStopRow("zzz".getBytes());
168 s.addFamily(FAMILY);
169 trr.setScan(s);
170 trr.setHTable(table);
171
172 trr.initialize(null, null);
173 Result r = new Result();
174 ImmutableBytesWritable key = new ImmutableBytesWritable();
175
176 boolean more = trr.nextKeyValue();
177 assertTrue(more);
178 key = trr.getCurrentKey();
179 r = trr.getCurrentValue();
180 checkResult(r, key, "aaa".getBytes(), "value aaa".getBytes());
181
182 more = trr.nextKeyValue();
183 assertTrue(more);
184 key = trr.getCurrentKey();
185 r = trr.getCurrentValue();
186 checkResult(r, key, "bbb".getBytes(), "value bbb".getBytes());
187
188
189 more = trr.nextKeyValue();
190 assertFalse(more);
191 }
192
193
194
195
196
197
198 static HTable createIOEScannerTable(byte[] name, final int failCnt)
199 throws IOException {
200
201 Answer<ResultScanner> a = new Answer<ResultScanner>() {
202 int cnt = 0;
203
204 @Override
205 public ResultScanner answer(InvocationOnMock invocation) throws Throwable {
206
207 if (cnt++ < failCnt) {
208
209 Scan scan = mock(Scan.class);
210 doReturn("bogus".getBytes()).when(scan).getStartRow();
211 ResultScanner scanner = mock(ResultScanner.class);
212
213 doThrow(new IOException("Injected exception")).when(scanner).next();
214 return scanner;
215 }
216
217
218 return (ResultScanner) invocation.callRealMethod();
219 }
220 };
221
222 HTable htable = spy(createTable(name));
223 doAnswer(a).when(htable).getScanner((Scan) anyObject());
224 return htable;
225 }
226
227
228
229
230
231
232
233 static HTable createDNRIOEScannerTable(byte[] name, final int failCnt)
234 throws IOException {
235
236 Answer<ResultScanner> a = new Answer<ResultScanner>() {
237 int cnt = 0;
238
239 @Override
240 public ResultScanner answer(InvocationOnMock invocation) throws Throwable {
241
242 if (cnt++ < failCnt) {
243
244 Scan scan = mock(Scan.class);
245 doReturn("bogus".getBytes()).when(scan).getStartRow();
246 ResultScanner scanner = mock(ResultScanner.class);
247
248 invocation.callRealMethod();
249 doThrow(
250 new UnknownScannerException("Injected simulated TimeoutException"))
251 .when(scanner).next();
252 return scanner;
253 }
254
255
256 return (ResultScanner) invocation.callRealMethod();
257 }
258 };
259
260 HTable htable = spy(createTable(name));
261 doAnswer(a).when(htable).getScanner((Scan) anyObject());
262 return htable;
263 }
264
265
266
267
268
269
270 @Test
271 public void testTableRecordReader() throws IOException {
272 HTable table = createTable("table1".getBytes());
273 runTestMapred(table);
274 }
275
276
277
278
279
280
281 @Test
282 public void testTableRecordReaderScannerFail() throws IOException {
283 HTable htable = createIOEScannerTable("table2".getBytes(), 1);
284 runTestMapred(htable);
285 }
286
287
288
289
290
291
292 @Test(expected = IOException.class)
293 public void testTableRecordReaderScannerFailTwice() throws IOException {
294 HTable htable = createIOEScannerTable("table3".getBytes(), 2);
295 runTestMapred(htable);
296 }
297
298
299
300
301
302
303
304 @Test
305 public void testTableRecordReaderScannerTimeout() throws IOException {
306 HTable htable = createDNRIOEScannerTable("table4".getBytes(), 1);
307 runTestMapred(htable);
308 }
309
310
311
312
313
314
315
316 @Test(expected = DoNotRetryIOException.class)
317 public void testTableRecordReaderScannerTimeoutTwice() throws IOException {
318 HTable htable = createDNRIOEScannerTable("table5".getBytes(), 2);
319 runTestMapred(htable);
320 }
321
322
323
324
325
326
327
328 @Test
329 public void testTableRecordReaderMapreduce() throws IOException,
330 InterruptedException {
331 HTable table = createTable("table1-mr".getBytes());
332 runTestMapreduce(table);
333 }
334
335
336
337
338
339
340
341 @Test
342 public void testTableRecordReaderScannerFailMapreduce() throws IOException,
343 InterruptedException {
344 HTable htable = createIOEScannerTable("table2-mr".getBytes(), 1);
345 runTestMapreduce(htable);
346 }
347
348
349
350
351
352
353
354 @Test(expected = IOException.class)
355 public void testTableRecordReaderScannerFailMapreduceTwice() throws IOException,
356 InterruptedException {
357 HTable htable = createIOEScannerTable("table3-mr".getBytes(), 2);
358 runTestMapreduce(htable);
359 }
360
361
362
363
364
365
366
367
368 @Test
369 public void testTableRecordReaderScannerTimeoutMapreduce()
370 throws IOException, InterruptedException {
371 HTable htable = createDNRIOEScannerTable("table4-mr".getBytes(), 1);
372 runTestMapreduce(htable);
373 }
374
375
376
377
378
379
380
381
382 @Test(expected = DoNotRetryIOException.class)
383 public void testTableRecordReaderScannerTimeoutMapreduceTwice()
384 throws IOException, InterruptedException {
385 HTable htable = createDNRIOEScannerTable("table5-mr".getBytes(), 2);
386 runTestMapreduce(htable);
387 }
388
389 @org.junit.Rule
390 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
391 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
392 }
393