1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.mapreduce;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertTrue;
22  import static org.junit.Assert.fail;
23  
24  import java.io.IOException;
25  import java.nio.ByteBuffer;
26  import java.util.Collection;
27  import java.util.Deque;
28  import java.util.List;
29  import java.util.NavigableMap;
30  import java.util.concurrent.ExecutorService;
31  import java.util.concurrent.atomic.AtomicInteger;
32  
33  import org.apache.commons.logging.Log;
34  import org.apache.commons.logging.LogFactory;
35  import org.apache.hadoop.conf.Configuration;
36  import org.apache.hadoop.fs.FileSystem;
37  import org.apache.hadoop.fs.Path;
38  import org.apache.hadoop.hbase.HBaseTestingUtility;
39  import org.apache.hadoop.hbase.HColumnDescriptor;
40  import org.apache.hadoop.hbase.HConstants;
41  import org.apache.hadoop.hbase.HRegionInfo;
42  import org.apache.hadoop.hbase.HRegionLocation;
43  import org.apache.hadoop.hbase.HTableDescriptor;
44  import org.apache.hadoop.hbase.LargeTests;
45  import org.apache.hadoop.hbase.TableExistsException;
46  import org.apache.hadoop.hbase.catalog.CatalogTracker;
47  import org.apache.hadoop.hbase.catalog.MetaEditor;
48  import org.apache.hadoop.hbase.catalog.MetaReader;
49  import org.apache.hadoop.hbase.client.HConnection;
50  import org.apache.hadoop.hbase.client.HTable;
51  import org.apache.hadoop.hbase.client.Result;
52  import org.apache.hadoop.hbase.client.ResultScanner;
53  import org.apache.hadoop.hbase.client.Scan;
54  import org.apache.hadoop.hbase.ipc.HRegionInterface;
55  import org.apache.hadoop.hbase.regionserver.HRegionServer;
56  import org.apache.hadoop.hbase.regionserver.TestHRegionServerBulkLoad;
57  import org.apache.hadoop.hbase.util.Bytes;
58  import org.apache.hadoop.hbase.util.Pair;
59  import org.junit.AfterClass;
60  import org.junit.BeforeClass;
61  import org.junit.Test;
62  import org.junit.experimental.categories.Category;
63  import org.mockito.Mockito;
64  
65  import com.google.common.collect.Multimap;
66  
67  /**
68   * Test cases for the atomic load error handling of the bulk load functionality.
69   */
70  @Category(LargeTests.class)
71  public class TestLoadIncrementalHFilesSplitRecovery {
72    final static Log LOG = LogFactory.getLog(TestHRegionServerBulkLoad.class);
73  
74    static HBaseTestingUtility util;
75  
76    final static int NUM_CFS = 10;
77    final static byte[] QUAL = Bytes.toBytes("qual");
78    final static int ROWCOUNT = 100;
79    
80    protected static boolean useSecureHBaseOverride = false;
81  
82    private final static byte[][] families = new byte[NUM_CFS][];
83    static {
84      for (int i = 0; i < NUM_CFS; i++) {
85        families[i] = Bytes.toBytes(family(i));
86      }
87    }
88  
89    static byte[] rowkey(int i) {
90      return Bytes.toBytes(String.format("row_%08d", i));
91    }
92  
93    static String family(int i) {
94      return String.format("family_%04d", i);
95    }
96  
97    static byte[] value(int i) {
98      return Bytes.toBytes(String.format("%010d", i));
99    }
100 
101   public static void buildHFiles(FileSystem fs, Path dir, int value)
102       throws IOException {
103     byte[] val = value(value);
104     for (int i = 0; i < NUM_CFS; i++) {
105       Path testIn = new Path(dir, family(i));
106 
107       TestHRegionServerBulkLoad.createHFile(fs, new Path(testIn, "hfile_" + i),
108           Bytes.toBytes(family(i)), QUAL, val, ROWCOUNT);
109     }
110   }
111 
112   /**
113    * Creates a table with given table name and specified number of column
114    * families if the table does not already exist.
115    */
116   private void setupTable(String table, int cfs) throws IOException {
117     try {
118       LOG.info("Creating table " + table);
119       HTableDescriptor htd = new HTableDescriptor(table);
120       for (int i = 0; i < 10; i++) {
121         htd.addFamily(new HColumnDescriptor(family(i)));
122       }
123 
124       util.getHBaseAdmin().createTable(htd);
125     } catch (TableExistsException tee) {
126       LOG.info("Table " + table + " already exists");
127     }
128   }
129 
130   /**
131    * Creates a table with given table name,specified number of column families<br>
132    * and splitkeys if the table does not already exist.
133    * @param table
134    * @param cfs
135    * @param SPLIT_KEYS
136    */
137   private void setupTableWithSplitkeys(String table, int cfs, byte[][] SPLIT_KEYS)
138       throws IOException {
139     try {
140       LOG.info("Creating table " + table);
141       HTableDescriptor htd = new HTableDescriptor(table);
142       for (int i = 0; i < cfs; i++) {
143         htd.addFamily(new HColumnDescriptor(family(i)));
144       }
145 
146       util.getHBaseAdmin().createTable(htd, SPLIT_KEYS);
147     } catch (TableExistsException tee) {
148       LOG.info("Table " + table + " already exists");
149     }
150   }
151 
152   
153   private Path buildBulkFiles(String table, int value) throws Exception {
154     Path dir = util.getDataTestDir(table);
155     Path bulk1 = new Path(dir, table+value);
156     FileSystem fs = util.getTestFileSystem();
157     buildHFiles(fs, bulk1, value);
158     return bulk1;
159   }
160 
161   /**
162    * Populate table with known values.
163    */
164   private void populateTable(String table, int value) throws Exception {
165     // create HFiles for different column families
166     LoadIncrementalHFiles lih =
167         new LoadIncrementalHFiles(util.getConfiguration(), useSecureHBaseOverride);
168     Path bulk1 = buildBulkFiles(table, value);
169     HTable t = new HTable(util.getConfiguration(), Bytes.toBytes(table));
170     lih.doBulkLoad(bulk1, t);
171   }
172 
173   /**
174    * Split the known table in half.  (this is hard coded for this test suite)
175    */
176   private void forceSplit(String table) {
177     try {
178       // need to call regions server to by synchronous but isn't visible.
179       HRegionServer hrs = util.getRSForFirstRegionInTable(Bytes
180           .toBytes(table));
181 
182       for (HRegionInfo hri : hrs.getOnlineRegions()) {
183         if (Bytes.equals(hri.getTableName(), Bytes.toBytes(table))) {
184           // splitRegion doesn't work if startkey/endkey are null
185           hrs.splitRegion(hri, rowkey(ROWCOUNT / 2)); // hard code split
186         }
187       }
188 
189       // verify that split completed.
190       int regions;
191       do {
192         regions = 0;
193         for (HRegionInfo hri : hrs.getOnlineRegions()) {
194           if (Bytes.equals(hri.getTableName(), Bytes.toBytes(table))) {
195             regions++;
196           }
197         }
198         if (regions != 2) {
199           LOG.info("Taking some time to complete split...");
200           Thread.sleep(250);
201         }
202       } while (regions != 2);
203     } catch (IOException e) {
204       e.printStackTrace();
205     } catch (InterruptedException e) {
206       e.printStackTrace();
207     }
208   }
209 
210   @BeforeClass
211   public static void setupCluster() throws Exception {
212     util = new HBaseTestingUtility();
213     util.startMiniCluster(1);
214   }
215 
216   @AfterClass
217   public static void teardownCluster() throws Exception {
218     util.shutdownMiniCluster();
219   }
220 
221   /**
222    * Checks that all columns have the expected value and that there is the
223    * expected number of rows.
224    */
225   void assertExpectedTable(String table, int count, int value) {
226     try {
227       assertEquals(util.getHBaseAdmin().listTables(table).length, 1);
228 
229       HTable t = new HTable(util.getConfiguration(), table);
230       Scan s = new Scan();
231       ResultScanner sr = t.getScanner(s);
232       int i = 0;
233       for (Result r : sr) {
234         i++;
235         for (NavigableMap<byte[], byte[]> nm : r.getNoVersionMap().values()) {
236           for (byte[] val : nm.values()) {
237             assertTrue(Bytes.equals(val, value(value)));
238           }
239         }
240       }
241       assertEquals(count, i);
242     } catch (IOException e) {
243       fail("Failed due to exception");
244     }
245   }
246 
247   /**
248    * Test that shows that exception thrown from the RS side will result in an
249    * exception on the LIHFile client.
250    */
251   @Test(expected=IOException.class)
252   public void testBulkLoadPhaseFailure() throws Exception {
253     String table = "bulkLoadPhaseFailure";
254     setupTable(table, 10);
255 
256     final AtomicInteger attmptedCalls = new AtomicInteger();
257     final AtomicInteger failedCalls = new AtomicInteger();
258     LoadIncrementalHFiles lih = new LoadIncrementalHFiles(
259         util.getConfiguration(), useSecureHBaseOverride) {
260 
261       protected List<LoadQueueItem> tryAtomicRegionLoad(final HConnection conn,
262           byte[] tableName, final byte[] first, Collection<LoadQueueItem> lqis)
263       throws IOException {
264         int i = attmptedCalls.incrementAndGet();
265         if (i == 1) {
266           HConnection errConn = null;
267           try {
268             errConn = getMockedConnection(util.getConfiguration());
269           } catch (Exception e) {
270             LOG.fatal("mocking cruft, should never happen", e);
271             throw new RuntimeException("mocking cruft, should never happen");
272           }
273           failedCalls.incrementAndGet();
274           return super.tryAtomicRegionLoad(errConn, tableName, first, lqis);
275         }
276 
277         return super.tryAtomicRegionLoad(conn, tableName, first, lqis);
278       }
279     };
280 
281     // create HFiles for different column families
282     Path dir = buildBulkFiles(table, 1);
283     HTable t = new HTable(util.getConfiguration(), Bytes.toBytes(table));
284     lih.doBulkLoad(dir, t);
285 
286     fail("doBulkLoad should have thrown an exception");
287   }
288 
289   private HConnection getMockedConnection(final Configuration conf)
290   throws IOException {
291     HConnection c = Mockito.mock(HConnection.class);
292     Mockito.when(c.getConfiguration()).thenReturn(conf);
293     Mockito.doNothing().when(c).close();
294     // Make it so we return a particular location when asked.
295     final HRegionLocation loc = new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO,
296         "example.org", 1234);
297     Mockito.when(c.getRegionLocation((byte[]) Mockito.any(),
298         (byte[]) Mockito.any(), Mockito.anyBoolean())).
299       thenReturn(loc);
300     Mockito.when(c.locateRegion((byte[]) Mockito.any(), (byte[]) Mockito.any())).
301       thenReturn(loc);
302     HRegionInterface hri = Mockito.mock(HRegionInterface.class);
303     Mockito.when(hri.bulkLoadHFiles(Mockito.anyList(), (byte [])Mockito.any(),
304       Mockito.anyBoolean())).thenThrow(new IOException("injecting bulk load error"));
305     Mockito.when(c.getHRegionConnection(Mockito.anyString(), Mockito.anyInt())).
306       thenReturn(hri);
307     return c;
308   }
309 
310   /**
311    * This test exercises the path where there is a split after initial
312    * validation but before the atomic bulk load call. We cannot use presplitting
313    * to test this path, so we actually inject a split just before the atomic
314    * region load.
315    */
316   @Test
317   public void testSplitWhileBulkLoadPhase() throws Exception {
318     final String table = "splitWhileBulkloadPhase";
319     setupTable(table, 10);
320     populateTable(table,1);
321     assertExpectedTable(table, ROWCOUNT, 1);
322 
323     // Now let's cause trouble.  This will occur after checks and cause bulk
324     // files to fail when attempt to atomically import.  This is recoverable.
325     final AtomicInteger attemptedCalls = new AtomicInteger();
326     LoadIncrementalHFiles lih2 =
327         new LoadIncrementalHFiles(util.getConfiguration(), useSecureHBaseOverride) {
328 
329       protected void bulkLoadPhase(final HTable htable, final HConnection conn,
330           ExecutorService pool, Deque<LoadQueueItem> queue,
331           final Multimap<ByteBuffer, LoadQueueItem> regionGroups) throws IOException {
332         int i = attemptedCalls.incrementAndGet();
333         if (i == 1) {
334           // On first attempt force a split.
335           forceSplit(table);
336         }
337 
338         super.bulkLoadPhase(htable, conn, pool, queue, regionGroups);
339       }
340     };
341 
342     // create HFiles for different column families
343     HTable t = new HTable(util.getConfiguration(), Bytes.toBytes(table));
344     Path bulk = buildBulkFiles(table, 2);
345     lih2.doBulkLoad(bulk, t);
346 
347     // check that data was loaded
348     // The three expected attempts are 1) failure because need to split, 2)
349     // load of split top 3) load of split bottom
350     assertEquals(attemptedCalls.get(), 3);
351     assertExpectedTable(table, ROWCOUNT, 2);
352   }
353 
354   /**
355    * This test splits a table and attempts to bulk load.  The bulk import files
356    * should be split before atomically importing.
357    */
358   @Test
359   public void testGroupOrSplitPresplit() throws Exception {
360     final String table = "groupOrSplitPresplit";
361     setupTable(table, 10);
362     populateTable(table, 1);
363     assertExpectedTable(table, ROWCOUNT, 1);
364     forceSplit(table);
365 
366     final AtomicInteger countedLqis= new AtomicInteger();
367     LoadIncrementalHFiles lih =
368         new LoadIncrementalHFiles(util.getConfiguration(), useSecureHBaseOverride) {
369       protected List<LoadQueueItem> groupOrSplit(
370           Multimap<ByteBuffer, LoadQueueItem> regionGroups,
371           final LoadQueueItem item, final HTable htable,
372           final Pair<byte[][], byte[][]> startEndKeys) throws IOException {
373         List<LoadQueueItem> lqis = super.groupOrSplit(regionGroups, item, htable, startEndKeys);
374         if (lqis != null) {
375           countedLqis.addAndGet(lqis.size());
376         }
377         return lqis;
378       }
379     };
380 
381     // create HFiles for different column families
382     Path bulk = buildBulkFiles(table, 2);
383     HTable ht = new HTable(util.getConfiguration(), Bytes.toBytes(table));
384     lih.doBulkLoad(bulk, ht);
385 
386     assertExpectedTable(table, ROWCOUNT, 2);
387     assertEquals(20, countedLqis.get());
388   }
389 
390   /**
391    * This simulates an remote exception which should cause LIHF to exit with an
392    * exception.
393    */
394   @Test(expected = IOException.class)
395   public void testGroupOrSplitFailure() throws Exception {
396     String table = "groupOrSplitFailure";
397     setupTable(table, 10);
398 
399     LoadIncrementalHFiles lih =
400         new LoadIncrementalHFiles(util.getConfiguration(), useSecureHBaseOverride) {
401       int i = 0;
402 
403       protected List<LoadQueueItem> groupOrSplit(
404           Multimap<ByteBuffer, LoadQueueItem> regionGroups,
405           final LoadQueueItem item, final HTable table,
406           final Pair<byte[][], byte[][]> startEndKeys) throws IOException {
407         i++;
408 
409         if (i == 5) {
410           throw new IOException("failure");
411         }
412         return super.groupOrSplit(regionGroups, item, table, startEndKeys);
413       }
414     };
415 
416     // create HFiles for different column families
417     Path dir = buildBulkFiles(table,1);
418     HTable t = new HTable(util.getConfiguration(), Bytes.toBytes(table));
419     lih.doBulkLoad(dir, t);
420 
421     fail("doBulkLoad should have thrown an exception");
422   }
423   
424   @Test
425   public void testGroupOrSplitWhenRegionHoleExistsInMeta() throws Exception {
426     String tableName = "testGroupOrSplitWhenRegionHoleExistsInMeta";
427     byte[][] SPLIT_KEYS = new byte[][] { Bytes.toBytes("row_00000100") };
428 
429     setupTableWithSplitkeys(tableName, 10, SPLIT_KEYS);
430     HTable table = new HTable(util.getConfiguration(), Bytes.toBytes(tableName));
431     Path dir = buildBulkFiles(tableName, 2);
432 
433     final AtomicInteger countedLqis = new AtomicInteger();
434     LoadIncrementalHFiles loader = new LoadIncrementalHFiles(
435       util.getConfiguration()) {
436       
437     protected List<LoadQueueItem> groupOrSplit(
438         Multimap<ByteBuffer, LoadQueueItem> regionGroups,
439         final LoadQueueItem item, final HTable htable,
440         final Pair<byte[][], byte[][]> startEndKeys) throws IOException {
441       List<LoadQueueItem> lqis = super.groupOrSplit(regionGroups, item, htable, startEndKeys);
442       if (lqis != null) {
443         countedLqis.addAndGet(lqis.size());
444       }
445       return lqis;
446     }
447   };
448 
449     // do bulkload when there is no region hole in hbase:meta.
450     try {
451       loader.doBulkLoad(dir, table);
452     } catch (Exception e) {
453       LOG.error("exeception=", e);
454     }
455     // check if all the data are loaded into the table.
456     this.assertExpectedTable(tableName, ROWCOUNT, 2);
457 
458     dir = buildBulkFiles(tableName, 3);
459 
460     // Mess it up by leaving a hole in the hbase:meta
461     CatalogTracker ct = new CatalogTracker(util.getConfiguration());
462     List<HRegionInfo> regionInfos = MetaReader.getTableRegions(ct, Bytes.toBytes(tableName));
463     for (HRegionInfo regionInfo : regionInfos) {
464       if (Bytes.equals(regionInfo.getStartKey(), HConstants.EMPTY_BYTE_ARRAY)) {
465         MetaEditor.deleteRegion(ct, regionInfo);
466         break;
467       }
468     }
469 
470     try {
471       loader.doBulkLoad(dir, table);
472     } catch (Exception e) {
473       LOG.error("exeception=", e);
474       assertTrue("IOException expected", e instanceof IOException);
475     }
476 
477     table.close();
478 
479     this.assertExpectedTable(tableName, ROWCOUNT, 2);
480   }
481 
482   @org.junit.Rule
483   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
484     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
485 }
486