View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver.wal;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertTrue;
23  import static org.junit.Assert.fail;
24  import static org.mockito.Matchers.any;
25  import static org.mockito.Matchers.eq;
26  import static org.mockito.Mockito.doAnswer;
27  import static org.mockito.Mockito.spy;
28  
29  import java.io.FilterInputStream;
30  import java.io.IOException;
31  import java.io.InputStream;
32  import java.lang.reflect.Field;
33  import java.security.PrivilegedExceptionAction;
34  import java.util.ArrayList;
35  import java.util.List;
36  import java.util.SortedSet;
37  import java.util.concurrent.atomic.AtomicBoolean;
38  import java.util.concurrent.atomic.AtomicInteger;
39  import java.util.concurrent.atomic.AtomicLong;
40  
41  import org.apache.commons.logging.Log;
42  import org.apache.commons.logging.LogFactory;
43  import org.apache.hadoop.conf.Configuration;
44  import org.apache.hadoop.fs.FSDataInputStream;
45  import org.apache.hadoop.fs.FileStatus;
46  import org.apache.hadoop.fs.FileSystem;
47  import org.apache.hadoop.fs.Path;
48  import org.apache.hadoop.hbase.Cell;
49  import org.apache.hadoop.hbase.HBaseConfiguration;
50  import org.apache.hadoop.hbase.HBaseTestingUtility;
51  import org.apache.hadoop.hbase.HColumnDescriptor;
52  import org.apache.hadoop.hbase.HConstants;
53  import org.apache.hadoop.hbase.HRegionInfo;
54  import org.apache.hadoop.hbase.HTableDescriptor;
55  import org.apache.hadoop.hbase.KeyValue;
56  import org.apache.hadoop.hbase.MasterNotRunningException;
57  import org.apache.hadoop.hbase.MiniHBaseCluster;
58  import org.apache.hadoop.hbase.ServerName;
59  import org.apache.hadoop.hbase.TableName;
60  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
61  import org.apache.hadoop.hbase.client.Delete;
62  import org.apache.hadoop.hbase.client.Get;
63  import org.apache.hadoop.hbase.client.HTable;
64  import org.apache.hadoop.hbase.client.Put;
65  import org.apache.hadoop.hbase.client.Result;
66  import org.apache.hadoop.hbase.client.ResultScanner;
67  import org.apache.hadoop.hbase.client.Scan;
68  import org.apache.hadoop.hbase.master.HMaster;
69  import org.apache.hadoop.hbase.monitoring.MonitoredTask;
70  import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
71  import org.apache.hadoop.hbase.regionserver.DefaultStoreEngine;
72  import org.apache.hadoop.hbase.regionserver.DefaultStoreFlusher;
73  import org.apache.hadoop.hbase.regionserver.FlushRequester;
74  import org.apache.hadoop.hbase.regionserver.HRegion;
75  import org.apache.hadoop.hbase.regionserver.HRegionServer;
76  import org.apache.hadoop.hbase.regionserver.RegionScanner;
77  import org.apache.hadoop.hbase.regionserver.RegionServerServices;
78  import org.apache.hadoop.hbase.regionserver.Store;
79  import org.apache.hadoop.hbase.regionserver.TimeRangeTracker;
80  import org.apache.hadoop.hbase.security.User;
81  import org.apache.hadoop.hbase.testclassification.MediumTests;
82  import org.apache.hadoop.hbase.util.Bytes;
83  import org.apache.hadoop.hbase.util.EnvironmentEdge;
84  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
85  import org.apache.hadoop.hbase.util.FSUtils;
86  import org.apache.hadoop.hbase.util.HFileTestUtil;
87  import org.apache.hadoop.hbase.util.Pair;
88  import org.junit.After;
89  import org.junit.AfterClass;
90  import org.junit.Before;
91  import org.junit.BeforeClass;
92  import org.junit.Test;
93  import org.junit.experimental.categories.Category;
94  import org.mockito.Mockito;
95  import org.mockito.invocation.InvocationOnMock;
96  import org.mockito.stubbing.Answer;
97  
98  /**
99   * Test replay of edits out of a WAL split.
100  */
101 @Category(MediumTests.class)
102 public class TestWALReplay {
103   public static final Log LOG = LogFactory.getLog(TestWALReplay.class);
104   static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
105   private final EnvironmentEdge ee = EnvironmentEdgeManager.getDelegate();
106   private Path hbaseRootDir = null;
107   private String logName;
108   private Path oldLogDir;
109   private Path logDir;
110   private FileSystem fs;
111   private Configuration conf;
112   private RecoveryMode mode;
113   
114 
115   @BeforeClass
116   public static void setUpBeforeClass() throws Exception {
117     Configuration conf = TEST_UTIL.getConfiguration();
118     conf.setBoolean("dfs.support.append", true);
119     // The below config supported by 0.20-append and CDH3b2
120     conf.setInt("dfs.client.block.recovery.retries", 2);
121     TEST_UTIL.startMiniCluster(3);
122     Path hbaseRootDir =
123       TEST_UTIL.getDFSCluster().getFileSystem().makeQualified(new Path("/hbase"));
124     LOG.info("hbase.rootdir=" + hbaseRootDir);
125     FSUtils.setRootDir(conf, hbaseRootDir);
126   }
127 
128   @AfterClass
129   public static void tearDownAfterClass() throws Exception {
130     TEST_UTIL.shutdownMiniCluster();
131   }
132 
133   @Before
134   public void setUp() throws Exception {
135     this.conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
136     this.fs = TEST_UTIL.getDFSCluster().getFileSystem();
137     this.hbaseRootDir = FSUtils.getRootDir(this.conf);
138     this.oldLogDir = new Path(this.hbaseRootDir, HConstants.HREGION_OLDLOGDIR_NAME);
139     this.logName = HConstants.HREGION_LOGDIR_NAME;
140     this.logDir = new Path(this.hbaseRootDir, logName);
141     if (TEST_UTIL.getDFSCluster().getFileSystem().exists(this.hbaseRootDir)) {
142       TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
143     }
144     this.mode = (conf.getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false) ? 
145         RecoveryMode.LOG_REPLAY : RecoveryMode.LOG_SPLITTING);
146   }
147 
148   @After
149   public void tearDown() throws Exception {
150     TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
151   }
152 
153   /*
154    * @param p Directory to cleanup
155    */
156   private void deleteDir(final Path p) throws IOException {
157     if (this.fs.exists(p)) {
158       if (!this.fs.delete(p, true)) {
159         throw new IOException("Failed remove of " + p);
160       }
161     }
162   }
163 
164   /**
165    * 
166    * @throws Exception
167    */
168   @Test
169   public void testReplayEditsAfterRegionMovedWithMultiCF() throws Exception {
170     final TableName tableName =
171         TableName.valueOf("testReplayEditsAfterRegionMovedWithMultiCF");
172     byte[] family1 = Bytes.toBytes("cf1");
173     byte[] family2 = Bytes.toBytes("cf2");
174     byte[] qualifier = Bytes.toBytes("q");
175     byte[] value = Bytes.toBytes("testV");
176     byte[][] familys = { family1, family2 };
177     TEST_UTIL.createTable(tableName, familys);
178     HTable htable = new HTable(TEST_UTIL.getConfiguration(), tableName);
179     Put put = new Put(Bytes.toBytes("r1"));
180     put.add(family1, qualifier, value);
181     htable.put(put);
182     ResultScanner resultScanner = htable.getScanner(new Scan());
183     int count = 0;
184     while (resultScanner.next() != null) {
185       count++;
186     }
187     resultScanner.close();
188     assertEquals(1, count);
189 
190     MiniHBaseCluster hbaseCluster = TEST_UTIL.getMiniHBaseCluster();
191     List<HRegion> regions = hbaseCluster.getRegions(tableName);
192     assertEquals(1, regions.size());
193 
194     // move region to another regionserver
195     HRegion destRegion = regions.get(0);
196     int originServerNum = hbaseCluster
197         .getServerWith(destRegion.getRegionName());
198     assertTrue("Please start more than 1 regionserver", hbaseCluster
199         .getRegionServerThreads().size() > 1);
200     int destServerNum = 0;
201     while (destServerNum == originServerNum) {
202       destServerNum++;
203     }
204     HRegionServer originServer = hbaseCluster.getRegionServer(originServerNum);
205     HRegionServer destServer = hbaseCluster.getRegionServer(destServerNum);
206     // move region to destination regionserver
207     moveRegionAndWait(destRegion, destServer);
208 
209     // delete the row
210     Delete del = new Delete(Bytes.toBytes("r1"));
211     htable.delete(del);
212     resultScanner = htable.getScanner(new Scan());
213     count = 0;
214     while (resultScanner.next() != null) {
215       count++;
216     }
217     resultScanner.close();
218     assertEquals(0, count);
219 
220     // flush region and make major compaction
221     destServer.getOnlineRegion(destRegion.getRegionName()).flushcache();
222     // wait to complete major compaction
223     for (Store store : destServer.getOnlineRegion(destRegion.getRegionName())
224         .getStores().values()) {
225       store.triggerMajorCompaction();
226     }
227     destServer.getOnlineRegion(destRegion.getRegionName()).compactStores();
228 
229     // move region to origin regionserver
230     moveRegionAndWait(destRegion, originServer);
231     // abort the origin regionserver
232     originServer.abort("testing");
233 
234     // see what we get
235     Result result = htable.get(new Get(Bytes.toBytes("r1")));
236     if (result != null) {
237       assertTrue("Row is deleted, but we get" + result.toString(),
238           (result == null) || result.isEmpty());
239     }
240     resultScanner.close();
241   }
242 
243   private void moveRegionAndWait(HRegion destRegion, HRegionServer destServer)
244       throws InterruptedException, MasterNotRunningException,
245       ZooKeeperConnectionException, IOException {
246     HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
247     TEST_UTIL.getHBaseAdmin().move(
248         destRegion.getRegionInfo().getEncodedNameAsBytes(),
249         Bytes.toBytes(destServer.getServerName().getServerName()));
250     while (true) {
251       ServerName serverName = master.getAssignmentManager()
252         .getRegionStates().getRegionServerOfRegion(destRegion.getRegionInfo());
253       if (serverName != null && serverName.equals(destServer.getServerName())) {
254         TEST_UTIL.assertRegionOnServer(
255           destRegion.getRegionInfo(), serverName, 200);
256         break;
257       }
258       Thread.sleep(10);
259     }
260   }
261 
262   /**
263    * Tests for hbase-2727.
264    * @throws Exception
265    * @see https://issues.apache.org/jira/browse/HBASE-2727
266    */
267   @Test
268   public void test2727() throws Exception {
269     // Test being able to have > 1 set of edits in the recovered.edits directory.
270     // Ensure edits are replayed properly.
271     final TableName tableName =
272         TableName.valueOf("test2727");
273     HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
274     Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
275     deleteDir(basedir);
276 
277     HTableDescriptor htd = createBasic3FamilyHTD(tableName);
278     HRegion region2 = HRegion.createHRegion(hri,
279         hbaseRootDir, this.conf, htd);
280     HRegion.closeHRegion(region2);
281     final byte [] rowName = tableName.getName();
282 
283     HLog wal1 = createWAL(this.conf);
284     // Add 1k to each family.
285     final int countPerFamily = 1000;
286     final AtomicLong sequenceId = new AtomicLong(1);
287     for (HColumnDescriptor hcd: htd.getFamilies()) {
288       addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily, ee,
289           wal1, htd, sequenceId);
290     }
291     wal1.close();
292     runWALSplit(this.conf);
293 
294     HLog wal2 = createWAL(this.conf);
295     // Add 1k to each family.
296     for (HColumnDescriptor hcd: htd.getFamilies()) {
297       addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily,
298           ee, wal2, htd, sequenceId);
299     }
300     wal2.close();
301     runWALSplit(this.conf);
302 
303     HLog wal3 = createWAL(this.conf);
304     try {
305       HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal3);
306       long seqid = region.getOpenSeqNum();
307       // The regions opens with sequenceId as 1. With 6k edits, its sequence number reaches 6k + 1.
308       // When opened, this region would apply 6k edits, and increment the sequenceId by 1
309       assertTrue(seqid > sequenceId.get());
310       assertEquals(seqid - 1, sequenceId.get());
311       LOG.debug("region.getOpenSeqNum(): " + region.getOpenSeqNum() + ", wal3.id: "
312           + sequenceId.get());
313 
314       // TODO: Scan all.
315       region.close();
316     } finally {
317       wal3.closeAndDelete();
318     }
319   }
320 
321   /**
322    * Test case of HRegion that is only made out of bulk loaded files.  Assert
323    * that we don't 'crash'.
324    * @throws IOException
325    * @throws IllegalAccessException
326    * @throws NoSuchFieldException
327    * @throws IllegalArgumentException
328    * @throws SecurityException
329    */
330   @Test
331   public void testRegionMadeOfBulkLoadedFilesOnly()
332   throws IOException, SecurityException, IllegalArgumentException,
333       NoSuchFieldException, IllegalAccessException, InterruptedException {
334     final TableName tableName =
335         TableName.valueOf("testRegionMadeOfBulkLoadedFilesOnly");
336     final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
337     final Path basedir = new Path(this.hbaseRootDir, tableName.getNameAsString());
338     deleteDir(basedir);
339     final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
340     HRegion region2 = HRegion.createHRegion(hri,
341         hbaseRootDir, this.conf, htd);
342     HRegion.closeHRegion(region2);
343     HLog wal = createWAL(this.conf);
344     HRegion region = HRegion.openHRegion(hri, htd, wal, this.conf);
345 
346     byte [] family = htd.getFamilies().iterator().next().getName();
347     Path f =  new Path(basedir, "hfile");
348     HFileTestUtil.createHFile(this.conf, fs, f, family, family, Bytes.toBytes(""),
349         Bytes.toBytes("z"), 10);
350     List <Pair<byte[],String>>  hfs= new ArrayList<Pair<byte[],String>>(1);
351     hfs.add(Pair.newPair(family, f.toString()));
352     region.bulkLoadHFiles(hfs, true);
353 
354     // Add an edit so something in the WAL
355     byte [] row = tableName.getName();
356     region.put((new Put(row)).add(family, family, family));
357     wal.sync();
358     final int rowsInsertedCount = 11;
359 
360     assertEquals(rowsInsertedCount, getScannedCount(region.getScanner(new Scan())));
361 
362     // Now 'crash' the region by stealing its wal
363     final Configuration newConf = HBaseConfiguration.create(this.conf);
364     User user = HBaseTestingUtility.getDifferentUser(newConf,
365         tableName.getNameAsString());
366     user.runAs(new PrivilegedExceptionAction() {
367       public Object run() throws Exception {
368         runWALSplit(newConf);
369         HLog wal2 = createWAL(newConf);
370 
371         HRegion region2 = HRegion.openHRegion(newConf, FileSystem.get(newConf),
372           hbaseRootDir, hri, htd, wal2);
373         long seqid2 = region2.getOpenSeqNum();
374         assertTrue(seqid2 > -1);
375         assertEquals(rowsInsertedCount, getScannedCount(region2.getScanner(new Scan())));
376 
377         // I can't close wal1.  Its been appropriated when we split.
378         region2.close();
379         wal2.closeAndDelete();
380         return null;
381       }
382     });
383   }
384 
385   /**
386    * HRegion test case that is made of a major compacted HFile (created with three bulk loaded
387    * files) and an edit in the memstore.
388    * This is for HBASE-10958 "[dataloss] Bulk loading with seqids can prevent some log entries
389    * from being replayed"
390    * @throws IOException
391    * @throws IllegalAccessException
392    * @throws NoSuchFieldException
393    * @throws IllegalArgumentException
394    * @throws SecurityException
395    */
396   @Test
397   public void testCompactedBulkLoadedFiles()
398       throws IOException, SecurityException, IllegalArgumentException,
399       NoSuchFieldException, IllegalAccessException, InterruptedException {
400     final TableName tableName =
401         TableName.valueOf("testCompactedBulkLoadedFiles");
402     final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
403     final Path basedir = new Path(this.hbaseRootDir, tableName.getNameAsString());
404     deleteDir(basedir);
405     final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
406     HRegion region2 = HRegion.createHRegion(hri,
407         hbaseRootDir, this.conf, htd);
408     HRegion.closeHRegion(region2);
409     HLog wal = createWAL(this.conf);
410     HRegion region = HRegion.openHRegion(hri, htd, wal, this.conf);
411 
412     // Add an edit so something in the WAL
413     byte [] row = tableName.getName();
414     byte [] family = htd.getFamilies().iterator().next().getName();
415     region.put((new Put(row)).add(family, family, family));
416     wal.sync();
417 
418     List <Pair<byte[],String>>  hfs= new ArrayList<Pair<byte[],String>>(1);
419     for (int i = 0; i < 3; i++) {
420       Path f = new Path(basedir, "hfile"+i);
421       HFileTestUtil.createHFile(this.conf, fs, f, family, family, Bytes.toBytes(i + "00"),
422           Bytes.toBytes(i + "50"), 10);
423       hfs.add(Pair.newPair(family, f.toString()));
424     }
425     region.bulkLoadHFiles(hfs, true);
426     final int rowsInsertedCount = 31;
427     assertEquals(rowsInsertedCount, getScannedCount(region.getScanner(new Scan())));
428 
429     // major compact to turn all the bulk loaded files into one normal file
430     region.compactStores(true);
431     assertEquals(rowsInsertedCount, getScannedCount(region.getScanner(new Scan())));
432 
433     // Now 'crash' the region by stealing its wal
434     final Configuration newConf = HBaseConfiguration.create(this.conf);
435     User user = HBaseTestingUtility.getDifferentUser(newConf,
436         tableName.getNameAsString());
437     user.runAs(new PrivilegedExceptionAction() {
438       public Object run() throws Exception {
439         runWALSplit(newConf);
440         HLog wal2 = createWAL(newConf);
441 
442         HRegion region2 = HRegion.openHRegion(newConf, FileSystem.get(newConf),
443             hbaseRootDir, hri, htd, wal2);
444         long seqid2 = region2.getOpenSeqNum();
445         assertTrue(seqid2 > -1);
446         assertEquals(rowsInsertedCount, getScannedCount(region2.getScanner(new Scan())));
447 
448         // I can't close wal1.  Its been appropriated when we split.
449         region2.close();
450         wal2.closeAndDelete();
451         return null;
452       }
453     });
454   }
455 
456 
457   /**
458    * Test writing edits into an HRegion, closing it, splitting logs, opening
459    * Region again.  Verify seqids.
460    * @throws IOException
461    * @throws IllegalAccessException
462    * @throws NoSuchFieldException
463    * @throws IllegalArgumentException
464    * @throws SecurityException
465    */
466   @Test
467   public void testReplayEditsWrittenViaHRegion()
468   throws IOException, SecurityException, IllegalArgumentException,
469       NoSuchFieldException, IllegalAccessException, InterruptedException {
470     final TableName tableName =
471         TableName.valueOf("testReplayEditsWrittenViaHRegion");
472     final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
473     final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
474     deleteDir(basedir);
475     final byte[] rowName = tableName.getName();
476     final int countPerFamily = 10;
477     final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
478     HRegion region3 = HRegion.createHRegion(hri,
479             hbaseRootDir, this.conf, htd);
480     HRegion.closeHRegion(region3);
481     // Write countPerFamily edits into the three families.  Do a flush on one
482     // of the families during the load of edits so its seqid is not same as
483     // others to test we do right thing when different seqids.
484     HLog wal = createWAL(this.conf);
485     HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
486     long seqid = region.getOpenSeqNum();
487     boolean first = true;
488     for (HColumnDescriptor hcd: htd.getFamilies()) {
489       addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
490       if (first) {
491         // If first, so we have at least one family w/ different seqid to rest.
492         region.flushcache();
493         first = false;
494       }
495     }
496     // Now assert edits made it in.
497     final Get g = new Get(rowName);
498     Result result = region.get(g);
499     assertEquals(countPerFamily * htd.getFamilies().size(),
500       result.size());
501     // Now close the region (without flush), split the log, reopen the region and assert that
502     // replay of log has the correct effect, that our seqids are calculated correctly so
503     // all edits in logs are seen as 'stale'/old.
504     region.close(true);
505     wal.close();
506     runWALSplit(this.conf);
507     HLog wal2 = createWAL(this.conf);
508     HRegion region2 = HRegion.openHRegion(conf, this.fs, hbaseRootDir, hri, htd, wal2);
509     long seqid2 = region2.getOpenSeqNum();
510     assertTrue(seqid + result.size() < seqid2);
511     final Result result1b = region2.get(g);
512     assertEquals(result.size(), result1b.size());
513 
514     // Next test.  Add more edits, then 'crash' this region by stealing its wal
515     // out from under it and assert that replay of the log adds the edits back
516     // correctly when region is opened again.
517     for (HColumnDescriptor hcd: htd.getFamilies()) {
518       addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region2, "y");
519     }
520     // Get count of edits.
521     final Result result2 = region2.get(g);
522     assertEquals(2 * result.size(), result2.size());
523     wal2.sync();
524     // Set down maximum recovery so we dfsclient doesn't linger retrying something
525     // long gone.
526     HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal2).getOutputStream(), 1);
527     final Configuration newConf = HBaseConfiguration.create(this.conf);
528     User user = HBaseTestingUtility.getDifferentUser(newConf,
529       tableName.getNameAsString());
530     user.runAs(new PrivilegedExceptionAction() {
531       public Object run() throws Exception {
532         runWALSplit(newConf);
533         FileSystem newFS = FileSystem.get(newConf);
534         // Make a new wal for new region open.
535         HLog wal3 = createWAL(newConf);
536         final AtomicInteger countOfRestoredEdits = new AtomicInteger(0);
537         HRegion region3 = new HRegion(basedir, wal3, newFS, newConf, hri, htd, null) {
538           @Override
539           protected boolean restoreEdit(Store s, KeyValue kv) {
540             boolean b = super.restoreEdit(s, kv);
541             countOfRestoredEdits.incrementAndGet();
542             return b;
543           }
544         };
545         long seqid3 = region3.initialize();
546         Result result3 = region3.get(g);
547         // Assert that count of cells is same as before crash.
548         assertEquals(result2.size(), result3.size());
549         assertEquals(htd.getFamilies().size() * countPerFamily,
550           countOfRestoredEdits.get());
551 
552         // I can't close wal1.  Its been appropriated when we split.
553         region3.close();
554         wal3.closeAndDelete();
555         return null;
556       }
557     });
558   }
559 
560   /**
561    * Test that we recover correctly when there is a failure in between the
562    * flushes. i.e. Some stores got flushed but others did not.
563    *
564    * Unfortunately, there is no easy hook to flush at a store level. The way
565    * we get around this is by flushing at the region level, and then deleting
566    * the recently flushed store file for one of the Stores. This would put us
567    * back in the situation where all but that store got flushed and the region
568    * died.
569    *
570    * We restart Region again, and verify that the edits were replayed.
571    *
572    * @throws IOException
573    * @throws IllegalAccessException
574    * @throws NoSuchFieldException
575    * @throws IllegalArgumentException
576    * @throws SecurityException
577    */
578   @Test
579   public void testReplayEditsAfterPartialFlush()
580   throws IOException, SecurityException, IllegalArgumentException,
581       NoSuchFieldException, IllegalAccessException, InterruptedException {
582     final TableName tableName =
583         TableName.valueOf("testReplayEditsWrittenViaHRegion");
584     final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
585     final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
586     deleteDir(basedir);
587     final byte[] rowName = tableName.getName();
588     final int countPerFamily = 10;
589     final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
590     HRegion region3 = HRegion.createHRegion(hri,
591             hbaseRootDir, this.conf, htd);
592     HRegion.closeHRegion(region3);
593     // Write countPerFamily edits into the three families.  Do a flush on one
594     // of the families during the load of edits so its seqid is not same as
595     // others to test we do right thing when different seqids.
596     HLog wal = createWAL(this.conf);
597     HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
598     long seqid = region.getOpenSeqNum();
599     for (HColumnDescriptor hcd: htd.getFamilies()) {
600       addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
601     }
602 
603     // Now assert edits made it in.
604     final Get g = new Get(rowName);
605     Result result = region.get(g);
606     assertEquals(countPerFamily * htd.getFamilies().size(),
607       result.size());
608 
609     // Let us flush the region
610     region.flushcache();
611     region.close(true);
612     wal.close();
613 
614     // delete the store files in the second column family to simulate a failure
615     // in between the flushcache();
616     // we have 3 families. killing the middle one ensures that taking the maximum
617     // will make us fail.
618     int cf_count = 0;
619     for (HColumnDescriptor hcd: htd.getFamilies()) {
620       cf_count++;
621       if (cf_count == 2) {
622         region.getRegionFileSystem().deleteFamily(hcd.getNameAsString());
623       }
624     }
625 
626 
627     // Let us try to split and recover
628     runWALSplit(this.conf);
629     HLog wal2 = createWAL(this.conf);
630     HRegion region2 = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal2);
631     long seqid2 = region2.getOpenSeqNum();
632     assertTrue(seqid + result.size() < seqid2);
633 
634     final Result result1b = region2.get(g);
635     assertEquals(result.size(), result1b.size());
636   }
637 
638 
639   // StoreFlusher implementation used in testReplayEditsAfterAbortingFlush.
640   // Only throws exception if throwExceptionWhenFlushing is set true.
641   public static class CustomStoreFlusher extends DefaultStoreFlusher {
642     // Switch between throw and not throw exception in flush
643     static final AtomicBoolean throwExceptionWhenFlushing = new AtomicBoolean(false);
644 
645     public CustomStoreFlusher(Configuration conf, Store store) {
646       super(conf, store);
647     }
648     @Override
649     public List<Path> flushSnapshot(SortedSet<KeyValue> snapshot, long cacheFlushId,
650         TimeRangeTracker snapshotTimeRangeTracker, AtomicLong flushedSize, MonitoredTask status)
651             throws IOException {
652       if (throwExceptionWhenFlushing.get()) {
653         throw new IOException("Simulated exception by tests");
654       }
655       return super.flushSnapshot(snapshot, cacheFlushId, snapshotTimeRangeTracker,
656           flushedSize, status);
657     }
658 
659   };
660 
661   /**
662    * Test that we could recover the data correctly after aborting flush. In the
663    * test, first we abort flush after writing some data, then writing more data
664    * and flush again, at last verify the data.
665    * @throws IOException
666    */
667   @Test
668   public void testReplayEditsAfterAbortingFlush() throws IOException {
669     final TableName tableName =
670         TableName.valueOf("testReplayEditsAfterAbortingFlush");
671     final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
672     final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
673     deleteDir(basedir);
674     final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
675     HRegion region3 = HRegion.createHRegion(hri, hbaseRootDir, this.conf, htd);
676     region3.close();
677     region3.getLog().closeAndDelete();
678     // Write countPerFamily edits into the three families. Do a flush on one
679     // of the families during the load of edits so its seqid is not same as
680     // others to test we do right thing when different seqids.
681     HLog wal = createWAL(this.conf);
682     RegionServerServices rsServices = Mockito.mock(RegionServerServices.class);
683     Mockito.doReturn(false).when(rsServices).isAborted();
684     Configuration customConf = new Configuration(this.conf);
685     customConf.set(DefaultStoreEngine.DEFAULT_STORE_FLUSHER_CLASS_KEY,
686         CustomStoreFlusher.class.getName());
687     HRegion region =
688       HRegion.openHRegion(this.hbaseRootDir, hri, htd, wal, customConf, rsServices, null);
689     int writtenRowCount = 10;
690     List<HColumnDescriptor> families = new ArrayList<HColumnDescriptor>(
691         htd.getFamilies());
692     for (int i = 0; i < writtenRowCount; i++) {
693       Put put = new Put(Bytes.toBytes(tableName + Integer.toString(i)));
694       put.add(families.get(i % families.size()).getName(), Bytes.toBytes("q"),
695           Bytes.toBytes("val"));
696       region.put(put);
697     }
698 
699     // Now assert edits made it in.
700     RegionScanner scanner = region.getScanner(new Scan());
701     assertEquals(writtenRowCount, getScannedCount(scanner));
702 
703     // Let us flush the region
704     CustomStoreFlusher.throwExceptionWhenFlushing.set(true);
705     try {
706       region.flushcache();
707       fail("Injected exception hasn't been thrown");
708     } catch (Throwable t) {
709       LOG.info("Expected simulated exception when flushing region,"
710           + t.getMessage());
711       // simulated to abort server
712       Mockito.doReturn(true).when(rsServices).isAborted();
713       region.setClosing(false); // region normally does not accept writes after
714       // DroppedSnapshotException. We mock around it for this test.
715     }
716     // writing more data
717     int moreRow = 10;
718     for (int i = writtenRowCount; i < writtenRowCount + moreRow; i++) {
719       Put put = new Put(Bytes.toBytes(tableName + Integer.toString(i)));
720       put.add(families.get(i % families.size()).getName(), Bytes.toBytes("q"),
721           Bytes.toBytes("val"));
722       region.put(put);
723     }
724     writtenRowCount += moreRow;
725     // call flush again
726     CustomStoreFlusher.throwExceptionWhenFlushing.set(false);
727     try {
728       region.flushcache();
729     } catch (IOException t) {
730       LOG.info("Expected exception when flushing region because server is stopped,"
731           + t.getMessage());
732     }
733 
734     region.close(true);
735     wal.close();
736 
737     // Let us try to split and recover
738     runWALSplit(this.conf);
739     HLog wal2 = createWAL(this.conf);
740     Mockito.doReturn(false).when(rsServices).isAborted();
741     HRegion region2 =
742       HRegion.openHRegion(this.hbaseRootDir, hri, htd, wal2, this.conf, rsServices, null);
743     scanner = region2.getScanner(new Scan());
744     assertEquals(writtenRowCount, getScannedCount(scanner));
745   }
746 
747   private int getScannedCount(RegionScanner scanner) throws IOException {
748     int scannedCount = 0;
749     List<Cell> results = new ArrayList<Cell>();
750     while (true) {
751       boolean existMore = scanner.next(results);
752       if (!results.isEmpty())
753         scannedCount++;
754       if (!existMore)
755         break;
756       results.clear();
757     }
758     return scannedCount;
759   }
760 
761   /**
762    * Create an HRegion with the result of a HLog split and test we only see the
763    * good edits
764    * @throws Exception
765    */
766   @Test
767   public void testReplayEditsWrittenIntoWAL() throws Exception {
768     final TableName tableName =
769         TableName.valueOf("testReplayEditsWrittenIntoWAL");
770     final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
771     final Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
772     deleteDir(basedir);
773 
774     final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
775     HRegion region2 = HRegion.createHRegion(hri,
776             hbaseRootDir, this.conf, htd);
777     HRegion.closeHRegion(region2);
778     final HLog wal = createWAL(this.conf);
779     final byte[] rowName = tableName.getName();
780     final byte[] regionName = hri.getEncodedNameAsBytes();
781     final AtomicLong sequenceId = new AtomicLong(1);
782 
783     // Add 1k to each family.
784     final int countPerFamily = 1000;
785     for (HColumnDescriptor hcd: htd.getFamilies()) {
786       addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily,
787           ee, wal, htd, sequenceId);
788     }
789 
790     // Add a cache flush, shouldn't have any effect
791     wal.startCacheFlush(regionName);
792     wal.completeCacheFlush(regionName);
793 
794     // Add an edit to another family, should be skipped.
795     WALEdit edit = new WALEdit();
796     long now = ee.currentTimeMillis();
797     edit.add(new KeyValue(rowName, Bytes.toBytes("another family"), rowName,
798       now, rowName));
799     wal.append(hri, tableName, edit, now, htd, sequenceId);
800 
801     // Delete the c family to verify deletes make it over.
802     edit = new WALEdit();
803     now = ee.currentTimeMillis();
804     edit.add(new KeyValue(rowName, Bytes.toBytes("c"), null, now,
805       KeyValue.Type.DeleteFamily));
806     wal.append(hri, tableName, edit, now, htd, sequenceId);
807 
808     // Sync.
809     wal.sync();
810     // Set down maximum recovery so we dfsclient doesn't linger retrying something
811     // long gone.
812     HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal).getOutputStream(), 1);
813     // Make a new conf and a new fs for the splitter to run on so we can take
814     // over old wal.
815     final Configuration newConf = HBaseConfiguration.create(this.conf);
816     User user = HBaseTestingUtility.getDifferentUser(newConf,
817       ".replay.wal.secondtime");
818     user.runAs(new PrivilegedExceptionAction<Void>() {
819       @Override
820       public Void run() throws Exception {
821         runWALSplit(newConf);
822         FileSystem newFS = FileSystem.get(newConf);
823         // 100k seems to make for about 4 flushes during HRegion#initialize.
824         newConf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024 * 100);
825         // Make a new wal for new region.
826         HLog newWal = createWAL(newConf);
827         final AtomicInteger flushcount = new AtomicInteger(0);
828         try {
829           final HRegion region =
830               new HRegion(basedir, newWal, newFS, newConf, hri, htd, null) {
831             protected FlushResult internalFlushcache(
832                 final HLog wal, final long myseqid, MonitoredTask status)
833             throws IOException {
834               LOG.info("InternalFlushCache Invoked");
835               FlushResult fs = super.internalFlushcache(wal, myseqid,
836                   Mockito.mock(MonitoredTask.class));
837               flushcount.incrementAndGet();
838               return fs;
839             };
840           };
841           long seqid = region.initialize();
842           // We flushed during init.
843           assertTrue("Flushcount=" + flushcount.get(), flushcount.get() > 0);
844           assertTrue(seqid - 1 == sequenceId.get());
845 
846           Get get = new Get(rowName);
847           Result result = region.get(get);
848           // Make sure we only see the good edits
849           assertEquals(countPerFamily * (htd.getFamilies().size() - 1),
850             result.size());
851           region.close();
852         } finally {
853           newWal.closeAndDelete();
854         }
855         return null;
856       }
857     });
858   }
859 
860   @Test
861   // the following test is for HBASE-6065
862   public void testSequentialEditLogSeqNum() throws IOException {
863     final TableName tableName =
864         TableName.valueOf("testSequentialEditLogSeqNum");
865     final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
866     final Path basedir =
867         FSUtils.getTableDir(this.hbaseRootDir, tableName);
868     deleteDir(basedir);
869     final byte[] rowName = tableName.getName();
870     final int countPerFamily = 10;
871     final HTableDescriptor htd = createBasic1FamilyHTD(tableName);
872 
873     // Mock the HLog
874     MockHLog wal = createMockWAL(this.conf);
875 
876     HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
877     for (HColumnDescriptor hcd : htd.getFamilies()) {
878       addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
879     }
880 
881     // Let us flush the region
882     // But this time completeflushcache is not yet done
883     region.flushcache();
884     for (HColumnDescriptor hcd : htd.getFamilies()) {
885       addRegionEdits(rowName, hcd.getName(), 5, this.ee, region, "x");
886     }
887     long lastestSeqNumber = region.getSequenceId().get();
888     // get the current seq no
889     wal.doCompleteCacheFlush = true;
890     // allow complete cache flush with the previous seq number got after first
891     // set of edits.
892     wal.completeCacheFlush(hri.getEncodedNameAsBytes());
893     wal.close();
894     FileStatus[] listStatus = this.fs.listStatus(wal.getDir());
895     HLogSplitter.splitLogFile(hbaseRootDir, listStatus[0],
896       this.fs, this.conf, null, null, null, mode);
897     FileStatus[] listStatus1 = this.fs.listStatus(
898         new Path(FSUtils.getTableDir(hbaseRootDir, tableName),
899             new Path(hri.getEncodedName(), "recovered.edits")));
900     int editCount = 0;
901     for (FileStatus fileStatus : listStatus1) {
902       editCount = Integer.parseInt(fileStatus.getPath().getName());
903     }
904     // The sequence number should be same 
905     assertEquals(
906         "The sequence number of the recoverd.edits and the current edit seq should be same",
907         lastestSeqNumber, editCount);
908   }
909 
910   /**
911    * testcase for https://issues.apache.org/jira/browse/HBASE-15252
912    */
913   @Test
914   public void testDatalossWhenInputError() throws IOException, InstantiationException,
915       IllegalAccessException {
916     final TableName tableName = TableName.valueOf("testDatalossWhenInputError");
917     final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
918     final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
919     deleteDir(basedir);
920     final byte[] rowName = tableName.getName();
921     final int countPerFamily = 10;
922     final HTableDescriptor htd = createBasic1FamilyHTD(tableName);
923     HRegion region1 = HRegion.createHRegion(hri,
924       hbaseRootDir, this.conf, htd);
925     Path regionDir = region1.getRegionFileSystem().getRegionDir();
926     HRegion.closeHRegion(region1);
927 
928     HLog wal = createWAL(this.conf);
929     HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
930     for (HColumnDescriptor hcd : htd.getFamilies()) {
931       addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
932     }
933     // Now assert edits made it in.
934     final Get g = new Get(rowName);
935     Result result = region.get(g);
936     assertEquals(countPerFamily * htd.getFamilies().size(), result.size());
937     // Now close the region (without flush), split the log, reopen the region and assert that
938     // replay of log has the correct effect.
939     region.close(true);
940     wal.close();
941 
942     runWALSplit(this.conf);
943 
944     // here we let the DFSInputStream throw an IOException just after the WALHeader.
945     Path editFile = HLogUtil.getSplitEditFilesSorted(this.fs, regionDir).first();
946     FSDataInputStream stream = fs.open(editFile);
947     stream.seek(ProtobufLogReader.PB_WAL_MAGIC.length);
948     Class<? extends HLog.Reader> logReaderClass =
949         conf.getClass("hbase.regionserver.hlog.reader.impl", ProtobufLogReader.class,
950           HLog.Reader.class);
951     HLog.Reader reader = logReaderClass.newInstance();
952     reader.init(this.fs, editFile, conf, stream);
953     final long headerLength = stream.getPos();
954     reader.close();
955     FileSystem spyFs = spy(this.fs);
956     doAnswer(new Answer<FSDataInputStream>() {
957 
958       @Override
959       public FSDataInputStream answer(InvocationOnMock invocation) throws Throwable {
960         FSDataInputStream stream = (FSDataInputStream) invocation.callRealMethod();
961         Field field = FilterInputStream.class.getDeclaredField("in");
962         field.setAccessible(true);
963         final InputStream in = (InputStream) field.get(stream);
964         InputStream spyIn = spy(in);
965         doAnswer(new Answer<Integer>() {
966 
967           private long pos;
968 
969           @Override
970           public Integer answer(InvocationOnMock invocation) throws Throwable {
971             if (pos >= headerLength) {
972               throw new IOException("read over limit");
973             }
974             int b = (Integer) invocation.callRealMethod();
975             if (b > 0) {
976               pos += b;
977             }
978             return b;
979           }
980         }).when(spyIn).read(any(byte[].class), any(int.class), any(int.class));
981         doAnswer(new Answer<Void>() {
982 
983           @Override
984           public Void answer(InvocationOnMock invocation) throws Throwable {
985             invocation.callRealMethod();
986             in.close();
987             return null;
988           }
989         }).when(spyIn).close();
990         field.set(stream, spyIn);
991         return stream;
992       }
993     }).when(spyFs).open(eq(editFile));
994 
995     HLog wal2 = createWAL(this.conf);
996     HRegion region2;
997     try {
998       // log replay should fail due to the IOException, otherwise we may lose data.
999       region2 = HRegion.openHRegion(conf, spyFs, hbaseRootDir, hri, htd, wal2);
1000       assertEquals(result.size(), region2.get(g).size());
1001     } catch (IOException e) {
1002       assertEquals("read over limit", e.getMessage());
1003     }
1004     region2 = HRegion.openHRegion(conf, fs, hbaseRootDir, hri, htd, wal2);
1005     assertEquals(result.size(), region2.get(g).size());
1006   }
1007 
1008   static class MockHLog extends FSHLog {
1009     boolean doCompleteCacheFlush = false;
1010 
1011     public MockHLog(FileSystem fs, Path rootDir, String logName, Configuration conf) throws IOException {
1012       super(fs, rootDir, logName, conf);
1013     }
1014 
1015     @Override
1016     public void completeCacheFlush(byte[] encodedRegionName) {
1017       if (!doCompleteCacheFlush) {
1018         return;
1019       }
1020       super.completeCacheFlush(encodedRegionName);
1021     }
1022   }
1023 
1024   private HTableDescriptor createBasic1FamilyHTD(final TableName tableName) {
1025     HTableDescriptor htd = new HTableDescriptor(tableName);
1026     HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a"));
1027     htd.addFamily(a);
1028     return htd;
1029   }
1030   
1031   private MockHLog createMockWAL(Configuration conf) throws IOException {
1032     MockHLog wal = new MockHLog(FileSystem.get(conf), hbaseRootDir, logName, conf);
1033     // Set down maximum recovery so we dfsclient doesn't linger retrying something
1034     // long gone.
1035     HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal).getOutputStream(), 1);
1036     return wal;
1037   }
1038 
1039   // Flusher used in this test.  Keep count of how often we are called and
1040   // actually run the flush inside here.
1041   class TestFlusher implements FlushRequester {
1042     private HRegion r;
1043 
1044     @Override
1045     public void requestFlush(HRegion region) {
1046       try {
1047         r.flushcache();
1048       } catch (IOException e) {
1049         throw new RuntimeException("Exception flushing", e);
1050       }
1051     }
1052 
1053     @Override
1054     public void requestDelayedFlush(HRegion region, long when) {
1055       // TODO Auto-generated method stub
1056       
1057     }
1058   }
1059 
1060   private void addWALEdits(final TableName tableName, final HRegionInfo hri, final byte[] rowName,
1061       final byte[] family, final int count, EnvironmentEdge ee, final HLog wal,
1062       final HTableDescriptor htd, final AtomicLong sequenceId)
1063   throws IOException {
1064     String familyStr = Bytes.toString(family);
1065     for (int j = 0; j < count; j++) {
1066       byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j));
1067       byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j));
1068       WALEdit edit = new WALEdit();
1069       edit.add(new KeyValue(rowName, family, qualifierBytes,
1070         ee.currentTimeMillis(), columnBytes));
1071       wal.append(hri, tableName, edit, ee.currentTimeMillis(), htd, sequenceId);
1072     }
1073   }
1074 
1075   private void addRegionEdits (final byte [] rowName, final byte [] family,
1076       final int count, EnvironmentEdge ee, final HRegion r,
1077       final String qualifierPrefix)
1078   throws IOException {
1079     for (int j = 0; j < count; j++) {
1080       byte[] qualifier = Bytes.toBytes(qualifierPrefix + Integer.toString(j));
1081       Put p = new Put(rowName);
1082       p.add(family, qualifier, ee.currentTimeMillis(), rowName);
1083       r.put(p);
1084     }
1085   }
1086 
1087   /*
1088    * Creates an HRI around an HTD that has <code>tableName</code> and three
1089    * column families named 'a','b', and 'c'.
1090    * @param tableName Name of table to use when we create HTableDescriptor.
1091    */
1092    private HRegionInfo createBasic3FamilyHRegionInfo(final TableName tableName) {
1093     return new HRegionInfo(tableName, null, null, false);
1094    }
1095 
1096   /*
1097    * Run the split.  Verify only single split file made.
1098    * @param c
1099    * @return The single split file made
1100    * @throws IOException
1101    */
1102   private Path runWALSplit(final Configuration c) throws IOException {
1103     List<Path> splits = HLogSplitter.split(
1104       hbaseRootDir, logDir, oldLogDir, FileSystem.get(c), c);
1105     // Split should generate only 1 file since there's only 1 region
1106     assertEquals("splits=" + splits, 1, splits.size());
1107     // Make sure the file exists
1108     assertTrue(fs.exists(splits.get(0)));
1109     LOG.info("Split file=" + splits.get(0));
1110     return splits.get(0);
1111   }
1112 
1113   /*
1114    * @param c
1115    * @return WAL with retries set down from 5 to 1 only.
1116    * @throws IOException
1117    */
1118   private HLog createWAL(final Configuration c) throws IOException {
1119     HLog wal = HLogFactory.createHLog(FileSystem.get(c), 
1120         hbaseRootDir, logName, c);
1121     // Set down maximum recovery so we dfsclient doesn't linger retrying something
1122     // long gone.
1123     HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal).getOutputStream(), 1);
1124     return wal;
1125   }
1126 
1127   private HTableDescriptor createBasic3FamilyHTD(final TableName tableName) {
1128     HTableDescriptor htd = new HTableDescriptor(tableName);
1129     HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a"));
1130     htd.addFamily(a);
1131     HColumnDescriptor b = new HColumnDescriptor(Bytes.toBytes("b"));
1132     htd.addFamily(b);
1133     HColumnDescriptor c = new HColumnDescriptor(Bytes.toBytes("c"));
1134     htd.addFamily(c);
1135     return htd;
1136   }
1137 
1138 }
1139