1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver.wal;
21
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertTrue;
24
25 import java.io.IOException;
26 import java.util.List;
27 import java.util.concurrent.atomic.AtomicInteger;
28
29 import org.apache.commons.logging.Log;
30 import org.apache.commons.logging.LogFactory;
31 import org.apache.hadoop.conf.Configuration;
32 import org.apache.hadoop.fs.FileSystem;
33 import org.apache.hadoop.fs.Path;
34 import org.apache.hadoop.hbase.HBaseConfiguration;
35 import org.apache.hadoop.hbase.HBaseTestingUtility;
36 import org.apache.hadoop.hbase.HColumnDescriptor;
37 import org.apache.hadoop.hbase.HConstants;
38 import org.apache.hadoop.hbase.HRegionInfo;
39 import org.apache.hadoop.hbase.HTableDescriptor;
40 import org.apache.hadoop.hbase.KeyValue;
41 import org.apache.hadoop.hbase.client.Get;
42 import org.apache.hadoop.hbase.client.Put;
43 import org.apache.hadoop.hbase.client.Result;
44 import org.apache.hadoop.hbase.io.hfile.HFile;
45 import org.apache.hadoop.hbase.regionserver.HRegion;
46 import org.apache.hadoop.hbase.regionserver.Store;
47 import org.apache.hadoop.hbase.util.Bytes;
48 import org.apache.hadoop.hbase.util.EnvironmentEdge;
49 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
50 import org.junit.After;
51 import org.junit.AfterClass;
52 import org.junit.Before;
53 import org.junit.BeforeClass;
54 import org.junit.Test;
55
56
57
58
59 public class TestWALReplay {
60 public static final Log LOG = LogFactory.getLog(TestWALReplay.class);
61 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
62 private final EnvironmentEdge ee = EnvironmentEdgeManager.getDelegate();
63 private Path hbaseRootDir = null;
64 private Path oldLogDir;
65 private Path logDir;
66 private FileSystem fs;
67 private Configuration conf;
68
69 @BeforeClass
70 public static void setUpBeforeClass() throws Exception {
71 Configuration conf = TEST_UTIL.getConfiguration();
72 conf.setBoolean("dfs.support.append", true);
73
74 conf.setInt("dfs.client.block.recovery.retries", 2);
75 conf.setInt("hbase.regionserver.flushlogentries", 1);
76 TEST_UTIL.startMiniDFSCluster(3);
77 TEST_UTIL.setNameNodeNameSystemLeasePeriod(100, 10000);
78 Path hbaseRootDir =
79 TEST_UTIL.getDFSCluster().getFileSystem().makeQualified(new Path("/hbase"));
80 LOG.info("hbase.rootdir=" + hbaseRootDir);
81 conf.set(HConstants.HBASE_DIR, hbaseRootDir.toString());
82 }
83
84 @AfterClass
85 public static void tearDownAfterClass() throws Exception {
86 TEST_UTIL.shutdownMiniDFSCluster();
87 }
88
89 @Before
90 public void setUp() throws Exception {
91 this.conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
92 this.fs = TEST_UTIL.getDFSCluster().getFileSystem();
93 this.hbaseRootDir = new Path(this.conf.get(HConstants.HBASE_DIR));
94 this.oldLogDir = new Path(this.hbaseRootDir, HConstants.HREGION_OLDLOGDIR_NAME);
95 this.logDir = new Path(this.hbaseRootDir, HConstants.HREGION_LOGDIR_NAME);
96 if (TEST_UTIL.getDFSCluster().getFileSystem().exists(this.hbaseRootDir)) {
97 TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
98 }
99 }
100
101 @After
102 public void tearDown() throws Exception {
103 TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
104 }
105
106
107
108
109 private void deleteDir(final Path p) throws IOException {
110 if (this.fs.exists(p)) {
111 if (!this.fs.delete(p, true)) {
112 throw new IOException("Failed remove of " + p);
113 }
114 }
115 }
116
117
118
119
120
121
122 @Test
123 public void test2727() throws Exception {
124
125
126 final String tableNameStr = "test2727";
127 HRegionInfo hri = createBasic3FamilyHRegionInfo(tableNameStr);
128 Path basedir = new Path(hbaseRootDir, tableNameStr);
129 deleteDir(basedir);
130
131 final byte [] tableName = Bytes.toBytes(tableNameStr);
132 final byte [] rowName = tableName;
133
134 HLog wal1 = createWAL(this.conf);
135
136 final int countPerFamily = 1000;
137 for (HColumnDescriptor hcd: hri.getTableDesc().getFamilies()) {
138 addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily, ee, wal1);
139 }
140 wal1.close();
141 runWALSplit(this.conf);
142
143 HLog wal2 = createWAL(this.conf);
144
145 wal2.setSequenceNumber(wal1.getSequenceNumber());
146
147 for (HColumnDescriptor hcd: hri.getTableDesc().getFamilies()) {
148 addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily, ee, wal2);
149 }
150 wal2.close();
151 runWALSplit(this.conf);
152
153 HLog wal3 = createWAL(this.conf);
154 wal3.setSequenceNumber(wal2.getSequenceNumber());
155 try {
156 final HRegion region = new HRegion(basedir, wal3, this.fs, this.conf, hri,
157 null);
158 long seqid = region.initialize();
159 assertTrue(seqid > wal3.getSequenceNumber());
160
161
162 region.close();
163 } finally {
164 wal3.closeAndDelete();
165 }
166 }
167
168
169
170
171
172
173
174
175
176
177 @Test
178 public void testRegionMadeOfBulkLoadedFilesOnly()
179 throws IOException, SecurityException, IllegalArgumentException,
180 NoSuchFieldException, IllegalAccessException {
181 final String tableNameStr = "testReplayEditsWrittenViaHRegion";
182 HRegionInfo hri = createBasic3FamilyHRegionInfo(tableNameStr);
183 Path basedir = new Path(this.hbaseRootDir, tableNameStr);
184 deleteDir(basedir);
185 HLog wal = createWAL(this.conf);
186 HRegion region = HRegion.openHRegion(hri, basedir, wal, this.conf);
187 Path f = new Path(basedir, "hfile");
188 HFile.Writer writer = new HFile.Writer(this.fs, f);
189 byte [] family = hri.getTableDesc().getFamilies().iterator().next().getName();
190 byte [] row = Bytes.toBytes(tableNameStr);
191 writer.append(new KeyValue(row, family, family, row));
192 writer.close();
193 region.bulkLoadHFile(f.toString(), family);
194
195 region.put((new Put(row)).add(family, family, family));
196 wal.sync();
197
198
199 Configuration newConf = HBaseTestingUtility.setDifferentUser(this.conf,
200 tableNameStr);
201 runWALSplit(newConf);
202 HLog wal2 = createWAL(newConf);
203 HRegion region2 = new HRegion(basedir, wal2, FileSystem.get(newConf),
204 newConf, hri, null);
205 long seqid2 = region2.initialize();
206 assertTrue(seqid2 > -1);
207
208
209 region2.close();
210 wal2.closeAndDelete();
211 }
212
213
214
215
216
217
218
219
220
221
222 @Test
223 public void testReplayEditsWrittenViaHRegion()
224 throws IOException, SecurityException, IllegalArgumentException,
225 NoSuchFieldException, IllegalAccessException {
226 final String tableNameStr = "testReplayEditsWrittenViaHRegion";
227 HRegionInfo hri = createBasic3FamilyHRegionInfo(tableNameStr);
228 Path basedir = new Path(this.hbaseRootDir, tableNameStr);
229 deleteDir(basedir);
230 final byte[] rowName = Bytes.toBytes(tableNameStr);
231 final int countPerFamily = 10;
232
233
234
235
236 HLog wal = createWAL(this.conf);
237 HRegion region = new HRegion(basedir, wal, this.fs, this.conf, hri, null);
238 long seqid = region.initialize();
239
240 wal.setSequenceNumber(seqid);
241 boolean first = true;
242 for (HColumnDescriptor hcd: hri.getTableDesc().getFamilies()) {
243 addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
244 if (first ) {
245
246 region.flushcache();
247 first = false;
248 }
249 }
250
251 Get g = new Get(rowName);
252 Result result = region.get(g, null);
253 assertEquals(countPerFamily * hri.getTableDesc().getFamilies().size(),
254 result.size());
255
256
257
258 region.close();
259 wal.close();
260 runWALSplit(this.conf);
261 HLog wal2 = createWAL(this.conf);
262 HRegion region2 = new HRegion(basedir, wal2, this.fs, this.conf, hri, null) {
263 @Override
264 protected boolean restoreEdit(Store s, KeyValue kv) {
265 super.restoreEdit(s, kv);
266 throw new RuntimeException("Called when it should not have been!");
267 }
268 };
269 long seqid2 = region2.initialize();
270
271 wal2.setSequenceNumber(seqid2);
272 assertTrue(seqid + result.size() < seqid2);
273
274
275
276
277 for (HColumnDescriptor hcd: hri.getTableDesc().getFamilies()) {
278 addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region2, "y");
279 }
280
281 Result result2 = region2.get(g, null);
282 assertEquals(2 * result.size(), result2.size());
283 wal2.sync();
284
285
286 HBaseTestingUtility.setMaxRecoveryErrorCount(wal2.getOutputStream(), 1);
287 Configuration newConf = HBaseTestingUtility.setDifferentUser(this.conf,
288 tableNameStr);
289 runWALSplit(newConf);
290 FileSystem newFS = FileSystem.get(newConf);
291
292 HLog wal3 = createWAL(newConf);
293 final AtomicInteger countOfRestoredEdits = new AtomicInteger(0);
294 HRegion region3 = new HRegion(basedir, wal3, newFS, newConf, hri, null) {
295 @Override
296 protected boolean restoreEdit(Store s, KeyValue kv) {
297 boolean b = super.restoreEdit(s, kv);
298 countOfRestoredEdits.incrementAndGet();
299 return b;
300 }
301 };
302 long seqid3 = region3.initialize();
303
304 wal3.setSequenceNumber(seqid3);
305 Result result3 = region3.get(g, null);
306
307 assertEquals(result2.size(), result3.size());
308 assertEquals(hri.getTableDesc().getFamilies().size() * countPerFamily,
309 countOfRestoredEdits.get());
310
311
312 region3.close();
313 wal3.closeAndDelete();
314 }
315
316
317
318
319
320
321 @Test
322 public void testReplayEditsWrittenIntoWAL() throws Exception {
323 final String tableNameStr = "testReplayEditsWrittenIntoWAL";
324 HRegionInfo hri = createBasic3FamilyHRegionInfo(tableNameStr);
325 Path basedir = new Path(hbaseRootDir, tableNameStr);
326 deleteDir(basedir);
327 HLog wal = createWAL(this.conf);
328 final byte[] tableName = Bytes.toBytes(tableNameStr);
329 final byte[] rowName = tableName;
330 final byte[] regionName = hri.getRegionName();
331
332
333 final int countPerFamily = 1000;
334 for (HColumnDescriptor hcd: hri.getTableDesc().getFamilies()) {
335 addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily, ee, wal);
336 }
337
338
339 long logSeqId = wal.startCacheFlush();
340 wal.completeCacheFlush(regionName, tableName, logSeqId, hri.isMetaRegion());
341
342
343 WALEdit edit = new WALEdit();
344 long now = ee.currentTimeMillis();
345 edit.add(new KeyValue(rowName, Bytes.toBytes("another family"), rowName,
346 now, rowName));
347 wal.append(hri, tableName, edit, now);
348
349
350 edit = new WALEdit();
351 now = ee.currentTimeMillis();
352 edit.add(new KeyValue(rowName, Bytes.toBytes("c"), null, now,
353 KeyValue.Type.DeleteFamily));
354 wal.append(hri, tableName, edit, now);
355
356
357 wal.sync();
358
359
360 HBaseTestingUtility.setMaxRecoveryErrorCount(wal.getOutputStream(), 1);
361
362
363
364 Configuration newConf = HBaseTestingUtility.setDifferentUser(this.conf,
365 ".replay.wal.secondtime");
366 runWALSplit(newConf);
367 FileSystem newFS = FileSystem.get(newConf);
368
369 newConf.setInt("hbase.hregion.memstore.flush.size", 1024 * 100);
370
371 HLog newWal = createWAL(newConf);
372 final AtomicInteger flushcount = new AtomicInteger(0);
373 try {
374 final HRegion region = new HRegion(basedir, newWal, newFS, newConf, hri,
375 null) {
376 protected boolean internalFlushcache(HLog wal, long myseqid)
377 throws IOException {
378 boolean b = super.internalFlushcache(wal, myseqid);
379 flushcount.incrementAndGet();
380 return b;
381 };
382 };
383 long seqid = region.initialize();
384
385 assertTrue(flushcount.get() > 0);
386 assertTrue(seqid > wal.getSequenceNumber());
387
388 Get get = new Get(rowName);
389 Result result = region.get(get, -1);
390
391 assertEquals(countPerFamily * (hri.getTableDesc().getFamilies().size() - 1),
392 result.size());
393 region.close();
394 } finally {
395 newWal.closeAndDelete();
396 }
397 }
398
399 private void addWALEdits (final byte [] tableName, final HRegionInfo hri,
400 final byte [] rowName, final byte [] family,
401 final int count, EnvironmentEdge ee, final HLog wal)
402 throws IOException {
403 String familyStr = Bytes.toString(family);
404 for (int j = 0; j < count; j++) {
405 byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j));
406 byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j));
407 WALEdit edit = new WALEdit();
408 edit.add(new KeyValue(rowName, family, qualifierBytes,
409 ee.currentTimeMillis(), columnBytes));
410 wal.append(hri, tableName, edit, ee.currentTimeMillis());
411 }
412 }
413
414 private void addRegionEdits (final byte [] rowName, final byte [] family,
415 final int count, EnvironmentEdge ee, final HRegion r,
416 final String qualifierPrefix)
417 throws IOException {
418 for (int j = 0; j < count; j++) {
419 byte[] qualifier = Bytes.toBytes(qualifierPrefix + Integer.toString(j));
420 Put p = new Put(rowName);
421 p.add(family, qualifier, ee.currentTimeMillis(), rowName);
422 r.put(p);
423 }
424 }
425
426
427
428
429
430
431 private HRegionInfo createBasic3FamilyHRegionInfo(final String tableName) {
432 HTableDescriptor htd = new HTableDescriptor(tableName);
433 HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a"));
434 htd.addFamily(a);
435 HColumnDescriptor b = new HColumnDescriptor(Bytes.toBytes("b"));
436 htd.addFamily(b);
437 HColumnDescriptor c = new HColumnDescriptor(Bytes.toBytes("c"));
438 htd.addFamily(c);
439 return new HRegionInfo(htd, null, null, false);
440 }
441
442
443
444
445
446
447
448
449 private Path runWALSplit(final Configuration c) throws IOException {
450 FileSystem fs = FileSystem.get(c);
451 List<Path> splits = HLog.splitLog(this.hbaseRootDir, this.logDir,
452 this.oldLogDir, fs, c);
453
454 assertEquals(1, splits.size());
455
456 assertTrue(fs.exists(splits.get(0)));
457 LOG.info("Split file=" + splits.get(0));
458 return splits.get(0);
459 }
460
461
462
463
464
465
466 private HLog createWAL(final Configuration c) throws IOException {
467 HLog wal = new HLog(FileSystem.get(c), logDir, oldLogDir, c, null);
468
469
470 HBaseTestingUtility.setMaxRecoveryErrorCount(wal.getOutputStream(), 1);
471 return wal;
472 }
473 }