1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.regionserver;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertTrue;
22
23 import java.io.IOException;
24 import java.util.List;
25
26 import org.apache.hadoop.conf.Configuration;
27 import org.apache.hadoop.fs.FileSystem;
28 import org.apache.hadoop.fs.Path;
29 import org.apache.hadoop.hbase.Cell;
30 import org.apache.hadoop.hbase.CellComparator;
31 import org.apache.hadoop.hbase.CellScanner;
32 import org.apache.hadoop.hbase.CellUtil;
33 import org.apache.hadoop.hbase.HBaseTestingUtility;
34 import org.apache.hadoop.hbase.HColumnDescriptor;
35 import org.apache.hadoop.hbase.HConstants;
36 import org.apache.hadoop.hbase.HRegionInfo;
37 import org.apache.hadoop.hbase.HTableDescriptor;
38 import org.apache.hadoop.hbase.TableName;
39 import org.apache.hadoop.hbase.client.Get;
40 import org.apache.hadoop.hbase.client.Result;
41 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
42 import org.apache.hadoop.hbase.testclassification.MediumTests;
43 import org.apache.hadoop.hbase.util.Bytes;
44 import org.apache.hadoop.hbase.wal.WAL;
45 import org.apache.hadoop.hbase.wal.WALFactory;
46 import org.apache.hadoop.hbase.wal.WALKey;
47 import org.apache.hadoop.hbase.wal.WALSplitter;
48 import org.junit.Rule;
49 import org.junit.Test;
50 import org.junit.experimental.categories.Category;
51 import org.junit.rules.TestName;
52 import org.mortbay.log.Log;
53
54
55
56
57 @Category({MediumTests.class})
58 public class TestRecoveredEdits {
59 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
60 @Rule public TestName testName = new TestName();
61
62
63
64
65
66
67
68
69 @Test (timeout=30000)
70 public void testReplayWorksThoughLotsOfFlushing() throws IOException {
71 Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
72
73 conf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024*1024);
74
75
76 final String encodedRegionName = "4823016d8fca70b25503ee07f4c6d79f";
77 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(testName.getMethodName()));
78 final String columnFamily = "meta";
79 byte [][] columnFamilyAsByteArray = new byte [][] {Bytes.toBytes(columnFamily)};
80 htd.addFamily(new HColumnDescriptor(columnFamily));
81 HRegionInfo hri = new HRegionInfo(htd.getTableName()) {
82 @Override
83 public synchronized String getEncodedName() {
84 return encodedRegionName;
85 }
86
87
88 private byte [] encodedRegionNameAsBytes = null;
89 @Override
90 public synchronized byte[] getEncodedNameAsBytes() {
91 if (encodedRegionNameAsBytes == null) {
92 this.encodedRegionNameAsBytes = Bytes.toBytes(getEncodedName());
93 }
94 return this.encodedRegionNameAsBytes;
95 }
96 };
97 Path hbaseRootDir = TEST_UTIL.getDataTestDir();
98 HRegion region = HRegion.createHRegion(hri, hbaseRootDir, conf, htd, null);
99 assertEquals(encodedRegionName, region.getRegionInfo().getEncodedName());
100 List<String> storeFiles = region.getStoreFileList(columnFamilyAsByteArray);
101
102 assertTrue(storeFiles.isEmpty());
103 region.close();
104 Path regionDir = region.getRegionDir(hbaseRootDir, hri);
105 Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regionDir);
106
107 Path recoveredEditsFile = new Path(new Path(
108 System.getProperty("project.build.testSourceDirectory", "src" + Path.SEPARATOR + "test"),
109 "data"), "0000000000000016310");
110
111 FileSystem fs = FileSystem.get(conf);
112 Path destination = new Path(recoveredEditsDir, recoveredEditsFile.getName());
113 fs.copyToLocalFile(recoveredEditsFile, destination);
114 assertTrue(fs.exists(destination));
115
116 region = HRegion.openHRegion(region, null);
117 assertEquals(encodedRegionName, region.getRegionInfo().getEncodedName());
118 storeFiles = region.getStoreFileList(columnFamilyAsByteArray);
119
120
121
122 assertTrue("Files count=" + storeFiles.size(), storeFiles.size() > 10);
123
124 int count = verifyAllEditsMadeItIn(fs, conf, recoveredEditsFile, region);
125 Log.info("Checked " + count + " edits made it in");
126 }
127
128
129
130
131
132
133
134
135
136 private int verifyAllEditsMadeItIn(final FileSystem fs, final Configuration conf,
137 final Path edits, final HRegion region)
138 throws IOException {
139 int count = 0;
140
141 WAL.Reader reader = null;
142 try {
143 reader = WALFactory.createReader(fs, edits, conf);
144 WAL.Entry entry;
145 while ((entry = reader.next()) != null) {
146 WALKey key = entry.getKey();
147 WALEdit val = entry.getEdit();
148 count++;
149
150 if (!Bytes.equals(key.getEncodedRegionName(),
151 region.getRegionInfo().getEncodedNameAsBytes())) {
152 continue;
153 }
154 Cell previous = null;
155 for (Cell cell: val.getCells()) {
156 if (CellUtil.matchingFamily(cell, WALEdit.METAFAMILY)) continue;
157 if (previous != null && CellComparator.compareRows(previous, cell) == 0) continue;
158 previous = cell;
159 Get g = new Get(CellUtil.cloneRow(cell));
160 Result r = region.get(g);
161 boolean found = false;
162 for (CellScanner scanner = r.cellScanner(); scanner.advance();) {
163 Cell current = scanner.current();
164 if (CellComparator.compare(cell, current, true) == 0) {
165 found = true;
166 break;
167 }
168 }
169 assertTrue("Failed to find " + cell, found);
170 }
171 }
172 } finally {
173 if (reader != null) reader.close();
174 }
175 return count;
176 }
177 }