1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.util;
21
22 import java.io.IOException;
23 import java.util.ArrayList;
24 import java.util.List;
25
26 import org.apache.commons.logging.Log;
27 import org.apache.commons.logging.LogFactory;
28 import org.apache.hadoop.fs.Path;
29 import org.apache.hadoop.hbase.Cell;
30 import org.apache.hadoop.hbase.CellUtil;
31 import org.apache.hadoop.hbase.HBaseTestCase;
32 import org.apache.hadoop.hbase.HBaseTestingUtility;
33 import org.apache.hadoop.hbase.HColumnDescriptor;
34 import org.apache.hadoop.hbase.HConstants;
35 import org.apache.hadoop.hbase.HRegionInfo;
36 import org.apache.hadoop.hbase.HTableDescriptor;
37 import org.apache.hadoop.hbase.LargeTests;
38 import org.apache.hadoop.hbase.TableName;
39 import org.apache.hadoop.hbase.client.Get;
40 import org.apache.hadoop.hbase.client.Put;
41 import org.apache.hadoop.hbase.client.Result;
42 import org.apache.hadoop.hbase.client.Scan;
43 import org.apache.hadoop.hbase.regionserver.HRegion;
44 import org.apache.hadoop.hbase.regionserver.InternalScanner;
45 import org.apache.hadoop.hbase.regionserver.wal.HLog;
46 import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
47 import org.apache.hadoop.hdfs.MiniDFSCluster;
48 import org.apache.hadoop.util.ToolRunner;
49 import org.junit.experimental.categories.Category;
50
51
52 @Category(LargeTests.class)
53 public class TestMergeTool extends HBaseTestCase {
54 static final Log LOG = LogFactory.getLog(TestMergeTool.class);
55 HBaseTestingUtility TEST_UTIL;
56
57 static final byte [] FAMILY = Bytes.toBytes("contents");
58 static final byte [] QUALIFIER = Bytes.toBytes("dc");
59
60 private final HRegionInfo[] sourceRegions = new HRegionInfo[5];
61 private final HRegion[] regions = new HRegion[5];
62 private HTableDescriptor desc;
63 private byte [][][] rows;
64 private MiniDFSCluster dfsCluster = null;
65
66 @Override
67 public void setUp() throws Exception {
68
69 this.conf.setLong("hbase.zookeeper.recoverable.waittime", 10);
70
71
72
73
74 this.conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, 10001);
75
76 this.conf.set("hbase.hstore.compactionThreshold", "2");
77
78
79 this.desc = new HTableDescriptor(TableName.valueOf("TestMergeTool"));
80 this.desc.addFamily(new HColumnDescriptor(FAMILY));
81
82
83
84
85
86 sourceRegions[0] = new HRegionInfo(this.desc.getTableName(),
87 Bytes.toBytes("row_0200"),
88 Bytes.toBytes("row_0300"));
89
90
91
92 sourceRegions[1] =
93 new HRegionInfo(this.desc.getTableName(),
94 Bytes.toBytes("row_0250"),
95 Bytes.toBytes("row_0400"));
96
97
98
99 sourceRegions[2] =
100 new HRegionInfo(this.desc.getTableName(),
101 Bytes.toBytes("row_0100"),
102 Bytes.toBytes("row_0200"));
103
104
105
106
107 sourceRegions[3] =
108 new HRegionInfo(this.desc.getTableName(),
109 Bytes.toBytes("row_0500"),
110 Bytes.toBytes("row_0600"));
111
112
113 sourceRegions[4] =
114 new HRegionInfo(this.desc.getTableName(),
115 HConstants.EMPTY_BYTE_ARRAY,
116 HConstants.EMPTY_BYTE_ARRAY);
117
118
119
120
121 this.rows = new byte [5][][];
122 this.rows[0] = Bytes.toByteArrays(new String[] { "row_0210", "row_0280" });
123 this.rows[1] = Bytes.toByteArrays(new String[] { "row_0260", "row_0350",
124 "row_035" });
125 this.rows[2] = Bytes.toByteArrays(new String[] { "row_0110", "row_0175",
126 "row_0175", "row_0175"});
127 this.rows[3] = Bytes.toByteArrays(new String[] { "row_0525", "row_0560",
128 "row_0560", "row_0560", "row_0560"});
129 this.rows[4] = Bytes.toByteArrays(new String[] { "row_0050", "row_1000",
130 "row_1000", "row_1000", "row_1000", "row_1000" });
131
132
133 TEST_UTIL = new HBaseTestingUtility(conf);
134 this.dfsCluster = TEST_UTIL.startMiniDFSCluster(2);
135 this.fs = this.dfsCluster.getFileSystem();
136 System.out.println("fs=" + this.fs);
137 FSUtils.setFsDefault(this.conf, new Path(fs.getUri()));
138 Path parentdir = fs.getHomeDirectory();
139 FSUtils.setRootDir(conf, parentdir);
140 fs.mkdirs(parentdir);
141 FSUtils.setVersion(fs, parentdir);
142
143
144
145
146 super.setUp();
147 try {
148
149 createMetaRegion();
150 new FSTableDescriptors(this.fs, this.testDir).createTableDescriptor(this.desc);
151
152
153
154 for (int i = 0; i < sourceRegions.length; i++) {
155 regions[i] =
156 HRegion.createHRegion(this.sourceRegions[i], this.testDir, this.conf,
157 this.desc);
158
159
160
161 for (int j = 0; j < rows[i].length; j++) {
162 byte [] row = rows[i][j];
163 Put put = new Put(row);
164 put.add(FAMILY, QUALIFIER, row);
165 regions[i].put(put);
166 }
167 HRegion.addRegionToMETA(meta, regions[i]);
168 }
169
170 closeRootAndMeta();
171
172 } catch (Exception e) {
173 TEST_UTIL.shutdownMiniCluster();
174 throw e;
175 }
176 }
177
178 @Override
179 public void tearDown() throws Exception {
180 super.tearDown();
181 for (int i = 0; i < sourceRegions.length; i++) {
182 HRegion r = regions[i];
183 if (r != null) {
184 HRegion.closeHRegion(r);
185 }
186 }
187 TEST_UTIL.shutdownMiniCluster();
188 }
189
190
191
192
193
194
195
196
197
198
199 private HRegion mergeAndVerify(final String msg, final String regionName1,
200 final String regionName2, final HLog log, final int upperbound)
201 throws Exception {
202 Merge merger = new Merge(this.conf);
203 LOG.info(msg);
204 LOG.info("fs2=" + this.conf.get("fs.defaultFS"));
205 int errCode = ToolRunner.run(this.conf, merger,
206 new String[] {this.desc.getTableName().getNameAsString(), regionName1, regionName2}
207 );
208 assertTrue("'" + msg + "' failed with errCode " + errCode, errCode == 0);
209 HRegionInfo mergedInfo = merger.getMergedHRegionInfo();
210
211
212
213 HRegion merged = HRegion.openHRegion(mergedInfo, this.desc, log, this.conf);
214 verifyMerge(merged, upperbound);
215 merged.close();
216 LOG.info("Verified " + msg);
217 return merged;
218 }
219
220 private void verifyMerge(final HRegion merged, final int upperbound)
221 throws IOException {
222
223 Scan scan = new Scan();
224 scan.addFamily(FAMILY);
225 InternalScanner scanner = merged.getScanner(scan);
226 try {
227 List<Cell> testRes = null;
228 while (true) {
229 testRes = new ArrayList<Cell>();
230 boolean hasNext = scanner.next(testRes);
231 if (!hasNext) {
232 break;
233 }
234 }
235 } finally {
236 scanner.close();
237 }
238
239
240
241 for (int i = 0; i < upperbound; i++) {
242 for (int j = 0; j < rows[i].length; j++) {
243 Get get = new Get(rows[i][j]);
244 get.addFamily(FAMILY);
245 Result result = merged.get(get);
246 assertEquals(1, result.size());
247 byte [] bytes = CellUtil.cloneValue(result.rawCells()[0]);
248 assertNotNull(Bytes.toStringBinary(rows[i][j]), bytes);
249 assertTrue(Bytes.equals(bytes, rows[i][j]));
250 }
251 }
252 }
253
254
255
256
257
258 public void testMergeTool() throws Exception {
259
260
261 for (int i = 0; i < regions.length; i++) {
262 for (int j = 0; j < rows[i].length; j++) {
263 Get get = new Get(rows[i][j]);
264 get.addFamily(FAMILY);
265 Result result = regions[i].get(get);
266 byte [] bytes = CellUtil.cloneValue(result.rawCells()[0]);
267 assertNotNull(bytes);
268 assertTrue(Bytes.equals(bytes, rows[i][j]));
269 }
270
271 HRegion.closeHRegion(regions[i]);
272 }
273
274
275 Path logPath = new Path("/tmp");
276 String logName = HConstants.HREGION_LOGDIR_NAME + "_"
277 + System.currentTimeMillis();
278 LOG.info("Creating log " + logPath.toString() + "/" + logName);
279
280 HLog log = HLogFactory.createHLog(this.fs, logPath,
281 logName, this.conf);
282
283 try {
284
285 HRegion merged = mergeAndVerify("merging regions 0 and 1 ",
286 this.sourceRegions[0].getRegionNameAsString(),
287 this.sourceRegions[1].getRegionNameAsString(), log, 2);
288
289
290 merged = mergeAndVerify("merging regions 0+1 and 2",
291 merged.getRegionInfo().getRegionNameAsString(),
292 this.sourceRegions[2].getRegionNameAsString(), log, 3);
293
294
295 merged = mergeAndVerify("merging regions 0+1+2 and 3",
296 merged.getRegionInfo().getRegionNameAsString(),
297 this.sourceRegions[3].getRegionNameAsString(), log, 4);
298
299
300 merged = mergeAndVerify("merging regions 0+1+2+3 and 4",
301 merged.getRegionInfo().getRegionNameAsString(),
302 this.sourceRegions[4].getRegionNameAsString(), log, rows.length);
303 } finally {
304 log.closeAndDelete();
305 }
306 }
307
308 }
309