1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 package org.apache.hadoop.hbase.util;
22
23 import java.io.IOException;
24 import java.util.ArrayList;
25 import java.util.List;
26
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.fs.Path;
30 import org.apache.hadoop.hbase.HBaseTestCase;
31 import org.apache.hadoop.hbase.HColumnDescriptor;
32 import org.apache.hadoop.hbase.HConstants;
33 import org.apache.hadoop.hbase.HRegionInfo;
34 import org.apache.hadoop.hbase.HTableDescriptor;
35 import org.apache.hadoop.hbase.KeyValue;
36 import org.apache.hadoop.hbase.client.Get;
37 import org.apache.hadoop.hbase.client.Put;
38 import org.apache.hadoop.hbase.client.Result;
39 import org.apache.hadoop.hbase.client.Scan;
40 import org.apache.hadoop.hbase.regionserver.wal.HLog;
41 import org.apache.hadoop.hbase.regionserver.HRegion;
42 import org.apache.hadoop.hbase.regionserver.InternalScanner;
43 import org.apache.hadoop.hdfs.MiniDFSCluster;
44 import org.apache.hadoop.util.ToolRunner;
45
46
47 public class TestMergeTool extends HBaseTestCase {
48 static final Log LOG = LogFactory.getLog(TestMergeTool.class);
49
50 static final byte [] FAMILY = Bytes.toBytes("contents");
51 static final byte [] QUALIFIER = Bytes.toBytes("dc");
52
53 private final HRegionInfo[] sourceRegions = new HRegionInfo[5];
54 private final HRegion[] regions = new HRegion[5];
55 private HTableDescriptor desc;
56 private byte [][][] rows;
57 private MiniDFSCluster dfsCluster = null;
58
59 @Override
60 public void setUp() throws Exception {
61 this.conf.set("hbase.hstore.compactionThreshold", "2");
62
63
64 this.desc = new HTableDescriptor("TestMergeTool");
65 this.desc.addFamily(new HColumnDescriptor(FAMILY));
66
67
68
69
70
71 sourceRegions[0] = new HRegionInfo(this.desc, Bytes.toBytes("row_0200"),
72 Bytes.toBytes("row_0300"));
73
74
75
76 sourceRegions[1] =
77 new HRegionInfo(this.desc, Bytes.toBytes("row_0250"),
78 Bytes.toBytes("row_0400"));
79
80
81
82 sourceRegions[2] =
83 new HRegionInfo(this.desc, Bytes.toBytes("row_0100"),
84 Bytes.toBytes("row_0200"));
85
86
87
88
89 sourceRegions[3] =
90 new HRegionInfo(this.desc, Bytes.toBytes("row_0500"),
91 Bytes.toBytes("row_0600"));
92
93
94 sourceRegions[4] =
95 new HRegionInfo(this.desc, HConstants.EMPTY_BYTE_ARRAY,
96 HConstants.EMPTY_BYTE_ARRAY);
97
98
99
100
101 this.rows = new byte [5][][];
102 this.rows[0] = Bytes.toByteArrays(new String[] { "row_0210", "row_0280" });
103 this.rows[1] = Bytes.toByteArrays(new String[] { "row_0260", "row_0350",
104 "row_035" });
105 this.rows[2] = Bytes.toByteArrays(new String[] { "row_0110", "row_0175",
106 "row_0175", "row_0175"});
107 this.rows[3] = Bytes.toByteArrays(new String[] { "row_0525", "row_0560",
108 "row_0560", "row_0560", "row_0560"});
109 this.rows[4] = Bytes.toByteArrays(new String[] { "row_0050", "row_1000",
110 "row_1000", "row_1000", "row_1000", "row_1000" });
111
112
113 this.dfsCluster = new MiniDFSCluster(conf, 2, true, (String[])null);
114 this.fs = this.dfsCluster.getFileSystem();
115 System.out.println("fs=" + this.fs);
116 this.conf.set("fs.defaultFS", fs.getUri().toString());
117 Path parentdir = fs.getHomeDirectory();
118 conf.set(HConstants.HBASE_DIR, parentdir.toString());
119 fs.mkdirs(parentdir);
120 FSUtils.setVersion(fs, parentdir);
121
122
123
124
125 super.setUp();
126 try {
127
128 createRootAndMetaRegions();
129
130
131
132 for (int i = 0; i < sourceRegions.length; i++) {
133 regions[i] =
134 HRegion.createHRegion(this.sourceRegions[i], this.testDir, this.conf);
135
136
137
138 for (int j = 0; j < rows[i].length; j++) {
139 byte [] row = rows[i][j];
140 Put put = new Put(row);
141 put.add(FAMILY, QUALIFIER, row);
142 regions[i].put(put);
143 }
144 HRegion.addRegionToMETA(meta, regions[i]);
145 }
146
147 closeRootAndMeta();
148
149 } catch (Exception e) {
150 shutdownDfs(dfsCluster);
151 throw e;
152 }
153 }
154
155 @Override
156 public void tearDown() throws Exception {
157 super.tearDown();
158 shutdownDfs(dfsCluster);
159 }
160
161
162
163
164
165
166
167
168
169
170 private HRegion mergeAndVerify(final String msg, final String regionName1,
171 final String regionName2, final HLog log, final int upperbound)
172 throws Exception {
173 Merge merger = new Merge(this.conf);
174 LOG.info(msg);
175 System.out.println("fs2=" + this.conf.get("fs.defaultFS"));
176 int errCode = ToolRunner.run(this.conf, merger,
177 new String[] {this.desc.getNameAsString(), regionName1, regionName2}
178 );
179 assertTrue("'" + msg + "' failed", errCode == 0);
180 HRegionInfo mergedInfo = merger.getMergedHRegionInfo();
181
182
183
184 HRegion merged =
185 HRegion.openHRegion(mergedInfo, this.testDir, log, this.conf);
186 verifyMerge(merged, upperbound);
187 merged.close();
188 LOG.info("Verified " + msg);
189 return merged;
190 }
191
192 private void verifyMerge(final HRegion merged, final int upperbound)
193 throws IOException {
194
195 Scan scan = new Scan();
196 scan.addFamily(FAMILY);
197 InternalScanner scanner = merged.getScanner(scan);
198 try {
199 List<KeyValue> testRes = null;
200 while (true) {
201 testRes = new ArrayList<KeyValue>();
202 boolean hasNext = scanner.next(testRes);
203 if (!hasNext) {
204 break;
205 }
206 }
207 } finally {
208 scanner.close();
209 }
210
211
212
213 for (int i = 0; i < upperbound; i++) {
214 for (int j = 0; j < rows[i].length; j++) {
215 Get get = new Get(rows[i][j]);
216 get.addFamily(FAMILY);
217 Result result = merged.get(get, null);
218 assertEquals(1, result.size());
219 byte [] bytes = result.sorted()[0].getValue();
220 assertNotNull(Bytes.toStringBinary(rows[i][j]), bytes);
221 assertTrue(Bytes.equals(bytes, rows[i][j]));
222 }
223 }
224 }
225
226
227
228
229
230 public void testMergeTool() throws Exception {
231
232
233 for (int i = 0; i < regions.length; i++) {
234 for (int j = 0; j < rows[i].length; j++) {
235 Get get = new Get(rows[i][j]);
236 get.addFamily(FAMILY);
237 Result result = regions[i].get(get, null);
238 byte [] bytes = result.sorted()[0].getValue();
239 assertNotNull(bytes);
240 assertTrue(Bytes.equals(bytes, rows[i][j]));
241 }
242
243 regions[i].close();
244 regions[i].getLog().closeAndDelete();
245 }
246
247
248 Path logPath = new Path("/tmp", HConstants.HREGION_LOGDIR_NAME + "_" +
249 System.currentTimeMillis());
250 LOG.info("Creating log " + logPath.toString());
251 Path oldLogDir = new Path("/tmp", HConstants.HREGION_OLDLOGDIR_NAME);
252 HLog log = new HLog(this.fs, logPath, oldLogDir, this.conf, null);
253 try {
254
255 HRegion merged = mergeAndVerify("merging regions 0 and 1",
256 this.sourceRegions[0].getRegionNameAsString(),
257 this.sourceRegions[1].getRegionNameAsString(), log, 2);
258
259
260 merged = mergeAndVerify("merging regions 0+1 and 2",
261 merged.getRegionInfo().getRegionNameAsString(),
262 this.sourceRegions[2].getRegionNameAsString(), log, 3);
263
264
265 merged = mergeAndVerify("merging regions 0+1+2 and 3",
266 merged.getRegionInfo().getRegionNameAsString(),
267 this.sourceRegions[3].getRegionNameAsString(), log, 4);
268
269
270 merged = mergeAndVerify("merging regions 0+1+2+3 and 4",
271 merged.getRegionInfo().getRegionNameAsString(),
272 this.sourceRegions[4].getRegionNameAsString(), log, rows.length);
273 } finally {
274 log.closeAndDelete();
275 }
276 }
277 }