1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.util.hbck;
19
20 import static org.junit.Assert.assertEquals;
21
22 import java.io.IOException;
23 import java.util.ArrayList;
24 import java.util.List;
25 import java.util.Map;
26 import java.util.Map.Entry;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.conf.Configuration;
31 import org.apache.hadoop.fs.FSDataOutputStream;
32 import org.apache.hadoop.fs.FileSystem;
33 import org.apache.hadoop.fs.Path;
34 import org.apache.hadoop.hbase.*;
35 import org.apache.hadoop.hbase.client.Delete;
36 import org.apache.hadoop.hbase.client.HBaseAdmin;
37 import org.apache.hadoop.hbase.client.HConnectionManager;
38 import org.apache.hadoop.hbase.client.HTable;
39 import org.apache.hadoop.hbase.client.Put;
40 import org.apache.hadoop.hbase.client.Result;
41 import org.apache.hadoop.hbase.client.ResultScanner;
42 import org.apache.hadoop.hbase.client.Scan;
43 import org.apache.hadoop.hbase.regionserver.HRegion;
44 import org.apache.hadoop.hbase.util.Bytes;
45 import org.apache.hadoop.hbase.util.Writables;
46 import org.apache.zookeeper.KeeperException;
47 import org.junit.After;
48 import org.junit.Before;
49 import org.junit.experimental.categories.Category;
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64 @Category(LargeTests.class)
65 public class OfflineMetaRebuildTestCore {
66 protected final static Log LOG = LogFactory
67 .getLog(OfflineMetaRebuildTestCore.class);
68 protected HBaseTestingUtility TEST_UTIL;
69 protected Configuration conf;
70 private final static byte[] FAM = Bytes.toBytes("fam");
71
72
73 protected HTable htbl;
74 protected final static byte[][] splits = new byte[][] { Bytes.toBytes("A"),
75 Bytes.toBytes("B"), Bytes.toBytes("C") };
76
77 private final static String TABLE_BASE = "tableMetaRebuild";
78 private static int tableIdx = 0;
79 protected String table = "tableMetaRebuild";
80
81 @Before
82 public void setUpBefore() throws Exception {
83 TEST_UTIL = new HBaseTestingUtility();
84 TEST_UTIL.getConfiguration().setInt("dfs.datanode.max.xceivers", 9192);
85 TEST_UTIL.startMiniCluster(3);
86 conf = TEST_UTIL.getConfiguration();
87 assertEquals(0, TEST_UTIL.getHBaseAdmin().listTables().length);
88
89
90 table = TABLE_BASE + "-" + tableIdx;
91 tableIdx++;
92 htbl = setupTable(table);
93 populateTable(htbl);
94 assertEquals(4, scanMeta());
95 LOG.info("Table " + table + " has " + tableRowCount(conf, table)
96 + " entries.");
97 assertEquals(16, tableRowCount(conf, table));
98 TEST_UTIL.getHBaseAdmin().disableTable(table);
99 assertEquals(1, TEST_UTIL.getHBaseAdmin().listTables().length);
100 }
101
102 @After
103 public void tearDownAfter() throws Exception {
104 TEST_UTIL.shutdownMiniCluster();
105 HConnectionManager.deleteConnection(conf);
106 }
107
108
109
110
111
112
113
114
115 private HTable setupTable(String tablename) throws Exception {
116 HTableDescriptor desc = new HTableDescriptor(tablename);
117 HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM));
118 desc.addFamily(hcd);
119 TEST_UTIL.getHBaseAdmin().createTable(desc, splits);
120 return new HTable(TEST_UTIL.getConfiguration(), tablename);
121 }
122
123 private void dumpMeta(HTableDescriptor htd) throws IOException {
124 List<byte[]> metaRows = TEST_UTIL.getMetaTableRows(htd.getName());
125 for (byte[] row : metaRows) {
126 LOG.info(Bytes.toString(row));
127 }
128 }
129
130 private void populateTable(HTable tbl) throws IOException {
131 byte[] values = { 'A', 'B', 'C', 'D' };
132 for (int i = 0; i < values.length; i++) {
133 for (int j = 0; j < values.length; j++) {
134 Put put = new Put(new byte[] { values[i], values[j] });
135 put.add(Bytes.toBytes("fam"), new byte[] {}, new byte[] { values[i],
136 values[j] });
137 tbl.put(put);
138 }
139 }
140 tbl.flushCommits();
141 }
142
143
144
145
146
147
148
149 void deleteTable(HBaseAdmin admin, String tablename) throws IOException {
150 try {
151 byte[] tbytes = Bytes.toBytes(tablename);
152 admin.disableTable(tbytes);
153 admin.deleteTable(tbytes);
154 } catch (Exception e) {
155
156 }
157 }
158
159 protected void deleteRegion(Configuration conf, final HTable tbl,
160 byte[] startKey, byte[] endKey) throws IOException {
161
162 LOG.info("Before delete:");
163 HTableDescriptor htd = tbl.getTableDescriptor();
164 dumpMeta(htd);
165
166 Map<HRegionInfo, HServerAddress> hris = tbl.getRegionsInfo();
167 for (Entry<HRegionInfo, HServerAddress> e : hris.entrySet()) {
168 HRegionInfo hri = e.getKey();
169 HServerAddress hsa = e.getValue();
170 if (Bytes.compareTo(hri.getStartKey(), startKey) == 0
171 && Bytes.compareTo(hri.getEndKey(), endKey) == 0) {
172
173 LOG.info("RegionName: " + hri.getRegionNameAsString());
174 byte[] deleteRow = hri.getRegionName();
175 TEST_UTIL.getHBaseAdmin().unassign(deleteRow, true);
176
177 LOG.info("deleting hdfs data: " + hri.toString() + hsa.toString());
178 Path rootDir = new Path(conf.get(HConstants.HBASE_DIR));
179 FileSystem fs = rootDir.getFileSystem(conf);
180 Path p = new Path(rootDir + "/" + htd.getNameAsString(),
181 hri.getEncodedName());
182 fs.delete(p, true);
183
184 HTable meta = new HTable(conf, HConstants.META_TABLE_NAME);
185 Delete delete = new Delete(deleteRow);
186 meta.delete(delete);
187 }
188 LOG.info(hri.toString() + hsa.toString());
189 }
190
191 TEST_UTIL.getMetaTableRows(htd.getName());
192 LOG.info("After delete:");
193 dumpMeta(htd);
194 }
195
196 protected HRegionInfo createRegion(Configuration conf, final HTable htbl,
197 byte[] startKey, byte[] endKey) throws IOException {
198 HTable meta = new HTable(conf, HConstants.META_TABLE_NAME);
199 HTableDescriptor htd = htbl.getTableDescriptor();
200 HRegionInfo hri = new HRegionInfo(htbl.getTableName(), startKey, endKey);
201
202 LOG.info("manually adding regioninfo and hdfs data: " + hri.toString());
203 Path rootDir = new Path(conf.get(HConstants.HBASE_DIR));
204 FileSystem fs = rootDir.getFileSystem(conf);
205 Path p = new Path(rootDir + "/" + htd.getNameAsString(),
206 hri.getEncodedName());
207 fs.mkdirs(p);
208 Path riPath = new Path(p, HRegion.REGIONINFO_FILE);
209 FSDataOutputStream out = fs.create(riPath);
210 hri.write(out);
211 out.close();
212
213
214 Put put = new Put(hri.getRegionName());
215 put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
216 Writables.getBytes(hri));
217 meta.put(put);
218 meta.flushCommits();
219 return hri;
220 }
221
222 protected void wipeOutMeta() throws IOException {
223
224 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
225 Scan s = new Scan();
226 HTable meta = new HTable(conf, HConstants.META_TABLE_NAME);
227 ResultScanner scanner = meta.getScanner(s);
228 List<Delete> dels = new ArrayList<Delete>();
229 for (Result r : scanner) {
230 Delete d = new Delete(r.getRow());
231 dels.add(d);
232 admin.unassign(r.getRow(), true);
233 }
234 meta.delete(dels);
235 meta.flushCommits();
236 scanner.close();
237 meta.close();
238 }
239
240
241
242
243
244
245
246 protected int tableRowCount(Configuration conf, String table)
247 throws IOException {
248 HTable t = new HTable(conf, table);
249 Scan st = new Scan();
250
251 ResultScanner rst = t.getScanner(st);
252 int count = 0;
253 for (@SuppressWarnings("unused")
254 Result rt : rst) {
255 count++;
256 }
257 return count;
258 }
259
260
261
262
263
264
265 protected int scanMeta() throws IOException {
266 int count = 0;
267 HTable meta = new HTable(conf, HTableDescriptor.META_TABLEDESC.getName());
268 ResultScanner scanner = meta.getScanner(new Scan());
269 LOG.info("Table: " + Bytes.toString(meta.getTableName()));
270 for (Result res : scanner) {
271 LOG.info(Bytes.toString(res.getRow()));
272 count++;
273 }
274 return count;
275 }
276 }