1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertNull;
24 import static org.junit.Assert.assertTrue;
25
26 import java.io.IOException;
27 import java.util.List;
28 import java.util.Random;
29 import org.apache.commons.logging.Log;
30 import org.apache.commons.logging.LogFactory;
31 import org.apache.hadoop.conf.Configuration;
32 import org.apache.hadoop.hbase.client.Admin;
33 import org.apache.hadoop.hbase.client.Get;
34 import org.apache.hadoop.hbase.client.Connection;
35 import org.apache.hadoop.hbase.client.ConnectionFactory;
36 import org.apache.hadoop.hbase.client.HConnectionManager;
37 import org.apache.hadoop.hbase.client.HTable;
38 import org.apache.hadoop.hbase.client.Result;
39 import org.apache.hadoop.hbase.client.Table;
40 import org.apache.hadoop.hbase.testclassification.MediumTests;
41 import org.apache.hadoop.hbase.util.Bytes;
42 import org.apache.hadoop.hbase.util.Pair;
43 import org.junit.AfterClass;
44 import org.junit.Assert;
45 import org.junit.BeforeClass;
46 import org.junit.Test;
47 import org.junit.experimental.categories.Category;
48
49
50
51
52 @Category(MediumTests.class)
53 public class TestMetaTableAccessor {
54 private static final Log LOG = LogFactory.getLog(TestMetaTableAccessor.class);
55 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
56 private static Connection connection;
57
58 @BeforeClass public static void beforeClass() throws Exception {
59 UTIL.startMiniCluster(3);
60
61 Configuration c = new Configuration(UTIL.getConfiguration());
62
63
64 c.setLong("hbase.client.pause", 1000);
65 c.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 10);
66 connection = ConnectionFactory.createConnection(c);
67 }
68
69 @AfterClass public static void afterClass() throws Exception {
70 connection.close();
71 UTIL.shutdownMiniCluster();
72 }
73
74
75
76
77
78
79
80
81 @Test public void testRetrying()
82 throws IOException, InterruptedException {
83 final TableName name =
84 TableName.valueOf("testRetrying");
85 LOG.info("Started " + name);
86 HTable t = UTIL.createTable(name, HConstants.CATALOG_FAMILY);
87 int regionCount = UTIL.createMultiRegions(t, HConstants.CATALOG_FAMILY);
88
89 final List<HRegionInfo> regions =
90 testGettingTableRegions(connection, name, regionCount);
91 MetaTask reader = new MetaTask(connection, "reader") {
92 @Override
93 void metaTask() throws Throwable {
94 testGetRegion(connection, regions.get(0));
95 LOG.info("Read " + regions.get(0).getEncodedName());
96 }
97 };
98 MetaTask writer = new MetaTask(connection, "writer") {
99 @Override
100 void metaTask() throws Throwable {
101 MetaTableAccessor.addRegionToMeta(connection, regions.get(0));
102 LOG.info("Wrote " + regions.get(0).getEncodedName());
103 }
104 };
105 reader.start();
106 writer.start();
107
108
109
110
111 final long timeOut = 180000;
112 long startTime = System.currentTimeMillis();
113
114 try {
115
116 assertTrue(reader.isProgressing());
117 assertTrue(writer.isProgressing());
118
119
120
121 for (int i = 0; i < 2; i++) {
122 LOG.info("Restart=" + i);
123 UTIL.ensureSomeRegionServersAvailable(2);
124 int index = -1;
125 do {
126 index = UTIL.getMiniHBaseCluster().getServerWithMeta();
127 } while (index == -1 &&
128 startTime + timeOut < System.currentTimeMillis());
129
130 if (index != -1){
131 UTIL.getMiniHBaseCluster().abortRegionServer(index);
132 UTIL.getMiniHBaseCluster().waitOnRegionServer(index);
133 }
134 }
135
136 assertTrue("reader: " + reader.toString(), reader.isProgressing());
137 assertTrue("writer: " + writer.toString(), writer.isProgressing());
138 } catch (IOException e) {
139 throw e;
140 } finally {
141 reader.stop = true;
142 writer.stop = true;
143 reader.join();
144 writer.join();
145 t.close();
146 }
147 long exeTime = System.currentTimeMillis() - startTime;
148 assertTrue("Timeout: test took " + exeTime / 1000 + " sec", exeTime < timeOut);
149 }
150
151
152
153
154 abstract static class MetaTask extends Thread {
155 boolean stop = false;
156 int count = 0;
157 Throwable t = null;
158 final Connection connection;
159
160 MetaTask(final Connection connection, final String name) {
161 super(name);
162 this.connection = connection;
163 }
164
165 @Override
166 public void run() {
167 try {
168 while(!this.stop) {
169 LOG.info("Before " + this.getName()+ ", count=" + this.count);
170 metaTask();
171 this.count += 1;
172 LOG.info("After " + this.getName() + ", count=" + this.count);
173 Thread.sleep(100);
174 }
175 } catch (Throwable t) {
176 LOG.info(this.getName() + " failed", t);
177 this.t = t;
178 }
179 }
180
181 boolean isProgressing() throws InterruptedException {
182 int currentCount = this.count;
183 while(currentCount == this.count) {
184 if (!isAlive()) return false;
185 if (this.t != null) return false;
186 Thread.sleep(10);
187 }
188 return true;
189 }
190
191 @Override
192 public String toString() {
193 return "count=" + this.count + ", t=" +
194 (this.t == null? "null": this.t.toString());
195 }
196
197 abstract void metaTask() throws Throwable;
198 }
199
200 @Test public void testGetRegionsCatalogTables()
201 throws IOException, InterruptedException {
202 List<HRegionInfo> regions =
203 MetaTableAccessor.getTableRegions(UTIL.getZooKeeperWatcher(),
204 connection, TableName.META_TABLE_NAME);
205 assertTrue(regions.size() >= 1);
206 assertTrue(MetaTableAccessor.getTableRegionsAndLocations(UTIL.getZooKeeperWatcher(),
207 connection,TableName.META_TABLE_NAME).size() >= 1);
208 }
209
210 @Test public void testTableExists() throws IOException {
211 final TableName name =
212 TableName.valueOf("testTableExists");
213 assertFalse(MetaTableAccessor.tableExists(connection, name));
214 UTIL.createTable(name, HConstants.CATALOG_FAMILY);
215 assertTrue(MetaTableAccessor.tableExists(connection, name));
216 Admin admin = UTIL.getHBaseAdmin();
217 admin.disableTable(name);
218 admin.deleteTable(name);
219 assertFalse(MetaTableAccessor.tableExists(connection, name));
220 assertTrue(MetaTableAccessor.tableExists(connection,
221 TableName.META_TABLE_NAME));
222 }
223
224 @Test public void testGetRegion() throws IOException, InterruptedException {
225 final String name = "testGetRegion";
226 LOG.info("Started " + name);
227
228 Pair<HRegionInfo, ServerName> pair =
229 MetaTableAccessor.getRegion(connection, Bytes.toBytes("nonexistent-region"));
230 assertNull(pair);
231 LOG.info("Finished " + name);
232 }
233
234
235 @Test public void testScanMetaForTable()
236 throws IOException, InterruptedException {
237 final TableName name =
238 TableName.valueOf("testScanMetaForTable");
239 LOG.info("Started " + name);
240
241
242
243
244
245
246 UTIL.createTable(name, HConstants.CATALOG_FAMILY);
247
248 TableName greaterName =
249 TableName.valueOf("testScanMetaForTablf");
250 UTIL.createTable(greaterName, HConstants.CATALOG_FAMILY);
251
252
253
254 assertEquals(1, MetaTableAccessor.getTableRegions(UTIL.getZooKeeperWatcher(),
255 connection, name).size());
256 assertEquals(1, MetaTableAccessor.getTableRegions(UTIL.getZooKeeperWatcher(),
257 connection, greaterName).size());
258 }
259
260 private static List<HRegionInfo> testGettingTableRegions(final Connection connection,
261 final TableName name, final int regionCount)
262 throws IOException, InterruptedException {
263 List<HRegionInfo> regions = MetaTableAccessor.getTableRegions(UTIL.getZooKeeperWatcher(),
264 connection, name);
265 assertEquals(regionCount, regions.size());
266 Pair<HRegionInfo, ServerName> pair =
267 MetaTableAccessor.getRegion(connection, regions.get(0).getRegionName());
268 assertEquals(regions.get(0).getEncodedName(),
269 pair.getFirst().getEncodedName());
270 return regions;
271 }
272
273 private static void testGetRegion(final Connection connection,
274 final HRegionInfo region)
275 throws IOException, InterruptedException {
276 Pair<HRegionInfo, ServerName> pair =
277 MetaTableAccessor.getRegion(connection, region.getRegionName());
278 assertEquals(region.getEncodedName(),
279 pair.getFirst().getEncodedName());
280 }
281
282 @Test
283 public void testParseReplicaIdFromServerColumn() {
284 String column1 = HConstants.SERVER_QUALIFIER_STR;
285 assertEquals(0, MetaTableAccessor.parseReplicaIdFromServerColumn(Bytes.toBytes(column1)));
286 String column2 = column1 + MetaTableAccessor.META_REPLICA_ID_DELIMITER;
287 assertEquals(-1, MetaTableAccessor.parseReplicaIdFromServerColumn(Bytes.toBytes(column2)));
288 String column3 = column2 + "00";
289 assertEquals(-1, MetaTableAccessor.parseReplicaIdFromServerColumn(Bytes.toBytes(column3)));
290 String column4 = column3 + "2A";
291 assertEquals(42, MetaTableAccessor.parseReplicaIdFromServerColumn(Bytes.toBytes(column4)));
292 String column5 = column4 + "2A";
293 assertEquals(-1, MetaTableAccessor.parseReplicaIdFromServerColumn(Bytes.toBytes(column5)));
294 String column6 = HConstants.STARTCODE_QUALIFIER_STR;
295 assertEquals(-1, MetaTableAccessor.parseReplicaIdFromServerColumn(Bytes.toBytes(column6)));
296 }
297
298 @Test
299 public void testMetaReaderGetColumnMethods() {
300 Assert.assertArrayEquals(HConstants.SERVER_QUALIFIER, MetaTableAccessor.getServerColumn(0));
301 Assert.assertArrayEquals(Bytes.toBytes(HConstants.SERVER_QUALIFIER_STR
302 + MetaTableAccessor.META_REPLICA_ID_DELIMITER + "002A"),
303 MetaTableAccessor.getServerColumn(42));
304
305 Assert.assertArrayEquals(HConstants.STARTCODE_QUALIFIER,
306 MetaTableAccessor.getStartCodeColumn(0));
307 Assert.assertArrayEquals(Bytes.toBytes(HConstants.STARTCODE_QUALIFIER_STR
308 + MetaTableAccessor.META_REPLICA_ID_DELIMITER + "002A"),
309 MetaTableAccessor.getStartCodeColumn(42));
310
311 Assert.assertArrayEquals(HConstants.SEQNUM_QUALIFIER,
312 MetaTableAccessor.getSeqNumColumn(0));
313 Assert.assertArrayEquals(Bytes.toBytes(HConstants.SEQNUM_QUALIFIER_STR
314 + MetaTableAccessor.META_REPLICA_ID_DELIMITER + "002A"),
315 MetaTableAccessor.getSeqNumColumn(42));
316 }
317
318 @Test
319 public void testMetaLocationsForRegionReplicas() throws IOException {
320 Random random = new Random();
321 ServerName serverName0 = ServerName.valueOf("foo", 60010, random.nextLong());
322 ServerName serverName1 = ServerName.valueOf("bar", 60010, random.nextLong());
323 ServerName serverName100 = ServerName.valueOf("baz", 60010, random.nextLong());
324
325 long regionId = System.currentTimeMillis();
326 HRegionInfo primary = new HRegionInfo(TableName.valueOf("table_foo"),
327 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, false, regionId, 0);
328 HRegionInfo replica1 = new HRegionInfo(TableName.valueOf("table_foo"),
329 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, false, regionId, 1);
330 HRegionInfo replica100 = new HRegionInfo(TableName.valueOf("table_foo"),
331 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, false, regionId, 100);
332
333 long seqNum0 = random.nextLong();
334 long seqNum1 = random.nextLong();
335 long seqNum100 = random.nextLong();
336
337
338 Table meta = MetaTableAccessor.getMetaHTable(connection);
339 try {
340 MetaTableAccessor.updateRegionLocation(connection, primary, serverName0, seqNum0);
341
342
343 assertMetaLocation(meta, primary.getRegionName(), serverName0, seqNum0, 0, true);
344
345
346 MetaTableAccessor.updateRegionLocation(connection, replica1, serverName1, seqNum1);
347
348 assertMetaLocation(meta, primary.getRegionName(), serverName0, seqNum0, 0, true);
349
350 assertMetaLocation(meta, primary.getRegionName(), serverName1, seqNum1, 1, true);
351
352
353 MetaTableAccessor.updateRegionLocation(connection, replica100, serverName100, seqNum100);
354
355 assertMetaLocation(meta, primary.getRegionName(), serverName0, seqNum0, 0, true);
356
357 assertMetaLocation(meta, primary.getRegionName(), serverName1, seqNum1, 1, true);
358
359 assertMetaLocation(meta, primary.getRegionName(), serverName100, seqNum100, 100, true);
360 } finally {
361 meta.close();
362 }
363 }
364
365 public static void assertMetaLocation(Table meta, byte[] row, ServerName serverName,
366 long seqNum, int replicaId, boolean checkSeqNum) throws IOException {
367 Get get = new Get(row);
368 Result result = meta.get(get);
369 assertTrue(Bytes.equals(
370 result.getValue(HConstants.CATALOG_FAMILY, MetaTableAccessor.getServerColumn(replicaId)),
371 Bytes.toBytes(serverName.getHostAndPort())));
372 assertTrue(Bytes.equals(
373 result.getValue(HConstants.CATALOG_FAMILY, MetaTableAccessor.getStartCodeColumn(replicaId)),
374 Bytes.toBytes(serverName.getStartcode())));
375 if (checkSeqNum) {
376 assertTrue(Bytes.equals(
377 result.getValue(HConstants.CATALOG_FAMILY, MetaTableAccessor.getSeqNumColumn(replicaId)),
378 Bytes.toBytes(seqNum)));
379 }
380 }
381 }
382