1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.catalog;
20
21 import java.io.IOException;
22 import java.util.List;
23 import java.util.concurrent.atomic.AtomicBoolean;
24
25 import com.google.common.collect.Lists;
26
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.hbase.Abortable;
31 import org.apache.hadoop.hbase.Cell;
32 import org.apache.hadoop.hbase.TableName;
33 import org.apache.hadoop.hbase.HBaseTestingUtility;
34 import org.apache.hadoop.hbase.HConstants;
35 import org.apache.hadoop.hbase.HRegionInfo;
36 import org.apache.hadoop.hbase.client.Get;
37 import org.apache.hadoop.hbase.client.Result;
38 import org.apache.hadoop.hbase.testclassification.MediumTests;
39 import org.apache.hadoop.hbase.ServerName;
40 import org.apache.hadoop.hbase.client.HBaseAdmin;
41 import org.apache.hadoop.hbase.client.HTable;
42 import org.apache.hadoop.hbase.util.Bytes;
43 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
44 import org.apache.hadoop.hbase.util.Pair;
45 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
46 import org.junit.AfterClass;
47 import org.junit.BeforeClass;
48 import org.junit.Test;
49 import org.junit.experimental.categories.Category;
50
51 import static org.junit.Assert.*;
52
53
54
55
56 @Category(MediumTests.class)
57 public class TestMetaReaderEditor {
58 private static final Log LOG = LogFactory.getLog(TestMetaReaderEditor.class);
59 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
60 private static ZooKeeperWatcher zkw;
61 private static CatalogTracker CT;
62 private final static Abortable ABORTABLE = new Abortable() {
63 private final AtomicBoolean abort = new AtomicBoolean(false);
64
65 @Override
66 public void abort(String why, Throwable e) {
67 LOG.info(why, e);
68 abort.set(true);
69 }
70
71 @Override
72 public boolean isAborted() {
73 return abort.get();
74 }
75 };
76
77 @BeforeClass public static void beforeClass() throws Exception {
78 UTIL.startMiniCluster(3);
79
80 Configuration c = new Configuration(UTIL.getConfiguration());
81
82
83 c.setLong("hbase.client.pause", 1000);
84 c.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 10);
85 zkw = new ZooKeeperWatcher(c, "TestMetaReaderEditor", ABORTABLE);
86 CT = new CatalogTracker(zkw, c, ABORTABLE);
87 CT.start();
88 }
89
90 @AfterClass public static void afterClass() throws Exception {
91 ABORTABLE.abort("test ending", null);
92 CT.stop();
93 UTIL.shutdownMiniCluster();
94 }
95
96
97
98
99
100
101
102
103 @Test public void testRetrying()
104 throws IOException, InterruptedException {
105 final TableName name =
106 TableName.valueOf("testRetrying");
107 LOG.info("Started " + name);
108 HTable t = UTIL.createTable(name, HConstants.CATALOG_FAMILY);
109 int regionCount = UTIL.createMultiRegions(t, HConstants.CATALOG_FAMILY);
110
111 final List<HRegionInfo> regions =
112 testGettingTableRegions(CT, name, regionCount);
113 MetaTask reader = new MetaTask(CT, "reader") {
114 @Override
115 void metaTask() throws Throwable {
116 testGetRegion(this.ct, regions.get(0));
117 LOG.info("Read " + regions.get(0).getEncodedName());
118 }
119 };
120 MetaTask writer = new MetaTask(CT, "writer") {
121 @Override
122 void metaTask() throws Throwable {
123 MetaEditor.addRegionToMeta(this.ct, regions.get(0));
124 LOG.info("Wrote " + regions.get(0).getEncodedName());
125 }
126 };
127 reader.start();
128 writer.start();
129
130
131
132
133 final long timeOut = 180000;
134 long startTime = System.currentTimeMillis();
135
136 try {
137
138 assertTrue(reader.isProgressing());
139 assertTrue(writer.isProgressing());
140
141
142
143 for (int i = 0; i < 2; i++) {
144 LOG.info("Restart=" + i);
145 UTIL.ensureSomeRegionServersAvailable(2);
146 int index = -1;
147 do {
148 index = UTIL.getMiniHBaseCluster().getServerWithMeta();
149 } while (index == -1 &&
150 startTime + timeOut < System.currentTimeMillis());
151
152 if (index != -1){
153 UTIL.getMiniHBaseCluster().abortRegionServer(index);
154 UTIL.getMiniHBaseCluster().waitOnRegionServer(index);
155 }
156 }
157
158 assertTrue("reader: " + reader.toString(), reader.isProgressing());
159 assertTrue("writer: " + writer.toString(), writer.isProgressing());
160 } catch (IOException e) {
161 throw e;
162 } finally {
163 reader.stop = true;
164 writer.stop = true;
165 reader.join();
166 writer.join();
167 t.close();
168 }
169 long exeTime = System.currentTimeMillis() - startTime;
170 assertTrue("Timeout: test took " + exeTime / 1000 + " sec", exeTime < timeOut);
171 }
172
173
174
175
176 abstract static class MetaTask extends Thread {
177 boolean stop = false;
178 int count = 0;
179 Throwable t = null;
180 final CatalogTracker ct;
181
182 MetaTask(final CatalogTracker ct, final String name) {
183 super(name);
184 this.ct = ct;
185 }
186
187 @Override
188 public void run() {
189 try {
190 while(!this.stop) {
191 LOG.info("Before " + this.getName()+ ", count=" + this.count);
192 metaTask();
193 this.count += 1;
194 LOG.info("After " + this.getName() + ", count=" + this.count);
195 Thread.sleep(100);
196 }
197 } catch (Throwable t) {
198 LOG.info(this.getName() + " failed", t);
199 this.t = t;
200 }
201 }
202
203 boolean isProgressing() throws InterruptedException {
204 int currentCount = this.count;
205 while(currentCount == this.count) {
206 if (!isAlive()) return false;
207 if (this.t != null) return false;
208 Thread.sleep(10);
209 }
210 return true;
211 }
212
213 @Override
214 public String toString() {
215 return "count=" + this.count + ", t=" +
216 (this.t == null? "null": this.t.toString());
217 }
218
219 abstract void metaTask() throws Throwable;
220 }
221
222 @Test public void testGetRegionsCatalogTables()
223 throws IOException, InterruptedException {
224 List<HRegionInfo> regions =
225 MetaReader.getTableRegions(CT, TableName.META_TABLE_NAME);
226 assertTrue(regions.size() >= 1);
227 assertTrue(MetaReader.getTableRegionsAndLocations(CT,
228 TableName.META_TABLE_NAME).size() >= 1);
229 }
230
231 @Test public void testTableExists() throws IOException {
232 final TableName name =
233 TableName.valueOf("testTableExists");
234 assertFalse(MetaReader.tableExists(CT, name));
235 UTIL.createTable(name, HConstants.CATALOG_FAMILY);
236 assertTrue(MetaReader.tableExists(CT, name));
237 HBaseAdmin admin = UTIL.getHBaseAdmin();
238 admin.disableTable(name);
239 admin.deleteTable(name);
240 assertFalse(MetaReader.tableExists(CT, name));
241 assertTrue(MetaReader.tableExists(CT,
242 TableName.META_TABLE_NAME));
243 }
244
245 @Test public void testGetRegion() throws IOException, InterruptedException {
246 final String name = "testGetRegion";
247 LOG.info("Started " + name);
248
249 Pair<HRegionInfo, ServerName> pair =
250 MetaReader.getRegion(CT, Bytes.toBytes("nonexistent-region"));
251 assertNull(pair);
252 LOG.info("Finished " + name);
253 }
254
255
256 @Test public void testScanMetaForTable()
257 throws IOException, InterruptedException {
258 final TableName name =
259 TableName.valueOf("testScanMetaForTable");
260 LOG.info("Started " + name);
261
262
263
264
265
266
267 UTIL.createTable(name, HConstants.CATALOG_FAMILY);
268
269 TableName greaterName =
270 TableName.valueOf("testScanMetaForTablf");
271 UTIL.createTable(greaterName, HConstants.CATALOG_FAMILY);
272
273
274
275 assertEquals(1, MetaReader.getTableRegions(CT, name).size());
276 assertEquals(1, MetaReader.getTableRegions(CT, greaterName).size());
277 }
278
279 private static List<HRegionInfo> testGettingTableRegions(final CatalogTracker ct,
280 final TableName name, final int regionCount)
281 throws IOException, InterruptedException {
282 List<HRegionInfo> regions = MetaReader.getTableRegions(ct, name);
283 assertEquals(regionCount, regions.size());
284 Pair<HRegionInfo, ServerName> pair =
285 MetaReader.getRegion(ct, regions.get(0).getRegionName());
286 assertEquals(regions.get(0).getEncodedName(),
287 pair.getFirst().getEncodedName());
288 return regions;
289 }
290
291 private static void testGetRegion(final CatalogTracker ct,
292 final HRegionInfo region)
293 throws IOException, InterruptedException {
294 Pair<HRegionInfo, ServerName> pair =
295 MetaReader.getRegion(ct, region.getRegionName());
296 assertEquals(region.getEncodedName(),
297 pair.getFirst().getEncodedName());
298 }
299
300 @Test
301 public void testMastersSystemTimeIsUsedInMergeRegions() throws IOException {
302 long regionId = System.currentTimeMillis();
303 HRegionInfo regionInfoA = new HRegionInfo(TableName.valueOf("table_foo"),
304 HConstants.EMPTY_START_ROW, new byte[] {'a'}, false, regionId);
305 HRegionInfo regionInfoB = new HRegionInfo(TableName.valueOf("table_foo"),
306 new byte[] {'a'}, HConstants.EMPTY_END_ROW, false, regionId);
307 HRegionInfo mergedRegionInfo = new HRegionInfo(TableName.valueOf("table_foo"),
308 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, false, regionId);
309
310 ServerName sn = ServerName.valueOf("bar", 0, 0);
311 HTable meta = new HTable(UTIL.getConfiguration(), TableName.META_TABLE_NAME);
312 try {
313 MetaEditor.addRegionToMeta(meta, regionInfoA);
314 MetaEditor.addRegionToMeta(meta, regionInfoB);
315
316
317
318
319 long serverNameTime = EnvironmentEdgeManager.currentTimeMillis() + 100000000;
320 long masterSystemTime = EnvironmentEdgeManager.currentTimeMillis() + 123456789;
321
322
323 MetaEditor.updateRegionLocation(CT, regionInfoA, sn, 1, serverNameTime);
324
325
326 Get get = new Get(mergedRegionInfo.getRegionName());
327 Result result = meta.get(get);
328 Cell serverCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
329 HConstants.SERVER_QUALIFIER);
330 assertNotNull(serverCell);
331 assertEquals(serverNameTime, serverCell.getTimestamp());
332
333
334 MetaEditor.mergeRegions(CT, mergedRegionInfo,
335 regionInfoA, regionInfoB, sn, masterSystemTime);
336
337 result = meta.get(get);
338 serverCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
339 HConstants.SERVER_QUALIFIER);
340 Cell startCodeCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
341 HConstants.STARTCODE_QUALIFIER);
342 Cell seqNumCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
343 HConstants.SEQNUM_QUALIFIER);
344 assertNull(serverCell);
345 assertNull(startCodeCell);
346 assertNull(seqNumCell);
347 } finally {
348 meta.close();
349 }
350 }
351
352 @Test
353 public void testMastersSystemTimeIsUsedInUpdateLocations() throws IOException {
354 long regionId = System.currentTimeMillis();
355 HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf("table_foo"),
356 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, false, regionId);
357
358 ServerName sn = ServerName.valueOf("bar", 0, 0);
359 HTable meta = MetaReader.getMetaHTable(CT);
360 try {
361 List<HRegionInfo> regionInfos = Lists.newArrayList(regionInfo);
362 MetaEditor.addRegionsToMeta(CT, regionInfos, 1);
363
364 long masterSystemTime = EnvironmentEdgeManager.currentTimeMillis() + 123456789;
365 MetaEditor.updateRegionLocation(CT, regionInfo, sn, 1, masterSystemTime);
366
367 Get get = new Get(regionInfo.getRegionName());
368 Result result = meta.get(get);
369 Cell serverCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
370 HConstants.SERVER_QUALIFIER);
371 Cell startCodeCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
372 HConstants.STARTCODE_QUALIFIER);
373 Cell seqNumCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
374 HConstants.SEQNUM_QUALIFIER);
375
376 assertNotNull(serverCell);
377 assertNotNull(startCodeCell);
378 assertNotNull(seqNumCell);
379 assertTrue(serverCell.getValueLength() > 0);
380 assertTrue(startCodeCell.getValueLength() > 0);
381 assertTrue(seqNumCell.getValueLength() > 0);
382 assertEquals(masterSystemTime, serverCell.getTimestamp());
383 assertEquals(masterSystemTime, startCodeCell.getTimestamp());
384 assertEquals(masterSystemTime, seqNumCell.getTimestamp());
385 } finally {
386 meta.close();
387 }
388 }
389 }
390