1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.regionserver;
19
20 import static org.junit.Assert.assertArrayEquals;
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertTrue;
24
25 import java.io.IOException;
26 import java.util.List;
27 import java.util.Map;
28 import java.util.NavigableMap;
29 import java.util.Random;
30 import java.util.Set;
31
32 import org.apache.commons.io.IOUtils;
33 import org.apache.commons.logging.Log;
34 import org.apache.commons.logging.LogFactory;
35 import org.apache.hadoop.conf.Configuration;
36 import org.apache.hadoop.hbase.Chore;
37 import org.apache.hadoop.hbase.HBaseTestingUtility;
38 import org.apache.hadoop.hbase.HConstants;
39 import org.apache.hadoop.hbase.HRegionInfo;
40 import org.apache.hadoop.hbase.HServerAddress;
41 import org.apache.hadoop.hbase.LargeTests;
42 import org.apache.hadoop.hbase.NotServingRegionException;
43 import org.apache.hadoop.hbase.ServerName;
44 import org.apache.hadoop.hbase.Stoppable;
45 import org.apache.hadoop.hbase.catalog.MetaEditor;
46 import org.apache.hadoop.hbase.catalog.MetaReader;
47 import org.apache.hadoop.hbase.client.Get;
48 import org.apache.hadoop.hbase.client.HBaseAdmin;
49 import org.apache.hadoop.hbase.client.HConnection;
50 import org.apache.hadoop.hbase.client.HConnectionManager;
51 import org.apache.hadoop.hbase.client.HTable;
52 import org.apache.hadoop.hbase.client.MetaScanner;
53 import org.apache.hadoop.hbase.client.Put;
54 import org.apache.hadoop.hbase.client.Result;
55 import org.apache.hadoop.hbase.client.Scan;
56 import org.apache.hadoop.hbase.util.Bytes;
57 import org.apache.hadoop.hbase.util.Pair;
58 import org.apache.hadoop.hbase.util.PairOfSameType;
59 import org.apache.hadoop.hbase.util.StoppableImplementation;
60 import org.apache.hadoop.hbase.util.Threads;
61 import org.junit.AfterClass;
62 import org.junit.BeforeClass;
63 import org.junit.Test;
64 import org.junit.experimental.categories.Category;
65
66 import com.google.common.collect.Iterators;
67 import com.google.common.collect.Sets;
68
69 @Category(LargeTests.class)
70 public class TestEndToEndSplitTransaction {
71 private static final Log LOG = LogFactory.getLog(TestEndToEndSplitTransaction.class);
72 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
73 private static final Configuration conf = TEST_UTIL.getConfiguration();
74
75 @BeforeClass
76 public static void beforeAllTests() throws Exception {
77 TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 5);
78 TEST_UTIL.startMiniCluster();
79 }
80
81 @AfterClass
82 public static void afterAllTests() throws Exception {
83 TEST_UTIL.shutdownMiniCluster();
84 }
85
86 @Test
87 public void testMasterOpsWhileSplitting() throws Exception {
88 byte[] tableName = Bytes.toBytes("TestSplit");
89 byte[] familyName = Bytes.toBytes("fam");
90 HTable ht = TEST_UTIL.createTable(tableName, familyName);
91 TEST_UTIL.loadTable(ht, familyName);
92 ht.close();
93 HRegionServer server = TEST_UTIL.getHBaseCluster().getRegionServer(0);
94 byte []firstRow = Bytes.toBytes("aaa");
95 byte []splitRow = Bytes.toBytes("lll");
96 byte []lastRow = Bytes.toBytes("zzz");
97 HConnection con = HConnectionManager
98 .getConnection(TEST_UTIL.getConfiguration());
99
100 byte[] regionName = con.locateRegion(tableName, splitRow).getRegionInfo()
101 .getRegionName();
102 HRegion region = server.getRegion(regionName);
103 SplitTransaction split = new SplitTransaction(region, splitRow);
104 split.prepare();
105
106
107 PairOfSameType<HRegion> regions = split.createDaughters(server, server);
108 assertFalse(test(con, tableName, firstRow, server));
109 assertFalse(test(con, tableName, lastRow, server));
110
111
112
113 split.openDaughters(server, null, regions.getFirst(), regions.getSecond());
114 assertFalse(test(con, tableName, firstRow, server));
115 assertFalse(test(con, tableName, lastRow, server));
116
117
118
119
120 server.postOpenDeployTasks(regions.getSecond(), server.getCatalogTracker(), true);
121
122 server.addToOnlineRegions(regions.getSecond());
123
124
125 assertFalse(test(con, tableName, firstRow, server));
126
127 assertTrue(test(con, tableName, lastRow, server));
128
129
130 server.postOpenDeployTasks(regions.getFirst(), server.getCatalogTracker(), true);
131
132 server.addToOnlineRegions(regions.getFirst());
133 assertTrue(test(con, tableName, firstRow, server));
134 assertTrue(test(con, tableName, lastRow, server));
135
136
137 split.transitionZKNode(server, server, regions.getFirst(),
138 regions.getSecond());
139 assertTrue(test(con, tableName, firstRow, server));
140 assertTrue(test(con, tableName, lastRow, server));
141 }
142
143
144
145
146
147 private boolean test(HConnection con, byte[] tableName, byte[] row,
148 HRegionServer server) {
149
150 try {
151 byte[] regionName = con.relocateRegion(tableName, row).getRegionInfo()
152 .getRegionName();
153
154 server.get(regionName, new Get(row));
155 server.openScanner(regionName, new Scan(row));
156 } catch (IOException x) {
157 return false;
158 }
159 return true;
160 }
161
162
163
164
165 @Test
166 public void testFromClientSideWhileSplitting() throws Throwable {
167 LOG.info("Starting testFromClientSideWhileSplitting");
168 final byte[] TABLENAME = Bytes.toBytes("testFromClientSideWhileSplitting");
169 final byte[] FAMILY = Bytes.toBytes("family");
170
171
172
173 HTable table = TEST_UTIL.createTable(TABLENAME, FAMILY);
174
175 Stoppable stopper = new StoppableImplementation();
176 RegionSplitter regionSplitter = new RegionSplitter(table);
177 RegionChecker regionChecker = new RegionChecker(conf, stopper, TABLENAME);
178
179 regionChecker.start();
180 regionSplitter.start();
181
182
183 regionSplitter.join();
184 stopper.stop(null);
185
186 if (regionChecker.ex != null) {
187 throw regionChecker.ex;
188 }
189
190 if (regionSplitter.ex != null) {
191 throw regionSplitter.ex;
192 }
193
194
195 regionChecker.verify();
196 }
197
198 static class RegionSplitter extends Thread {
199 Throwable ex;
200 HTable table;
201 byte[] tableName, family;
202 HBaseAdmin admin;
203 HRegionServer rs;
204
205 RegionSplitter(HTable table) throws IOException {
206 this.table = table;
207 this.tableName = table.getTableName();
208 this.family = table.getTableDescriptor().getFamiliesKeys().iterator().next();
209 admin = TEST_UTIL.getHBaseAdmin();
210 rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
211 }
212
213 public void run() {
214 try {
215 Random random = new Random();
216 for (int i=0; i< 5; i++) {
217 NavigableMap<HRegionInfo, ServerName> regions = MetaScanner.allTableRegions(conf, null,
218 tableName, false);
219 if (regions.size() == 0) {
220 continue;
221 }
222 int regionIndex = random.nextInt(regions.size());
223
224
225 HRegionInfo region = Iterators.get(regions.keySet().iterator(), regionIndex);
226
227
228 int start = 0, end = Integer.MAX_VALUE;
229 if (region.getStartKey().length > 0) {
230 start = Bytes.toInt(region.getStartKey());
231 }
232 if (region.getEndKey().length > 0) {
233 end = Bytes.toInt(region.getEndKey());
234 }
235 int mid = start + ((end - start) / 2);
236 byte[] splitPoint = Bytes.toBytes(mid);
237
238
239 addData(start);
240 addData(mid);
241
242 flushAndBlockUntilDone(admin, rs, region.getRegionName());
243 compactAndBlockUntilDone(admin, rs, region.getRegionName());
244
245 log("Initiating region split for:" + region.getRegionNameAsString());
246 try {
247 admin.split(region.getRegionName(), splitPoint);
248
249 blockUntilRegionSplit(conf, 50000, region.getRegionName(), true);
250
251 } catch (NotServingRegionException ex) {
252
253 }
254 }
255 } catch (Throwable ex) {
256 this.ex = ex;
257 }
258 }
259
260 void addData(int start) throws IOException {
261 for (int i=start; i< start + 100; i++) {
262 Put put = new Put(Bytes.toBytes(i));
263
264 put.add(family, family, Bytes.toBytes(i));
265 table.put(put);
266 }
267 table.flushCommits();
268 }
269 }
270
271
272
273
274 static class RegionChecker extends Chore {
275 Configuration conf;
276 byte[] tableName;
277 Throwable ex;
278
279 RegionChecker(Configuration conf, Stoppable stopper, byte[] tableName) {
280 super("RegionChecker", 10, stopper);
281 this.conf = conf;
282 this.tableName = tableName;
283 this.setDaemon(true);
284 }
285
286
287 void verifyRegionsUsingMetaScanner() throws Exception {
288
289
290 NavigableMap<HRegionInfo, ServerName> regions = MetaScanner.allTableRegions(conf, null,
291 tableName, false);
292 verifyTableRegions(regions.keySet());
293
294
295 List<HRegionInfo> regionList = MetaScanner.listAllRegions(conf, false);
296 verifyTableRegions(Sets.newTreeSet(regionList));
297 }
298
299
300 void verifyRegionsUsingHTable() throws IOException {
301 HTable table = null;
302 try {
303
304 table = new HTable(conf, tableName);
305 Pair<byte[][], byte[][]> keys = table.getStartEndKeys();
306 verifyStartEndKeys(keys);
307
308
309 Map<HRegionInfo, HServerAddress> regions = table.getRegionsInfo();
310 verifyTableRegions(regions.keySet());
311 } finally {
312 IOUtils.closeQuietly(table);
313 }
314 }
315
316 void verify() throws Exception {
317 verifyRegionsUsingMetaScanner();
318 verifyRegionsUsingHTable();
319 }
320
321 void verifyTableRegions(Set<HRegionInfo> regions) {
322 log("Verifying " + regions.size() + " regions");
323
324 byte[][] startKeys = new byte[regions.size()][];
325 byte[][] endKeys = new byte[regions.size()][];
326
327 int i=0;
328 for (HRegionInfo region : regions) {
329 startKeys[i] = region.getStartKey();
330 endKeys[i] = region.getEndKey();
331 i++;
332 }
333
334 Pair<byte[][], byte[][]> keys = new Pair<byte[][], byte[][]>(startKeys, endKeys);
335 verifyStartEndKeys(keys);
336 }
337
338 void verifyStartEndKeys(Pair<byte[][], byte[][]> keys) {
339 byte[][] startKeys = keys.getFirst();
340 byte[][] endKeys = keys.getSecond();
341 assertEquals(startKeys.length, endKeys.length);
342 assertTrue("Found 0 regions for the table", startKeys.length > 0);
343
344 assertArrayEquals("Start key for the first region is not byte[0]",
345 HConstants.EMPTY_START_ROW, startKeys[0]);
346 byte[] prevEndKey = HConstants.EMPTY_START_ROW;
347
348
349 for (int i=0; i<startKeys.length; i++) {
350 assertArrayEquals(
351 "Hole in .META. is detected. prevEndKey=" + Bytes.toStringBinary(prevEndKey)
352 + " ,regionStartKey=" + Bytes.toStringBinary(startKeys[i]), prevEndKey,
353 startKeys[i]);
354 prevEndKey = endKeys[i];
355 }
356 assertArrayEquals("End key for the last region is not byte[0]", HConstants.EMPTY_END_ROW,
357 endKeys[endKeys.length - 1]);
358 }
359
360 @Override
361 protected void chore() {
362 try {
363 verify();
364 } catch (Throwable ex) {
365 this.ex = ex;
366 stopper.stop("caught exception");
367 }
368 }
369 }
370
371 public static void log(String msg) {
372 LOG.info(msg);
373 }
374
375
376
377 public static void flushAndBlockUntilDone(HBaseAdmin admin, HRegionServer rs, byte[] regionName)
378 throws IOException, InterruptedException {
379 log("flushing region: " + Bytes.toStringBinary(regionName));
380 admin.flush(regionName);
381 log("blocking until flush is complete: " + Bytes.toStringBinary(regionName));
382 Threads.sleepWithoutInterrupt(500);
383 while (rs.cacheFlusher.getFlushQueueSize() > 0) {
384 Threads.sleep(50);
385 }
386 }
387
388 public static void compactAndBlockUntilDone(HBaseAdmin admin, HRegionServer rs, byte[] regionName)
389 throws IOException, InterruptedException {
390 log("Compacting region: " + Bytes.toStringBinary(regionName));
391 admin.majorCompact(regionName);
392 log("blocking until compaction is complete: " + Bytes.toStringBinary(regionName));
393 Threads.sleepWithoutInterrupt(500);
394 while (rs.compactSplitThread.getCompactionQueueSize() > 0) {
395 Threads.sleep(50);
396 }
397 }
398
399
400 public static void blockUntilRegionSplit(Configuration conf, long timeout,
401 final byte[] regionName, boolean waitForDaughters)
402 throws IOException, InterruptedException {
403 long start = System.currentTimeMillis();
404 log("blocking until region is split:" + Bytes.toStringBinary(regionName));
405 HRegionInfo daughterA = null, daughterB = null;
406 HTable metaTable = new HTable(conf, HConstants.META_TABLE_NAME);
407
408 try {
409 while (System.currentTimeMillis() - start < timeout) {
410 Result result = getRegionRow(metaTable, regionName);
411 if (result == null) {
412 break;
413 }
414
415 HRegionInfo region = MetaReader.parseCatalogResult(result).getFirst();
416 if(region.isSplitParent()) {
417 log("found parent region: " + region.toString());
418 PairOfSameType<HRegionInfo> pair = MetaReader.getDaughterRegions(result);
419 daughterA = pair.getFirst();
420 daughterB = pair.getSecond();
421 break;
422 }
423 Threads.sleep(100);
424 }
425
426
427 if (waitForDaughters) {
428 long rem = timeout - (System.currentTimeMillis() - start);
429 blockUntilRegionIsInMeta(metaTable, rem, daughterA);
430
431 rem = timeout - (System.currentTimeMillis() - start);
432 blockUntilRegionIsInMeta(metaTable, rem, daughterB);
433
434 rem = timeout - (System.currentTimeMillis() - start);
435 blockUntilRegionIsOpened(conf, rem, daughterA);
436
437 rem = timeout - (System.currentTimeMillis() - start);
438 blockUntilRegionIsOpened(conf, rem, daughterB);
439 }
440 } finally {
441 IOUtils.closeQuietly(metaTable);
442 }
443 }
444
445 public static Result getRegionRow(HTable metaTable, byte[] regionName) throws IOException {
446 Get get = new Get(regionName);
447 return metaTable.get(get);
448 }
449
450 public static void blockUntilRegionIsInMeta(HTable metaTable, long timeout, HRegionInfo hri)
451 throws IOException, InterruptedException {
452 log("blocking until region is in META: " + hri.getRegionNameAsString());
453 long start = System.currentTimeMillis();
454 while (System.currentTimeMillis() - start < timeout) {
455 Result result = getRegionRow(metaTable, hri.getRegionName());
456 if (result != null) {
457 HRegionInfo info = MetaReader.parseCatalogResult(result).getFirst();
458 if (info != null && !info.isOffline()) {
459 log("found region in META: " + hri.getRegionNameAsString());
460 break;
461 }
462 }
463 Threads.sleep(10);
464 }
465 }
466
467 public static void blockUntilRegionIsOpened(Configuration conf, long timeout, HRegionInfo hri)
468 throws IOException, InterruptedException {
469 log("blocking until region is opened for reading:" + hri.getRegionNameAsString());
470 long start = System.currentTimeMillis();
471 HTable table = new HTable(conf, hri.getTableName());
472
473 try {
474 Get get = new Get(hri.getStartKey());
475 while (System.currentTimeMillis() - start < timeout) {
476 try {
477 table.get(get);
478 break;
479 } catch(IOException ex) {
480
481 }
482 Threads.sleep(10);
483 }
484 } finally {
485 IOUtils.closeQuietly(table);
486 }
487 }
488
489 @org.junit.Rule
490 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
491 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
492 }
493