1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.regionserver;
19
20 import static org.junit.Assert.assertArrayEquals;
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertTrue;
24
25 import java.io.IOException;
26 import java.util.ArrayList;
27 import java.util.List;
28 import java.util.Map;
29 import java.util.NavigableMap;
30 import java.util.Random;
31 import java.util.Set;
32
33 import org.apache.commons.io.IOUtils;
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.hbase.Chore;
38 import org.apache.hadoop.hbase.TableName;
39 import org.apache.hadoop.hbase.HBaseTestingUtility;
40 import org.apache.hadoop.hbase.HConstants;
41 import org.apache.hadoop.hbase.HRegionInfo;
42 import org.apache.hadoop.hbase.testclassification.LargeTests;
43 import org.apache.hadoop.hbase.NotServingRegionException;
44 import org.apache.hadoop.hbase.client.Admin;
45 import org.apache.hadoop.hbase.ipc.PayloadCarryingRpcController;
46 import org.apache.hadoop.hbase.ServerName;
47 import org.apache.hadoop.hbase.Stoppable;
48 import org.apache.hadoop.hbase.TableName;
49 import org.apache.hadoop.hbase.client.Admin;
50 import org.apache.hadoop.hbase.client.Connection;
51 import org.apache.hadoop.hbase.client.ConnectionFactory;
52 import org.apache.hadoop.hbase.client.Get;
53 import org.apache.hadoop.hbase.client.HConnection;
54 import org.apache.hadoop.hbase.client.HConnectionManager;
55 import org.apache.hadoop.hbase.client.HTable;
56 import org.apache.hadoop.hbase.client.MetaScanner;
57 import org.apache.hadoop.hbase.client.Put;
58 import org.apache.hadoop.hbase.client.Result;
59 import org.apache.hadoop.hbase.client.Scan;
60 import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager;
61 import org.apache.hadoop.hbase.client.Table;
62 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
63 import org.apache.hadoop.hbase.protobuf.RequestConverter;
64 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
65 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest;
66 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos;
67 import org.apache.hadoop.hbase.util.Bytes;
68 import org.apache.hadoop.hbase.util.ConfigUtil;
69 import org.apache.hadoop.hbase.util.Pair;
70 import org.apache.hadoop.hbase.util.PairOfSameType;
71 import org.apache.hadoop.hbase.util.StoppableImplementation;
72 import org.apache.hadoop.hbase.util.Threads;
73 import org.junit.AfterClass;
74 import org.junit.BeforeClass;
75 import org.junit.Test;
76 import org.junit.experimental.categories.Category;
77
78 import com.google.common.collect.Iterators;
79 import com.google.common.collect.Sets;
80 import com.google.protobuf.ServiceException;
81
82 @Category(LargeTests.class)
83 public class TestEndToEndSplitTransaction {
84 private static final Log LOG = LogFactory.getLog(TestEndToEndSplitTransaction.class);
85 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
86 private static final Configuration conf = TEST_UTIL.getConfiguration();
87
88 @BeforeClass
89 public static void beforeAllTests() throws Exception {
90 TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 5);
91 TEST_UTIL.startMiniCluster();
92 }
93
94 @AfterClass
95 public static void afterAllTests() throws Exception {
96 TEST_UTIL.shutdownMiniCluster();
97 }
98
99 @Test
100 public void testMasterOpsWhileSplitting() throws Exception {
101 TableName tableName =
102 TableName.valueOf("TestSplit");
103 byte[] familyName = Bytes.toBytes("fam");
104 try (HTable ht = TEST_UTIL.createTable(tableName, familyName)) {
105 TEST_UTIL.loadTable(ht, familyName, false);
106 }
107 HRegionServer server = TEST_UTIL.getHBaseCluster().getRegionServer(0);
108 byte []firstRow = Bytes.toBytes("aaa");
109 byte []splitRow = Bytes.toBytes("lll");
110 byte []lastRow = Bytes.toBytes("zzz");
111 HConnection con = HConnectionManager
112 .getConnection(TEST_UTIL.getConfiguration());
113
114 byte[] regionName = con.locateRegion(tableName, splitRow).getRegionInfo()
115 .getRegionName();
116 HRegion region = server.getRegion(regionName);
117 SplitTransaction split = new SplitTransaction(region, splitRow);
118 split.useZKForAssignment = ConfigUtil.useZKForAssignment(conf);
119 split.prepare();
120
121
122 PairOfSameType<HRegion> regions = split.createDaughters(server, server);
123 assertFalse(test(con, tableName, firstRow, server));
124 assertFalse(test(con, tableName, lastRow, server));
125
126
127
128 split.openDaughters(server, null, regions.getFirst(), regions.getSecond());
129 assertFalse(test(con, tableName, firstRow, server));
130 assertFalse(test(con, tableName, lastRow, server));
131
132
133
134
135 if (split.useZKForAssignment) {
136 server.postOpenDeployTasks(regions.getSecond());
137 } else {
138 server.reportRegionStateTransition(
139 RegionServerStatusProtos.RegionStateTransition.TransitionCode.SPLIT,
140 region.getRegionInfo(), regions.getFirst().getRegionInfo(),
141 regions.getSecond().getRegionInfo());
142 }
143
144
145 server.addToOnlineRegions(regions.getSecond());
146
147
148 assertFalse(test(con, tableName, firstRow, server));
149
150 assertTrue(test(con, tableName, lastRow, server));
151
152
153 if (split.useZKForAssignment) {
154 server.postOpenDeployTasks(regions.getFirst());
155 }
156
157 server.addToOnlineRegions(regions.getFirst());
158 assertTrue(test(con, tableName, firstRow, server));
159 assertTrue(test(con, tableName, lastRow, server));
160
161 if (split.useZKForAssignment) {
162
163 ((BaseCoordinatedStateManager) server.getCoordinatedStateManager())
164 .getSplitTransactionCoordination().completeSplitTransaction(server, regions.getFirst(),
165 regions.getSecond(), split.std, region);
166 }
167 assertTrue(test(con, tableName, firstRow, server));
168 assertTrue(test(con, tableName, lastRow, server));
169 }
170
171
172
173
174
175 private boolean test(HConnection con, TableName tableName, byte[] row,
176 HRegionServer server) {
177
178 try {
179 byte[] regionName = con.relocateRegion(tableName, row).getRegionInfo()
180 .getRegionName();
181
182 ClientProtos.GetRequest request =
183 RequestConverter.buildGetRequest(regionName, new Get(row));
184 server.getRSRpcServices().get(null, request);
185 ScanRequest scanRequest = RequestConverter.buildScanRequest(
186 regionName, new Scan(row), 1, true);
187 try {
188 server.getRSRpcServices().scan(
189 new PayloadCarryingRpcController(), scanRequest);
190 } catch (ServiceException se) {
191 throw ProtobufUtil.getRemoteException(se);
192 }
193 } catch (IOException x) {
194 return false;
195 } catch (ServiceException e) {
196 return false;
197 }
198 return true;
199 }
200
201
202
203
204 @Test
205 public void testFromClientSideWhileSplitting() throws Throwable {
206 LOG.info("Starting testFromClientSideWhileSplitting");
207 final TableName TABLENAME =
208 TableName.valueOf("testFromClientSideWhileSplitting");
209 final byte[] FAMILY = Bytes.toBytes("family");
210
211
212
213 Table table = TEST_UTIL.createTable(TABLENAME, FAMILY);
214
215 Stoppable stopper = new StoppableImplementation();
216 RegionSplitter regionSplitter = new RegionSplitter(table);
217 RegionChecker regionChecker = new RegionChecker(conf, stopper, TABLENAME);
218
219 regionChecker.start();
220 regionSplitter.start();
221
222
223 regionSplitter.join();
224 stopper.stop(null);
225
226 if (regionChecker.ex != null) {
227 throw regionChecker.ex;
228 }
229
230 if (regionSplitter.ex != null) {
231 throw regionSplitter.ex;
232 }
233
234
235 regionChecker.verify();
236 }
237
238 static class RegionSplitter extends Thread {
239 final Connection connection;
240 Throwable ex;
241 Table table;
242 TableName tableName;
243 byte[] family;
244 Admin admin;
245 HRegionServer rs;
246
247 RegionSplitter(Table table) throws IOException {
248 this.table = table;
249 this.tableName = table.getName();
250 this.family = table.getTableDescriptor().getFamiliesKeys().iterator().next();
251 admin = TEST_UTIL.getHBaseAdmin();
252 rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
253 connection = TEST_UTIL.getConnection();
254 }
255
256 @Override
257 public void run() {
258 try {
259 Random random = new Random();
260 for (int i= 0; i< 5; i++) {
261 NavigableMap<HRegionInfo, ServerName> regions =
262 MetaScanner.allTableRegions(connection, tableName);
263 if (regions.size() == 0) {
264 continue;
265 }
266 int regionIndex = random.nextInt(regions.size());
267
268
269 HRegionInfo region = Iterators.get(regions.keySet().iterator(), regionIndex);
270
271
272 int start = 0, end = Integer.MAX_VALUE;
273 if (region.getStartKey().length > 0) {
274 start = Bytes.toInt(region.getStartKey());
275 }
276 if (region.getEndKey().length > 0) {
277 end = Bytes.toInt(region.getEndKey());
278 }
279 int mid = start + ((end - start) / 2);
280 byte[] splitPoint = Bytes.toBytes(mid);
281
282
283 addData(start);
284 addData(mid);
285
286 flushAndBlockUntilDone(admin, rs, region.getRegionName());
287 compactAndBlockUntilDone(admin, rs, region.getRegionName());
288
289 log("Initiating region split for:" + region.getRegionNameAsString());
290 try {
291 admin.splitRegion(region.getRegionName(), splitPoint);
292
293 blockUntilRegionSplit(conf, 50000, region.getRegionName(), true);
294
295 } catch (NotServingRegionException ex) {
296
297 }
298 }
299 } catch (Throwable ex) {
300 this.ex = ex;
301 }
302 }
303
304 void addData(int start) throws IOException {
305 List<Put> puts = new ArrayList<>();
306 for (int i=start; i< start + 100; i++) {
307 Put put = new Put(Bytes.toBytes(i));
308 put.add(family, family, Bytes.toBytes(i));
309 puts.add(put);
310 }
311 table.put(puts);
312 }
313 }
314
315
316
317
318 static class RegionChecker extends Chore {
319 Connection connection;
320 Configuration conf;
321 TableName tableName;
322 Throwable ex;
323
324 RegionChecker(Configuration conf, Stoppable stopper, TableName tableName) throws IOException {
325 super("RegionChecker", 10, stopper);
326 this.conf = conf;
327 this.tableName = tableName;
328 this.setDaemon(true);
329
330 this.connection = ConnectionFactory.createConnection(conf);
331 }
332
333
334 void verifyRegionsUsingMetaScanner() throws Exception {
335
336
337 NavigableMap<HRegionInfo, ServerName> regions = MetaScanner.allTableRegions(connection,
338 tableName);
339 verifyTableRegions(regions.keySet());
340
341
342 List<HRegionInfo> regionList = MetaScanner.listAllRegions(conf, connection, false);
343 verifyTableRegions(Sets.newTreeSet(regionList));
344 }
345
346
347 void verifyRegionsUsingHTable() throws IOException {
348 HTable table = null;
349 try {
350
351 table = new HTable(conf, tableName);
352 Pair<byte[][], byte[][]> keys = table.getStartEndKeys();
353 verifyStartEndKeys(keys);
354
355
356 Map<HRegionInfo, ServerName> regions = table.getRegionLocations();
357 verifyTableRegions(regions.keySet());
358 } finally {
359 IOUtils.closeQuietly(table);
360 }
361 }
362
363 void verify() throws Exception {
364 verifyRegionsUsingMetaScanner();
365 verifyRegionsUsingHTable();
366 }
367
368 void verifyTableRegions(Set<HRegionInfo> regions) {
369 log("Verifying " + regions.size() + " regions: " + regions);
370
371 byte[][] startKeys = new byte[regions.size()][];
372 byte[][] endKeys = new byte[regions.size()][];
373
374 int i=0;
375 for (HRegionInfo region : regions) {
376 startKeys[i] = region.getStartKey();
377 endKeys[i] = region.getEndKey();
378 i++;
379 }
380
381 Pair<byte[][], byte[][]> keys = new Pair<byte[][], byte[][]>(startKeys, endKeys);
382 verifyStartEndKeys(keys);
383 }
384
385 void verifyStartEndKeys(Pair<byte[][], byte[][]> keys) {
386 byte[][] startKeys = keys.getFirst();
387 byte[][] endKeys = keys.getSecond();
388 assertEquals(startKeys.length, endKeys.length);
389 assertTrue("Found 0 regions for the table", startKeys.length > 0);
390
391 assertArrayEquals("Start key for the first region is not byte[0]",
392 HConstants.EMPTY_START_ROW, startKeys[0]);
393 byte[] prevEndKey = HConstants.EMPTY_START_ROW;
394
395
396 for (int i=0; i<startKeys.length; i++) {
397 assertArrayEquals(
398 "Hole in hbase:meta is detected. prevEndKey=" + Bytes.toStringBinary(prevEndKey)
399 + " ,regionStartKey=" + Bytes.toStringBinary(startKeys[i]), prevEndKey,
400 startKeys[i]);
401 prevEndKey = endKeys[i];
402 }
403 assertArrayEquals("End key for the last region is not byte[0]", HConstants.EMPTY_END_ROW,
404 endKeys[endKeys.length - 1]);
405 }
406
407 @Override
408 protected void chore() {
409 try {
410 verify();
411 } catch (Throwable ex) {
412 this.ex = ex;
413 stopper.stop("caught exception");
414 }
415 }
416 }
417
418 public static void log(String msg) {
419 LOG.info(msg);
420 }
421
422
423
424 public static void flushAndBlockUntilDone(Admin admin, HRegionServer rs, byte[] regionName)
425 throws IOException, InterruptedException {
426 log("flushing region: " + Bytes.toStringBinary(regionName));
427 admin.flushRegion(regionName);
428 log("blocking until flush is complete: " + Bytes.toStringBinary(regionName));
429 Threads.sleepWithoutInterrupt(500);
430 while (rs.cacheFlusher.getFlushQueueSize() > 0) {
431 Threads.sleep(50);
432 }
433 }
434
435 public static void compactAndBlockUntilDone(Admin admin, HRegionServer rs, byte[] regionName)
436 throws IOException, InterruptedException {
437 log("Compacting region: " + Bytes.toStringBinary(regionName));
438 admin.majorCompactRegion(regionName);
439 log("blocking until compaction is complete: " + Bytes.toStringBinary(regionName));
440 Threads.sleepWithoutInterrupt(500);
441 while (rs.compactSplitThread.getCompactionQueueSize() > 0) {
442 Threads.sleep(50);
443 }
444 }
445
446
447 public static void blockUntilRegionSplit(Configuration conf, long timeout,
448 final byte[] regionName, boolean waitForDaughters)
449 throws IOException, InterruptedException {
450 long start = System.currentTimeMillis();
451 log("blocking until region is split:" + Bytes.toStringBinary(regionName));
452 HRegionInfo daughterA = null, daughterB = null;
453 Table metaTable = new HTable(conf, TableName.META_TABLE_NAME);
454
455 try {
456 Result result = null;
457 HRegionInfo region = null;
458 while ((System.currentTimeMillis() - start) < timeout) {
459 result = getRegionRow(metaTable, regionName);
460 if (result == null) {
461 break;
462 }
463
464 region = HRegionInfo.getHRegionInfo(result);
465 if (region.isSplitParent()) {
466 log("found parent region: " + region.toString());
467 PairOfSameType<HRegionInfo> pair = HRegionInfo.getDaughterRegions(result);
468 daughterA = pair.getFirst();
469 daughterB = pair.getSecond();
470 break;
471 }
472 Threads.sleep(100);
473 }
474 if (daughterA == null || daughterB == null) {
475 throw new IOException("Failed to get daughters, daughterA=" + daughterA + ", daughterB=" +
476 daughterB + ", timeout=" + timeout + ", result=" + result + ", regionName=" + regionName +
477 ", region=" + region);
478 }
479
480
481 if (waitForDaughters) {
482 long rem = timeout - (System.currentTimeMillis() - start);
483 blockUntilRegionIsInMeta(metaTable, rem, daughterA);
484
485 rem = timeout - (System.currentTimeMillis() - start);
486 blockUntilRegionIsInMeta(metaTable, rem, daughterB);
487
488 rem = timeout - (System.currentTimeMillis() - start);
489 blockUntilRegionIsOpened(conf, rem, daughterA);
490
491 rem = timeout - (System.currentTimeMillis() - start);
492 blockUntilRegionIsOpened(conf, rem, daughterB);
493 }
494 } finally {
495 IOUtils.closeQuietly(metaTable);
496 }
497 }
498
499 public static Result getRegionRow(Table metaTable, byte[] regionName) throws IOException {
500 Get get = new Get(regionName);
501 return metaTable.get(get);
502 }
503
504 public static void blockUntilRegionIsInMeta(Table metaTable, long timeout, HRegionInfo hri)
505 throws IOException, InterruptedException {
506 log("blocking until region is in META: " + hri.getRegionNameAsString());
507 long start = System.currentTimeMillis();
508 while (System.currentTimeMillis() - start < timeout) {
509 Result result = getRegionRow(metaTable, hri.getRegionName());
510 if (result != null) {
511 HRegionInfo info = HRegionInfo.getHRegionInfo(result);
512 if (info != null && !info.isOffline()) {
513 log("found region in META: " + hri.getRegionNameAsString());
514 break;
515 }
516 }
517 Threads.sleep(10);
518 }
519 }
520
521 public static void blockUntilRegionIsOpened(Configuration conf, long timeout, HRegionInfo hri)
522 throws IOException, InterruptedException {
523 log("blocking until region is opened for reading:" + hri.getRegionNameAsString());
524 long start = System.currentTimeMillis();
525 Table table = new HTable(conf, hri.getTable());
526
527 try {
528 byte [] row = hri.getStartKey();
529
530 if (row == null || row.length <= 0) row = new byte [] {'0'};
531 Get get = new Get(row);
532 while (System.currentTimeMillis() - start < timeout) {
533 try {
534 table.get(get);
535 break;
536 } catch(IOException ex) {
537
538 }
539 Threads.sleep(10);
540 }
541 } finally {
542 IOUtils.closeQuietly(table);
543 }
544 }
545 }
546