1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.regionserver;
19
20 import static org.junit.Assert.assertArrayEquals;
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertTrue;
24
25 import java.io.IOException;
26 import java.util.List;
27 import java.util.Map;
28 import java.util.NavigableMap;
29 import java.util.Random;
30 import java.util.Set;
31
32 import org.apache.commons.io.IOUtils;
33 import org.apache.commons.logging.Log;
34 import org.apache.commons.logging.LogFactory;
35 import org.apache.hadoop.conf.Configuration;
36 import org.apache.hadoop.hbase.Chore;
37 import org.apache.hadoop.hbase.HBaseTestingUtility;
38 import org.apache.hadoop.hbase.HConstants;
39 import org.apache.hadoop.hbase.HRegionInfo;
40 import org.apache.hadoop.hbase.HServerAddress;
41 import org.apache.hadoop.hbase.LargeTests;
42 import org.apache.hadoop.hbase.NotServingRegionException;
43 import org.apache.hadoop.hbase.ServerName;
44 import org.apache.hadoop.hbase.Stoppable;
45 import org.apache.hadoop.hbase.catalog.MetaEditor;
46 import org.apache.hadoop.hbase.catalog.MetaReader;
47 import org.apache.hadoop.hbase.client.Get;
48 import org.apache.hadoop.hbase.client.HBaseAdmin;
49 import org.apache.hadoop.hbase.client.HConnection;
50 import org.apache.hadoop.hbase.client.HConnectionManager;
51 import org.apache.hadoop.hbase.client.HTable;
52 import org.apache.hadoop.hbase.client.MetaScanner;
53 import org.apache.hadoop.hbase.client.Put;
54 import org.apache.hadoop.hbase.client.Result;
55 import org.apache.hadoop.hbase.client.Scan;
56 import org.apache.hadoop.hbase.util.Bytes;
57 import org.apache.hadoop.hbase.util.Pair;
58 import org.apache.hadoop.hbase.util.PairOfSameType;
59 import org.apache.hadoop.hbase.util.StoppableImplementation;
60 import org.apache.hadoop.hbase.util.Threads;
61 import org.junit.AfterClass;
62 import org.junit.BeforeClass;
63 import org.junit.Test;
64 import org.junit.experimental.categories.Category;
65
66 import com.google.common.collect.Iterators;
67 import com.google.common.collect.Sets;
68
69 @Category(LargeTests.class)
70 public class TestEndToEndSplitTransaction {
71 private static final Log LOG = LogFactory.getLog(TestEndToEndSplitTransaction.class);
72 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
73 private static final Configuration conf = TEST_UTIL.getConfiguration();
74
75 @BeforeClass
76 public static void beforeAllTests() throws Exception {
77 TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 5);
78 TEST_UTIL.startMiniCluster();
79 }
80
81 @AfterClass
82 public static void afterAllTests() throws Exception {
83 TEST_UTIL.shutdownMiniCluster();
84 }
85
86 @Test
87 public void testMasterOpsWhileSplitting() throws Exception {
88 byte[] tableName = Bytes.toBytes("TestSplit");
89 byte[] familyName = Bytes.toBytes("fam");
90 HTable ht = TEST_UTIL.createTable(tableName, familyName);
91 TEST_UTIL.loadTable(ht, familyName);
92 ht.close();
93 HRegionServer server = TEST_UTIL.getHBaseCluster().getRegionServer(0);
94 byte []firstRow = Bytes.toBytes("aaa");
95 byte []splitRow = Bytes.toBytes("lll");
96 byte []lastRow = Bytes.toBytes("zzz");
97 HConnection con = HConnectionManager
98 .getConnection(TEST_UTIL.getConfiguration());
99
100 byte[] regionName = con.locateRegion(tableName, splitRow).getRegionInfo()
101 .getRegionName();
102 HRegion region = server.getRegion(regionName);
103 SplitTransaction split = new SplitTransaction(region, splitRow);
104 split.prepare();
105
106
107 PairOfSameType<HRegion> regions = split.createDaughters(server, server);
108 assertFalse(test(con, tableName, firstRow, server));
109 assertFalse(test(con, tableName, lastRow, server));
110
111
112
113 split.openDaughters(server, null, regions.getFirst(), regions.getSecond());
114 assertFalse(test(con, tableName, firstRow, server));
115 assertFalse(test(con, tableName, lastRow, server));
116
117
118
119
120 server.postOpenDeployTasks(regions.getSecond(), server.getCatalogTracker(), true);
121
122 server.addToOnlineRegions(regions.getSecond());
123
124
125 assertFalse(test(con, tableName, firstRow, server));
126
127 assertTrue(test(con, tableName, lastRow, server));
128
129
130 server.postOpenDeployTasks(regions.getFirst(), server.getCatalogTracker(), true);
131
132 server.addToOnlineRegions(regions.getFirst());
133 assertTrue(test(con, tableName, firstRow, server));
134 assertTrue(test(con, tableName, lastRow, server));
135
136
137 split.transitionZKNode(server, server, regions.getFirst(),
138 regions.getSecond());
139 assertTrue(test(con, tableName, firstRow, server));
140 assertTrue(test(con, tableName, lastRow, server));
141 }
142
143
144
145
146
147 private boolean test(HConnection con, byte[] tableName, byte[] row,
148 HRegionServer server) {
149
150 try {
151 byte[] regionName = con.relocateRegion(tableName, row).getRegionInfo()
152 .getRegionName();
153
154 server.get(regionName, new Get(row));
155 server.openScanner(regionName, new Scan(row));
156 } catch (IOException x) {
157 return false;
158 }
159 return true;
160 }
161
162
163
164
165 @Test
166 public void testFromClientSideWhileSplitting() throws Throwable {
167 LOG.info("Starting testFromClientSideWhileSplitting");
168 final byte[] TABLENAME = Bytes.toBytes("testFromClientSideWhileSplitting");
169 final byte[] FAMILY = Bytes.toBytes("family");
170
171
172
173 HTable table = TEST_UTIL.createTable(TABLENAME, FAMILY);
174
175 Stoppable stopper = new StoppableImplementation();
176 RegionSplitter regionSplitter = new RegionSplitter(table);
177 RegionChecker regionChecker = new RegionChecker(conf, stopper, TABLENAME);
178
179 regionChecker.start();
180 regionSplitter.start();
181
182
183 regionSplitter.join();
184 stopper.stop(null);
185
186 if (regionChecker.ex != null) {
187 throw regionChecker.ex;
188 }
189
190 if (regionSplitter.ex != null) {
191 throw regionSplitter.ex;
192 }
193
194
195 regionChecker.verify();
196 }
197
198 static class RegionSplitter extends Thread {
199 Throwable ex;
200 HTable table;
201 byte[] tableName, family;
202 HBaseAdmin admin;
203 HRegionServer rs;
204
205 RegionSplitter(HTable table) throws IOException {
206 this.table = table;
207 this.tableName = table.getTableName();
208 this.family = table.getTableDescriptor().getFamiliesKeys().iterator().next();
209 admin = TEST_UTIL.getHBaseAdmin();
210 rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
211 }
212
213 public void run() {
214 try {
215 Random random = new Random();
216 for (int i=0; i< 5; i++) {
217 NavigableMap<HRegionInfo, ServerName> regions = MetaScanner.allTableRegions(conf, tableName, false);
218 if (regions.size() == 0) {
219 continue;
220 }
221 int regionIndex = random.nextInt(regions.size());
222
223
224 HRegionInfo region = Iterators.get(regions.keySet().iterator(), regionIndex);
225
226
227 int start = 0, end = Integer.MAX_VALUE;
228 if (region.getStartKey().length > 0) {
229 start = Bytes.toInt(region.getStartKey());
230 }
231 if (region.getEndKey().length > 0) {
232 end = Bytes.toInt(region.getEndKey());
233 }
234 int mid = start + ((end - start) / 2);
235 byte[] splitPoint = Bytes.toBytes(mid);
236
237
238 addData(start);
239 addData(mid);
240
241 flushAndBlockUntilDone(admin, rs, region.getRegionName());
242 compactAndBlockUntilDone(admin, rs, region.getRegionName());
243
244 log("Initiating region split for:" + region.getRegionNameAsString());
245 try {
246 admin.split(region.getRegionName(), splitPoint);
247
248 blockUntilRegionSplit(conf, 50000, region.getRegionName(), true);
249
250 } catch (NotServingRegionException ex) {
251
252 }
253 }
254 } catch (Throwable ex) {
255 this.ex = ex;
256 }
257 }
258
259 void addData(int start) throws IOException {
260 for (int i=start; i< start + 100; i++) {
261 Put put = new Put(Bytes.toBytes(i));
262
263 put.add(family, family, Bytes.toBytes(i));
264 table.put(put);
265 }
266 table.flushCommits();
267 }
268 }
269
270
271
272
273 static class RegionChecker extends Chore {
274 Configuration conf;
275 byte[] tableName;
276 Throwable ex;
277
278 RegionChecker(Configuration conf, Stoppable stopper, byte[] tableName) {
279 super("RegionChecker", 10, stopper);
280 this.conf = conf;
281 this.tableName = tableName;
282 this.setDaemon(true);
283 }
284
285
286 void verifyRegionsUsingMetaScanner() throws Exception {
287
288
289 NavigableMap<HRegionInfo, ServerName> regions = MetaScanner.allTableRegions(conf, tableName,
290 false);
291 verifyTableRegions(regions.keySet());
292
293
294 List<HRegionInfo> regionList = MetaScanner.listAllRegions(conf, false);
295 verifyTableRegions(Sets.newTreeSet(regionList));
296 }
297
298
299 void verifyRegionsUsingHTable() throws IOException {
300 HTable table = null;
301 try {
302
303 table = new HTable(conf, tableName);
304 Pair<byte[][], byte[][]> keys = table.getStartEndKeys();
305 verifyStartEndKeys(keys);
306
307
308 Map<HRegionInfo, HServerAddress> regions = table.getRegionsInfo();
309 verifyTableRegions(regions.keySet());
310 } finally {
311 IOUtils.closeQuietly(table);
312 }
313 }
314
315 void verify() throws Exception {
316 verifyRegionsUsingMetaScanner();
317 verifyRegionsUsingHTable();
318 }
319
320 void verifyTableRegions(Set<HRegionInfo> regions) {
321 log("Verifying " + regions.size() + " regions");
322
323 byte[][] startKeys = new byte[regions.size()][];
324 byte[][] endKeys = new byte[regions.size()][];
325
326 int i=0;
327 for (HRegionInfo region : regions) {
328 startKeys[i] = region.getStartKey();
329 endKeys[i] = region.getEndKey();
330 i++;
331 }
332
333 Pair<byte[][], byte[][]> keys = new Pair<byte[][], byte[][]>(startKeys, endKeys);
334 verifyStartEndKeys(keys);
335 }
336
337 void verifyStartEndKeys(Pair<byte[][], byte[][]> keys) {
338 byte[][] startKeys = keys.getFirst();
339 byte[][] endKeys = keys.getSecond();
340 assertEquals(startKeys.length, endKeys.length);
341 assertTrue("Found 0 regions for the table", startKeys.length > 0);
342
343 assertArrayEquals("Start key for the first region is not byte[0]",
344 HConstants.EMPTY_START_ROW, startKeys[0]);
345 byte[] prevEndKey = HConstants.EMPTY_START_ROW;
346
347
348 for (int i=0; i<startKeys.length; i++) {
349 assertArrayEquals(
350 "Hole in .META. is detected. prevEndKey=" + Bytes.toStringBinary(prevEndKey)
351 + " ,regionStartKey=" + Bytes.toStringBinary(startKeys[i]), prevEndKey,
352 startKeys[i]);
353 prevEndKey = endKeys[i];
354 }
355 assertArrayEquals("End key for the last region is not byte[0]", HConstants.EMPTY_END_ROW,
356 endKeys[endKeys.length - 1]);
357 }
358
359 @Override
360 protected void chore() {
361 try {
362 verify();
363 } catch (Throwable ex) {
364 this.ex = ex;
365 stopper.stop("caught exception");
366 }
367 }
368 }
369
370 public static void log(String msg) {
371 LOG.info(msg);
372 }
373
374
375
376 public static void flushAndBlockUntilDone(HBaseAdmin admin, HRegionServer rs, byte[] regionName)
377 throws IOException, InterruptedException {
378 log("flushing region: " + Bytes.toStringBinary(regionName));
379 admin.flush(regionName);
380 log("blocking until flush is complete: " + Bytes.toStringBinary(regionName));
381 Threads.sleepWithoutInterrupt(500);
382 while (rs.cacheFlusher.getFlushQueueSize() > 0) {
383 Threads.sleep(50);
384 }
385 }
386
387 public static void compactAndBlockUntilDone(HBaseAdmin admin, HRegionServer rs, byte[] regionName)
388 throws IOException, InterruptedException {
389 log("Compacting region: " + Bytes.toStringBinary(regionName));
390 admin.majorCompact(regionName);
391 log("blocking until compaction is complete: " + Bytes.toStringBinary(regionName));
392 Threads.sleepWithoutInterrupt(500);
393 while (rs.compactSplitThread.getCompactionQueueSize() > 0) {
394 Threads.sleep(50);
395 }
396 }
397
398
399 public static void blockUntilRegionSplit(Configuration conf, long timeout,
400 final byte[] regionName, boolean waitForDaughters)
401 throws IOException, InterruptedException {
402 long start = System.currentTimeMillis();
403 log("blocking until region is split:" + Bytes.toStringBinary(regionName));
404 HRegionInfo daughterA = null, daughterB = null;
405 HTable metaTable = new HTable(conf, HConstants.META_TABLE_NAME);
406
407 try {
408 while (System.currentTimeMillis() - start < timeout) {
409 Result result = getRegionRow(metaTable, regionName);
410 if (result == null) {
411 break;
412 }
413
414 HRegionInfo region = MetaReader.parseCatalogResult(result).getFirst();
415 if(region.isSplitParent()) {
416 log("found parent region: " + region.toString());
417 PairOfSameType<HRegionInfo> pair = MetaReader.getDaughterRegions(result);
418 daughterA = pair.getFirst();
419 daughterB = pair.getSecond();
420 break;
421 }
422 Threads.sleep(100);
423 }
424
425
426 if (waitForDaughters) {
427 long rem = timeout - (System.currentTimeMillis() - start);
428 blockUntilRegionIsInMeta(metaTable, rem, daughterA);
429
430 rem = timeout - (System.currentTimeMillis() - start);
431 blockUntilRegionIsInMeta(metaTable, rem, daughterB);
432
433 rem = timeout - (System.currentTimeMillis() - start);
434 blockUntilRegionIsOpened(conf, rem, daughterA);
435
436 rem = timeout - (System.currentTimeMillis() - start);
437 blockUntilRegionIsOpened(conf, rem, daughterB);
438 }
439 } finally {
440 IOUtils.closeQuietly(metaTable);
441 }
442 }
443
444 public static Result getRegionRow(HTable metaTable, byte[] regionName) throws IOException {
445 Get get = new Get(regionName);
446 return metaTable.get(get);
447 }
448
449 public static void blockUntilRegionIsInMeta(HTable metaTable, long timeout, HRegionInfo hri)
450 throws IOException, InterruptedException {
451 log("blocking until region is in META: " + hri.getRegionNameAsString());
452 long start = System.currentTimeMillis();
453 while (System.currentTimeMillis() - start < timeout) {
454 Result result = getRegionRow(metaTable, hri.getRegionName());
455 if (result != null) {
456 HRegionInfo info = MetaReader.parseCatalogResult(result).getFirst();
457 if (info != null && !info.isOffline()) {
458 log("found region in META: " + hri.getRegionNameAsString());
459 break;
460 }
461 }
462 Threads.sleep(10);
463 }
464 }
465
466 public static void blockUntilRegionIsOpened(Configuration conf, long timeout, HRegionInfo hri)
467 throws IOException, InterruptedException {
468 log("blocking until region is opened for reading:" + hri.getRegionNameAsString());
469 long start = System.currentTimeMillis();
470 HTable table = new HTable(conf, hri.getTableName());
471
472 try {
473 Get get = new Get(hri.getStartKey());
474 while (System.currentTimeMillis() - start < timeout) {
475 try {
476 table.get(get);
477 break;
478 } catch(IOException ex) {
479
480 }
481 Threads.sleep(10);
482 }
483 } finally {
484 IOUtils.closeQuietly(table);
485 }
486 }
487
488 @org.junit.Rule
489 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
490 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
491 }
492