1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import java.io.IOException;
22 import java.util.Random;
23 import java.util.concurrent.ExecutorService;
24 import java.util.concurrent.Executors;
25 import java.util.concurrent.TimeUnit;
26 import java.util.concurrent.atomic.AtomicBoolean;
27 import java.util.concurrent.atomic.AtomicReference;
28
29 import org.apache.commons.logging.Log;
30 import org.apache.commons.logging.LogFactory;
31 import org.apache.hadoop.hbase.Cell;
32 import org.apache.hadoop.hbase.HBaseTestingUtility;
33 import org.apache.hadoop.hbase.HConstants;
34 import org.apache.hadoop.hbase.HRegionInfo;
35 import org.apache.hadoop.hbase.testclassification.MediumTests;
36 import org.apache.hadoop.hbase.NotServingRegionException;
37 import org.apache.hadoop.hbase.TableName;
38 import org.apache.hadoop.hbase.TestMetaTableAccessor;
39 import org.apache.hadoop.hbase.client.Consistency;
40 import org.apache.hadoop.hbase.client.Get;
41 import org.apache.hadoop.hbase.client.HTable;
42 import org.apache.hadoop.hbase.client.Put;
43 import org.apache.hadoop.hbase.client.Result;
44 import org.apache.hadoop.hbase.client.Table;
45 import org.apache.hadoop.hbase.io.hfile.HFileScanner;
46 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
47 import org.apache.hadoop.hbase.protobuf.RequestConverter;
48 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
49 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
50 import org.apache.hadoop.hbase.util.Bytes;
51 import org.apache.hadoop.hbase.util.Threads;
52 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
53 import org.apache.hadoop.hdfs.DFSConfigKeys;
54 import org.apache.hadoop.util.StringUtils;
55 import org.junit.After;
56 import org.junit.AfterClass;
57 import org.junit.Assert;
58 import org.junit.BeforeClass;
59 import org.junit.Test;
60 import org.junit.experimental.categories.Category;
61
62 import com.google.protobuf.ServiceException;
63
64
65
66
67
68 @Category(MediumTests.class)
69 public class TestRegionReplicas {
70 private static final Log LOG = LogFactory.getLog(TestRegionReplicas.class);
71
72 private static final int NB_SERVERS = 1;
73 private static HTable table;
74 private static final byte[] row = "TestRegionReplicas".getBytes();
75
76 private static HRegionInfo hriPrimary;
77 private static HRegionInfo hriSecondary;
78
79 private static final HBaseTestingUtility HTU = new HBaseTestingUtility();
80 private static final byte[] f = HConstants.CATALOG_FAMILY;
81
82 @BeforeClass
83 public static void before() throws Exception {
84
85
86 HTU.getConfiguration().setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 8192);
87 HTU.getConfiguration().setInt(DFSConfigKeys.DFS_CLIENT_READ_PREFETCH_SIZE_KEY, 1);
88 HTU.getConfiguration().setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 128 * 1024 * 1024);
89
90 HTU.startMiniCluster(NB_SERVERS);
91 final TableName tableName = TableName.valueOf(TestRegionReplicas.class.getSimpleName());
92
93
94 table = HTU.createTable(tableName, f);
95
96 hriPrimary = table.getRegionLocation(row, false).getRegionInfo();
97
98
99 hriSecondary = new HRegionInfo(hriPrimary.getTable(), hriPrimary.getStartKey(),
100 hriPrimary.getEndKey(), hriPrimary.isSplit(), hriPrimary.getRegionId(), 1);
101
102
103 TestRegionServerNoMaster.stopMasterAndAssignMeta(HTU);
104 }
105
106 @AfterClass
107 public static void afterClass() throws Exception {
108 table.close();
109 HTU.shutdownMiniCluster();
110 }
111
112 @After
113 public void after() throws Exception {
114
115
116
117 ZKAssign.deleteNodeFailSilent(HTU.getZooKeeperWatcher(), hriPrimary);
118 }
119
120 private HRegionServer getRS() {
121 return HTU.getMiniHBaseCluster().getRegionServer(0);
122 }
123
124 private void openRegion(HRegionInfo hri) throws Exception {
125 ZKAssign.createNodeOffline(HTU.getZooKeeperWatcher(), hri, getRS().getServerName());
126
127 AdminProtos.OpenRegionRequest orr = RequestConverter.buildOpenRegionRequest(getRS().getServerName(), hri, 0, null, null);
128 AdminProtos.OpenRegionResponse responseOpen = getRS().getRSRpcServices().openRegion(null, orr);
129 Assert.assertTrue(responseOpen.getOpeningStateCount() == 1);
130 Assert.assertTrue(responseOpen.getOpeningState(0).
131 equals(AdminProtos.OpenRegionResponse.RegionOpeningState.OPENED));
132 checkRegionIsOpened(hri.getEncodedName());
133 }
134
135 private void closeRegion(HRegionInfo hri) throws Exception {
136 ZKAssign.createNodeClosing(HTU.getZooKeeperWatcher(), hri, getRS().getServerName());
137
138 AdminProtos.CloseRegionRequest crr = RequestConverter.buildCloseRegionRequest(getRS().getServerName(),
139 hri.getEncodedName(), true);
140 AdminProtos.CloseRegionResponse responseClose = getRS().getRSRpcServices().closeRegion(null, crr);
141 Assert.assertTrue(responseClose.getClosed());
142
143 checkRegionIsClosed(hri.getEncodedName());
144
145 ZKAssign.deleteClosedNode(HTU.getZooKeeperWatcher(), hri.getEncodedName(), getRS().getServerName());
146 }
147
148 private void checkRegionIsOpened(String encodedRegionName) throws Exception {
149
150 while (!getRS().getRegionsInTransitionInRS().isEmpty()) {
151 Thread.sleep(1);
152 }
153
154 Assert.assertTrue(getRS().getRegionByEncodedName(encodedRegionName).isAvailable());
155
156 Assert.assertTrue(
157 ZKAssign.deleteOpenedNode(HTU.getZooKeeperWatcher(), encodedRegionName, getRS().getServerName()));
158 }
159
160
161 private void checkRegionIsClosed(String encodedRegionName) throws Exception {
162
163 while (!getRS().getRegionsInTransitionInRS().isEmpty()) {
164 Thread.sleep(1);
165 }
166
167 try {
168 Assert.assertFalse(getRS().getRegionByEncodedName(encodedRegionName).isAvailable());
169 } catch (NotServingRegionException expected) {
170
171 }
172
173
174 }
175
176 @Test(timeout = 60000)
177 public void testOpenRegionReplica() throws Exception {
178 openRegion(hriSecondary);
179 try {
180
181 HTU.loadNumericRows(table, f, 0, 1000);
182
183
184 Assert.assertEquals(1000, HTU.countRows(table));
185 } finally {
186 HTU.deleteNumericRows(table, f, 0, 1000);
187 closeRegion(hriSecondary);
188 }
189 }
190
191
192 @Test(timeout = 60000)
193 public void testRegionReplicaUpdatesMetaLocation() throws Exception {
194 openRegion(hriSecondary);
195 Table meta = null;
196 try {
197 meta = new HTable(HTU.getConfiguration(), TableName.META_TABLE_NAME);
198 TestMetaTableAccessor.assertMetaLocation(meta, hriPrimary.getRegionName()
199 , getRS().getServerName(), -1, 1, false);
200 } finally {
201 if (meta != null ) meta.close();
202 closeRegion(hriSecondary);
203 }
204 }
205
206 @Test(timeout = 60000)
207 public void testRegionReplicaGets() throws Exception {
208 try {
209
210 HTU.loadNumericRows(table, f, 0, 1000);
211
212 Assert.assertEquals(1000, HTU.countRows(table));
213
214 getRS().getRegionByEncodedName(hriPrimary.getEncodedName()).flushcache();
215
216 openRegion(hriSecondary);
217
218
219 HRegion region = getRS().getFromOnlineRegions(hriSecondary.getEncodedName());
220 assertGet(region, 42, true);
221
222 assertGetRpc(hriSecondary, 42, true);
223 } finally {
224 HTU.deleteNumericRows(table, HConstants.CATALOG_FAMILY, 0, 1000);
225 closeRegion(hriSecondary);
226 }
227 }
228
229 @Test(timeout = 60000)
230 public void testGetOnTargetRegionReplica() throws Exception {
231 try {
232
233 HTU.loadNumericRows(table, f, 0, 1000);
234
235 Assert.assertEquals(1000, HTU.countRows(table));
236
237 getRS().getRegionByEncodedName(hriPrimary.getEncodedName()).flushcache();
238
239 openRegion(hriSecondary);
240
241
242 byte[] row = Bytes.toBytes(String.valueOf(42));
243 Get get = new Get(row);
244 get.setConsistency(Consistency.TIMELINE);
245 get.setReplicaId(1);
246 Result result = table.get(get);
247 Assert.assertArrayEquals(row, result.getValue(f, null));
248 } finally {
249 HTU.deleteNumericRows(table, HConstants.CATALOG_FAMILY, 0, 1000);
250 closeRegion(hriSecondary);
251 }
252 }
253
254 private void assertGet(HRegion region, int value, boolean expect) throws IOException {
255 byte[] row = Bytes.toBytes(String.valueOf(value));
256 Get get = new Get(row);
257 Result result = region.get(get);
258 if (expect) {
259 Assert.assertArrayEquals(row, result.getValue(f, null));
260 } else {
261 result.isEmpty();
262 }
263 }
264
265
266 private void assertGetRpc(HRegionInfo info, int value, boolean expect) throws IOException, ServiceException {
267 byte[] row = Bytes.toBytes(String.valueOf(value));
268 Get get = new Get(row);
269 ClientProtos.GetRequest getReq = RequestConverter.buildGetRequest(info.getRegionName(), get);
270 ClientProtos.GetResponse getResp = getRS().getRSRpcServices().get(null, getReq);
271 Result result = ProtobufUtil.toResult(getResp.getResult());
272 if (expect) {
273 Assert.assertArrayEquals(row, result.getValue(f, null));
274 } else {
275 result.isEmpty();
276 }
277 }
278
279 private void restartRegionServer() throws Exception {
280 afterClass();
281 before();
282 }
283
284 @Test(timeout = 300000)
285 public void testRefreshStoreFiles() throws Exception {
286
287 final int refreshPeriod = 2000;
288 HTU.getConfiguration().setInt("hbase.hstore.compactionThreshold", 100);
289 HTU.getConfiguration().setInt(StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, refreshPeriod);
290
291 restartRegionServer();
292
293 try {
294 LOG.info("Opening the secondary region " + hriSecondary.getEncodedName());
295 openRegion(hriSecondary);
296
297
298 LOG.info("Loading data to primary region");
299 HTU.loadNumericRows(table, f, 0, 1000);
300
301 Assert.assertEquals(1000, HTU.countRows(table));
302
303 LOG.info("Flushing primary region");
304 getRS().getRegionByEncodedName(hriPrimary.getEncodedName()).flushcache();
305
306
307 LOG.info("Sleeping for " + (4 * refreshPeriod));
308 Threads.sleep(4 * refreshPeriod);
309
310 LOG.info("Checking results from secondary region replica");
311 HRegion secondaryRegion = getRS().getFromOnlineRegions(hriSecondary.getEncodedName());
312 Assert.assertEquals(1, secondaryRegion.getStore(f).getStorefilesCount());
313
314 assertGet(secondaryRegion, 42, true);
315 assertGetRpc(hriSecondary, 42, true);
316 assertGetRpc(hriSecondary, 1042, false);
317
318
319 HTU.loadNumericRows(table, f, 1000, 1100);
320 getRS().getRegionByEncodedName(hriPrimary.getEncodedName()).flushcache();
321
322 HTU.loadNumericRows(table, f, 2000, 2100);
323 getRS().getRegionByEncodedName(hriPrimary.getEncodedName()).flushcache();
324
325
326 Threads.sleep(4 * refreshPeriod);
327
328 assertGetRpc(hriSecondary, 42, true);
329 assertGetRpc(hriSecondary, 1042, true);
330 assertGetRpc(hriSecondary, 2042, true);
331
332
333 Assert.assertEquals(3, secondaryRegion.getStore(f).getStorefilesCount());
334
335
336 HTU.compact(table.getName(), true);
337
338 long wakeUpTime = System.currentTimeMillis() + 4 * refreshPeriod;
339 while (System.currentTimeMillis() < wakeUpTime) {
340 assertGetRpc(hriSecondary, 42, true);
341 assertGetRpc(hriSecondary, 1042, true);
342 assertGetRpc(hriSecondary, 2042, true);
343 Threads.sleep(10);
344 }
345
346
347 Assert.assertEquals(1, secondaryRegion.getStore(f).getStorefilesCount());
348
349 } finally {
350 HTU.deleteNumericRows(table, HConstants.CATALOG_FAMILY, 0, 1000);
351 closeRegion(hriSecondary);
352 }
353 }
354
355 @Test(timeout = 300000)
356 public void testFlushAndCompactionsInPrimary() throws Exception {
357
358 long runtime = 30 * 1000;
359
360 final int refreshPeriod = 100;
361 HTU.getConfiguration().setInt("hbase.hstore.compactionThreshold", 3);
362 HTU.getConfiguration().setInt(StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, refreshPeriod);
363
364 restartRegionServer();
365 final int startKey = 0, endKey = 1000;
366
367 try {
368 openRegion(hriSecondary);
369
370
371 HTU.loadNumericRows(table, f, startKey, endKey);
372 TestRegionServerNoMaster.flushRegion(HTU, hriPrimary);
373
374 Threads.sleep(2 * refreshPeriod);
375
376 final AtomicBoolean running = new AtomicBoolean(true);
377 @SuppressWarnings("unchecked")
378 final AtomicReference<Exception>[] exceptions = new AtomicReference[3];
379 for (int i=0; i < exceptions.length; i++) {
380 exceptions[i] = new AtomicReference<Exception>();
381 }
382
383 Runnable writer = new Runnable() {
384 int key = startKey;
385 @Override
386 public void run() {
387 try {
388 while (running.get()) {
389 byte[] data = Bytes.toBytes(String.valueOf(key));
390 Put put = new Put(data);
391 put.add(f, null, data);
392 table.put(put);
393 key++;
394 if (key == endKey) key = startKey;
395 }
396 } catch (Exception ex) {
397 LOG.warn(ex);
398 exceptions[0].compareAndSet(null, ex);
399 }
400 }
401 };
402
403 Runnable flusherCompactor = new Runnable() {
404 Random random = new Random();
405 @Override
406 public void run() {
407 try {
408 while (running.get()) {
409
410 if (random.nextBoolean()) {
411 TestRegionServerNoMaster.flushRegion(HTU, hriPrimary);
412 } else {
413 HTU.compact(table.getName(), random.nextBoolean());
414 }
415 }
416 } catch (Exception ex) {
417 LOG.warn(ex);
418 exceptions[1].compareAndSet(null, ex);
419 }
420 }
421 };
422
423 Runnable reader = new Runnable() {
424 Random random = new Random();
425 @Override
426 public void run() {
427 try {
428 while (running.get()) {
429
430 if (random.nextInt(10) == 0) {
431 try {
432 closeRegion(hriSecondary);
433 } catch (Exception ex) {
434 LOG.warn("Failed closing the region " + hriSecondary + " " + StringUtils.stringifyException(ex));
435 exceptions[2].compareAndSet(null, ex);
436 }
437 try {
438 openRegion(hriSecondary);
439 } catch (Exception ex) {
440 LOG.warn("Failed opening the region " + hriSecondary + " " + StringUtils.stringifyException(ex));
441 exceptions[2].compareAndSet(null, ex);
442 }
443 }
444
445 int key = random.nextInt(endKey - startKey) + startKey;
446 assertGetRpc(hriSecondary, key, true);
447 }
448 } catch (Exception ex) {
449 LOG.warn("Failed getting the value in the region " + hriSecondary + " " + StringUtils.stringifyException(ex));
450 exceptions[2].compareAndSet(null, ex);
451 }
452 }
453 };
454
455 LOG.info("Starting writer and reader");
456 ExecutorService executor = Executors.newFixedThreadPool(3);
457 executor.submit(writer);
458 executor.submit(flusherCompactor);
459 executor.submit(reader);
460
461
462 Threads.sleep(runtime);
463 running.set(false);
464 executor.shutdown();
465 executor.awaitTermination(30, TimeUnit.SECONDS);
466
467 for (AtomicReference<Exception> exRef : exceptions) {
468 Assert.assertNull(exRef.get());
469 }
470 } finally {
471 HTU.deleteNumericRows(table, HConstants.CATALOG_FAMILY, startKey, endKey);
472 closeRegion(hriSecondary);
473 }
474 }
475
476 @Test(timeout = 300000)
477 public void testVerifySecondaryAbilityToReadWithOnFiles() throws Exception {
478
479 HTU.getConfiguration().setInt(StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, 0);
480 restartRegionServer();
481
482 try {
483 LOG.info("Opening the secondary region " + hriSecondary.getEncodedName());
484 openRegion(hriSecondary);
485
486
487 LOG.info("Loading data to primary region");
488 for (int i = 0; i < 3; ++i) {
489 HTU.loadNumericRows(table, f, i * 1000, (i + 1) * 1000);
490 getRS().getRegionByEncodedName(hriPrimary.getEncodedName()).flushcache();
491 }
492
493 HRegion primaryRegion = getRS().getFromOnlineRegions(hriPrimary.getEncodedName());
494 Assert.assertEquals(3, primaryRegion.getStore(f).getStorefilesCount());
495
496
497 HRegion secondaryRegion = getRS().getFromOnlineRegions(hriSecondary.getEncodedName());
498 secondaryRegion.getStore(f).refreshStoreFiles();
499 Assert.assertEquals(3, secondaryRegion.getStore(f).getStorefilesCount());
500
501
502 LOG.info("Force Major compaction on primary region " + hriPrimary);
503 primaryRegion.compactStores(true);
504 Assert.assertEquals(1, primaryRegion.getStore(f).getStorefilesCount());
505
506
507
508
509
510 int keys = 0;
511 int sum = 0;
512 for (StoreFile sf: secondaryRegion.getStore(f).getStorefiles()) {
513
514 LOG.debug(getRS().getFileSystem().exists(sf.getPath()));
515 Assert.assertFalse(getRS().getFileSystem().exists(sf.getPath()));
516
517 HFileScanner scanner = sf.getReader().getScanner(false, false);
518 scanner.seekTo();
519 do {
520 keys++;
521
522 Cell cell = scanner.getKeyValue();
523 sum += Integer.parseInt(Bytes.toString(cell.getRowArray(),
524 cell.getRowOffset(), cell.getRowLength()));
525 } while (scanner.next());
526 }
527 Assert.assertEquals(3000, keys);
528 Assert.assertEquals(4498500, sum);
529 } finally {
530 HTU.deleteNumericRows(table, HConstants.CATALOG_FAMILY, 0, 1000);
531 closeRegion(hriSecondary);
532 }
533 }
534 }