View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import java.io.IOException;
22  import java.util.Random;
23  import java.util.concurrent.ExecutorService;
24  import java.util.concurrent.Executors;
25  import java.util.concurrent.TimeUnit;
26  import java.util.concurrent.atomic.AtomicBoolean;
27  import java.util.concurrent.atomic.AtomicReference;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.hbase.Cell;
32  import org.apache.hadoop.hbase.HBaseTestingUtility;
33  import org.apache.hadoop.hbase.HConstants;
34  import org.apache.hadoop.hbase.HRegionInfo;
35  import org.apache.hadoop.hbase.testclassification.MediumTests;
36  import org.apache.hadoop.hbase.NotServingRegionException;
37  import org.apache.hadoop.hbase.TableName;
38  import org.apache.hadoop.hbase.TestMetaTableAccessor;
39  import org.apache.hadoop.hbase.client.Consistency;
40  import org.apache.hadoop.hbase.client.Get;
41  import org.apache.hadoop.hbase.client.HTable;
42  import org.apache.hadoop.hbase.client.Put;
43  import org.apache.hadoop.hbase.client.Result;
44  import org.apache.hadoop.hbase.client.Table;
45  import org.apache.hadoop.hbase.io.hfile.HFileScanner;
46  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
47  import org.apache.hadoop.hbase.protobuf.RequestConverter;
48  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
49  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
50  import org.apache.hadoop.hbase.util.Bytes;
51  import org.apache.hadoop.hbase.util.Threads;
52  import org.apache.hadoop.hbase.zookeeper.ZKAssign;
53  import org.apache.hadoop.hdfs.DFSConfigKeys;
54  import org.apache.hadoop.util.StringUtils;
55  import org.junit.After;
56  import org.junit.AfterClass;
57  import org.junit.Assert;
58  import org.junit.BeforeClass;
59  import org.junit.Test;
60  import org.junit.experimental.categories.Category;
61  
62  import com.google.protobuf.ServiceException;
63  
64  /**
65   * Tests for region replicas. Sad that we cannot isolate these without bringing up a whole
66   * cluster. See {@link TestRegionServerNoMaster}.
67   */
68  @Category(MediumTests.class)
69  public class TestRegionReplicas {
70    private static final Log LOG = LogFactory.getLog(TestRegionReplicas.class);
71  
72    private static final int NB_SERVERS = 1;
73    private static HTable table;
74    private static final byte[] row = "TestRegionReplicas".getBytes();
75  
76    private static HRegionInfo hriPrimary;
77    private static HRegionInfo hriSecondary;
78  
79    private static final HBaseTestingUtility HTU = new HBaseTestingUtility();
80    private static final byte[] f = HConstants.CATALOG_FAMILY;
81  
82    @BeforeClass
83    public static void before() throws Exception {
84      // Reduce the hdfs block size and prefetch to trigger the file-link reopen
85      // when the file is moved to archive (e.g. compaction)
86      HTU.getConfiguration().setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 8192);
87      HTU.getConfiguration().setInt(DFSConfigKeys.DFS_CLIENT_READ_PREFETCH_SIZE_KEY, 1);
88      HTU.getConfiguration().setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 128 * 1024 * 1024);
89  
90      HTU.startMiniCluster(NB_SERVERS);
91      final TableName tableName = TableName.valueOf(TestRegionReplicas.class.getSimpleName());
92  
93      // Create table then get the single region for our new table.
94      table = HTU.createTable(tableName, f);
95  
96      hriPrimary = table.getRegionLocation(row, false).getRegionInfo();
97  
98      // mock a secondary region info to open
99      hriSecondary = new HRegionInfo(hriPrimary.getTable(), hriPrimary.getStartKey(),
100         hriPrimary.getEndKey(), hriPrimary.isSplit(), hriPrimary.getRegionId(), 1);
101 
102     // No master
103     TestRegionServerNoMaster.stopMasterAndAssignMeta(HTU);
104   }
105 
106   @AfterClass
107   public static void afterClass() throws Exception {
108     table.close();
109     HTU.shutdownMiniCluster();
110   }
111 
112   @After
113   public void after() throws Exception {
114     // Clean the state if the test failed before cleaning the znode
115     // It does not manage all bad failures, so if there are multiple failures, only
116     //  the first one should be looked at.
117     ZKAssign.deleteNodeFailSilent(HTU.getZooKeeperWatcher(), hriPrimary);
118   }
119 
120   private HRegionServer getRS() {
121     return HTU.getMiniHBaseCluster().getRegionServer(0);
122   }
123 
124   private void openRegion(HRegionInfo hri) throws Exception {
125     ZKAssign.createNodeOffline(HTU.getZooKeeperWatcher(), hri, getRS().getServerName());
126     // first version is '0'
127     AdminProtos.OpenRegionRequest orr = RequestConverter.buildOpenRegionRequest(getRS().getServerName(), hri, 0, null, null);
128     AdminProtos.OpenRegionResponse responseOpen = getRS().getRSRpcServices().openRegion(null, orr);
129     Assert.assertTrue(responseOpen.getOpeningStateCount() == 1);
130     Assert.assertTrue(responseOpen.getOpeningState(0).
131         equals(AdminProtos.OpenRegionResponse.RegionOpeningState.OPENED));
132     checkRegionIsOpened(hri.getEncodedName());
133   }
134 
135   private void closeRegion(HRegionInfo hri) throws Exception {
136     ZKAssign.createNodeClosing(HTU.getZooKeeperWatcher(), hri, getRS().getServerName());
137 
138     AdminProtos.CloseRegionRequest crr = RequestConverter.buildCloseRegionRequest(getRS().getServerName(),
139         hri.getEncodedName(), true);
140     AdminProtos.CloseRegionResponse responseClose = getRS().getRSRpcServices().closeRegion(null, crr);
141     Assert.assertTrue(responseClose.getClosed());
142 
143     checkRegionIsClosed(hri.getEncodedName());
144 
145     ZKAssign.deleteClosedNode(HTU.getZooKeeperWatcher(), hri.getEncodedName(), getRS().getServerName());
146   }
147 
148   private void checkRegionIsOpened(String encodedRegionName) throws Exception {
149 
150     while (!getRS().getRegionsInTransitionInRS().isEmpty()) {
151       Thread.sleep(1);
152     }
153 
154     Assert.assertTrue(getRS().getRegionByEncodedName(encodedRegionName).isAvailable());
155 
156     Assert.assertTrue(
157         ZKAssign.deleteOpenedNode(HTU.getZooKeeperWatcher(), encodedRegionName, getRS().getServerName()));
158   }
159 
160 
161   private void checkRegionIsClosed(String encodedRegionName) throws Exception {
162 
163     while (!getRS().getRegionsInTransitionInRS().isEmpty()) {
164       Thread.sleep(1);
165     }
166 
167     try {
168       Assert.assertFalse(getRS().getRegionByEncodedName(encodedRegionName).isAvailable());
169     } catch (NotServingRegionException expected) {
170       // That's how it work: if the region is closed we have an exception.
171     }
172 
173     // We don't delete the znode here, because there is not always a znode.
174   }
175 
176   @Test(timeout = 60000)
177   public void testOpenRegionReplica() throws Exception {
178     openRegion(hriSecondary);
179     try {
180       //load some data to primary
181       HTU.loadNumericRows(table, f, 0, 1000);
182 
183       // assert that we can read back from primary
184       Assert.assertEquals(1000, HTU.countRows(table));
185     } finally {
186       HTU.deleteNumericRows(table, f, 0, 1000);
187       closeRegion(hriSecondary);
188     }
189   }
190 
191   /** Tests that the meta location is saved for secondary regions */
192   @Test(timeout = 60000)
193   public void testRegionReplicaUpdatesMetaLocation() throws Exception {
194     openRegion(hriSecondary);
195     Table meta = null;
196     try {
197       meta = new HTable(HTU.getConfiguration(), TableName.META_TABLE_NAME);
198       TestMetaTableAccessor.assertMetaLocation(meta, hriPrimary.getRegionName()
199         , getRS().getServerName(), -1, 1, false);
200     } finally {
201       if (meta != null ) meta.close();
202       closeRegion(hriSecondary);
203     }
204   }
205 
206   @Test(timeout = 60000)
207   public void testRegionReplicaGets() throws Exception {
208     try {
209       //load some data to primary
210       HTU.loadNumericRows(table, f, 0, 1000);
211       // assert that we can read back from primary
212       Assert.assertEquals(1000, HTU.countRows(table));
213       // flush so that region replica can read
214       getRS().getRegionByEncodedName(hriPrimary.getEncodedName()).flushcache();
215 
216       openRegion(hriSecondary);
217 
218       // first try directly against region
219       HRegion region = getRS().getFromOnlineRegions(hriSecondary.getEncodedName());
220       assertGet(region, 42, true);
221 
222       assertGetRpc(hriSecondary, 42, true);
223     } finally {
224       HTU.deleteNumericRows(table, HConstants.CATALOG_FAMILY, 0, 1000);
225       closeRegion(hriSecondary);
226     }
227   }
228 
229   @Test(timeout = 60000)
230   public void testGetOnTargetRegionReplica() throws Exception {
231     try {
232       //load some data to primary
233       HTU.loadNumericRows(table, f, 0, 1000);
234       // assert that we can read back from primary
235       Assert.assertEquals(1000, HTU.countRows(table));
236       // flush so that region replica can read
237       getRS().getRegionByEncodedName(hriPrimary.getEncodedName()).flushcache();
238 
239       openRegion(hriSecondary);
240 
241       // try directly Get against region replica
242       byte[] row = Bytes.toBytes(String.valueOf(42));
243       Get get = new Get(row);
244       get.setConsistency(Consistency.TIMELINE);
245       get.setReplicaId(1);
246       Result result = table.get(get);
247       Assert.assertArrayEquals(row, result.getValue(f, null));
248     } finally {
249       HTU.deleteNumericRows(table, HConstants.CATALOG_FAMILY, 0, 1000);
250       closeRegion(hriSecondary);
251     }
252   }
253 
254   private void assertGet(HRegion region, int value, boolean expect) throws IOException {
255     byte[] row = Bytes.toBytes(String.valueOf(value));
256     Get get = new Get(row);
257     Result result = region.get(get);
258     if (expect) {
259       Assert.assertArrayEquals(row, result.getValue(f, null));
260     } else {
261       result.isEmpty();
262     }
263   }
264 
265   // build a mock rpc
266   private void assertGetRpc(HRegionInfo info, int value, boolean expect) throws IOException, ServiceException {
267     byte[] row = Bytes.toBytes(String.valueOf(value));
268     Get get = new Get(row);
269     ClientProtos.GetRequest getReq = RequestConverter.buildGetRequest(info.getRegionName(), get);
270     ClientProtos.GetResponse getResp =  getRS().getRSRpcServices().get(null, getReq);
271     Result result = ProtobufUtil.toResult(getResp.getResult());
272     if (expect) {
273       Assert.assertArrayEquals(row, result.getValue(f, null));
274     } else {
275       result.isEmpty();
276     }
277   }
278 
279   private void restartRegionServer() throws Exception {
280     afterClass();
281     before();
282   }
283 
284   @Test(timeout = 300000)
285   public void testRefreshStoreFiles() throws Exception {
286     // enable store file refreshing
287     final int refreshPeriod = 2000; // 2 sec
288     HTU.getConfiguration().setInt("hbase.hstore.compactionThreshold", 100);
289     HTU.getConfiguration().setInt(StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, refreshPeriod);
290     // restart the region server so that it starts the refresher chore
291     restartRegionServer();
292 
293     try {
294       LOG.info("Opening the secondary region " + hriSecondary.getEncodedName());
295       openRegion(hriSecondary);
296 
297       //load some data to primary
298       LOG.info("Loading data to primary region");
299       HTU.loadNumericRows(table, f, 0, 1000);
300       // assert that we can read back from primary
301       Assert.assertEquals(1000, HTU.countRows(table));
302       // flush so that region replica can read
303       LOG.info("Flushing primary region");
304       getRS().getRegionByEncodedName(hriPrimary.getEncodedName()).flushcache();
305 
306       // ensure that chore is run
307       LOG.info("Sleeping for " + (4 * refreshPeriod));
308       Threads.sleep(4 * refreshPeriod);
309 
310       LOG.info("Checking results from secondary region replica");
311       HRegion secondaryRegion = getRS().getFromOnlineRegions(hriSecondary.getEncodedName());
312       Assert.assertEquals(1, secondaryRegion.getStore(f).getStorefilesCount());
313 
314       assertGet(secondaryRegion, 42, true);
315       assertGetRpc(hriSecondary, 42, true);
316       assertGetRpc(hriSecondary, 1042, false);
317 
318       //load some data to primary
319       HTU.loadNumericRows(table, f, 1000, 1100);
320       getRS().getRegionByEncodedName(hriPrimary.getEncodedName()).flushcache();
321 
322       HTU.loadNumericRows(table, f, 2000, 2100);
323       getRS().getRegionByEncodedName(hriPrimary.getEncodedName()).flushcache();
324 
325       // ensure that chore is run
326       Threads.sleep(4 * refreshPeriod);
327 
328       assertGetRpc(hriSecondary, 42, true);
329       assertGetRpc(hriSecondary, 1042, true);
330       assertGetRpc(hriSecondary, 2042, true);
331 
332       // ensure that we are see the 3 store files
333       Assert.assertEquals(3, secondaryRegion.getStore(f).getStorefilesCount());
334 
335       // force compaction
336       HTU.compact(table.getName(), true);
337 
338       long wakeUpTime = System.currentTimeMillis() + 4 * refreshPeriod;
339       while (System.currentTimeMillis() < wakeUpTime) {
340         assertGetRpc(hriSecondary, 42, true);
341         assertGetRpc(hriSecondary, 1042, true);
342         assertGetRpc(hriSecondary, 2042, true);
343         Threads.sleep(10);
344       }
345 
346       // ensure that we see the compacted file only
347       Assert.assertEquals(1, secondaryRegion.getStore(f).getStorefilesCount());
348 
349     } finally {
350       HTU.deleteNumericRows(table, HConstants.CATALOG_FAMILY, 0, 1000);
351       closeRegion(hriSecondary);
352     }
353   }
354 
355   @Test(timeout = 300000)
356   public void testFlushAndCompactionsInPrimary() throws Exception {
357 
358     long runtime = 30 * 1000;
359     // enable store file refreshing
360     final int refreshPeriod = 100; // 100ms refresh is a lot
361     HTU.getConfiguration().setInt("hbase.hstore.compactionThreshold", 3);
362     HTU.getConfiguration().setInt(StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, refreshPeriod);
363     // restart the region server so that it starts the refresher chore
364     restartRegionServer();
365     final int startKey = 0, endKey = 1000;
366 
367     try {
368       openRegion(hriSecondary);
369 
370       //load some data to primary so that reader won't fail
371       HTU.loadNumericRows(table, f, startKey, endKey);
372       TestRegionServerNoMaster.flushRegion(HTU, hriPrimary);
373       // ensure that chore is run
374       Threads.sleep(2 * refreshPeriod);
375 
376       final AtomicBoolean running = new AtomicBoolean(true);
377       @SuppressWarnings("unchecked")
378       final AtomicReference<Exception>[] exceptions = new AtomicReference[3];
379       for (int i=0; i < exceptions.length; i++) {
380         exceptions[i] = new AtomicReference<Exception>();
381       }
382 
383       Runnable writer = new Runnable() {
384         int key = startKey;
385         @Override
386         public void run() {
387           try {
388             while (running.get()) {
389               byte[] data = Bytes.toBytes(String.valueOf(key));
390               Put put = new Put(data);
391               put.add(f, null, data);
392               table.put(put);
393               key++;
394               if (key == endKey) key = startKey;
395             }
396           } catch (Exception ex) {
397             LOG.warn(ex);
398             exceptions[0].compareAndSet(null, ex);
399           }
400         }
401       };
402 
403       Runnable flusherCompactor = new Runnable() {
404         Random random = new Random();
405         @Override
406         public void run() {
407           try {
408             while (running.get()) {
409               // flush or compact
410               if (random.nextBoolean()) {
411                 TestRegionServerNoMaster.flushRegion(HTU, hriPrimary);
412               } else {
413                 HTU.compact(table.getName(), random.nextBoolean());
414               }
415             }
416           } catch (Exception ex) {
417             LOG.warn(ex);
418             exceptions[1].compareAndSet(null, ex);
419           }
420         }
421       };
422 
423       Runnable reader = new Runnable() {
424         Random random = new Random();
425         @Override
426         public void run() {
427           try {
428             while (running.get()) {
429               // whether to do a close and open
430               if (random.nextInt(10) == 0) {
431                 try {
432                   closeRegion(hriSecondary);
433                 } catch (Exception ex) {
434                   LOG.warn("Failed closing the region " + hriSecondary + " "  + StringUtils.stringifyException(ex));
435                   exceptions[2].compareAndSet(null, ex);
436                 }
437                 try {
438                   openRegion(hriSecondary);
439                 } catch (Exception ex) {
440                   LOG.warn("Failed opening the region " + hriSecondary + " "  + StringUtils.stringifyException(ex));
441                   exceptions[2].compareAndSet(null, ex);
442                 }
443               }
444 
445               int key = random.nextInt(endKey - startKey) + startKey;
446               assertGetRpc(hriSecondary, key, true);
447             }
448           } catch (Exception ex) {
449             LOG.warn("Failed getting the value in the region " + hriSecondary + " "  + StringUtils.stringifyException(ex));
450             exceptions[2].compareAndSet(null, ex);
451           }
452         }
453       };
454 
455       LOG.info("Starting writer and reader");
456       ExecutorService executor = Executors.newFixedThreadPool(3);
457       executor.submit(writer);
458       executor.submit(flusherCompactor);
459       executor.submit(reader);
460 
461       // wait for threads
462       Threads.sleep(runtime);
463       running.set(false);
464       executor.shutdown();
465       executor.awaitTermination(30, TimeUnit.SECONDS);
466 
467       for (AtomicReference<Exception> exRef : exceptions) {
468         Assert.assertNull(exRef.get());
469       }
470     } finally {
471       HTU.deleteNumericRows(table, HConstants.CATALOG_FAMILY, startKey, endKey);
472       closeRegion(hriSecondary);
473     }
474   }
475 
476   @Test(timeout = 300000)
477   public void testVerifySecondaryAbilityToReadWithOnFiles() throws Exception {
478     // disable the store file refresh chore (we do this by hand)
479     HTU.getConfiguration().setInt(StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, 0);
480     restartRegionServer();
481 
482     try {
483       LOG.info("Opening the secondary region " + hriSecondary.getEncodedName());
484       openRegion(hriSecondary);
485 
486       // load some data to primary
487       LOG.info("Loading data to primary region");
488       for (int i = 0; i < 3; ++i) {
489         HTU.loadNumericRows(table, f, i * 1000, (i + 1) * 1000);
490         getRS().getRegionByEncodedName(hriPrimary.getEncodedName()).flushcache();
491       }
492 
493       HRegion primaryRegion = getRS().getFromOnlineRegions(hriPrimary.getEncodedName());
494       Assert.assertEquals(3, primaryRegion.getStore(f).getStorefilesCount());
495 
496       // Refresh store files on the secondary
497       HRegion secondaryRegion = getRS().getFromOnlineRegions(hriSecondary.getEncodedName());
498       secondaryRegion.getStore(f).refreshStoreFiles();
499       Assert.assertEquals(3, secondaryRegion.getStore(f).getStorefilesCount());
500 
501       // force compaction
502       LOG.info("Force Major compaction on primary region " + hriPrimary);
503       primaryRegion.compactStores(true);
504       Assert.assertEquals(1, primaryRegion.getStore(f).getStorefilesCount());
505 
506       // scan all the hfiles on the secondary.
507       // since there are no read on the secondary when we ask locations to
508       // the NN a FileNotFound exception will be returned and the FileLink
509       // should be able to deal with it giving us all the result we expect.
510       int keys = 0;
511       int sum = 0;
512       for (StoreFile sf: secondaryRegion.getStore(f).getStorefiles()) {
513         // Our file does not exist anymore. was moved by the compaction above.
514         LOG.debug(getRS().getFileSystem().exists(sf.getPath()));
515         Assert.assertFalse(getRS().getFileSystem().exists(sf.getPath()));
516 
517         HFileScanner scanner = sf.getReader().getScanner(false, false);
518         scanner.seekTo();
519         do {
520           keys++;
521 
522           Cell cell = scanner.getKeyValue();
523           sum += Integer.parseInt(Bytes.toString(cell.getRowArray(),
524             cell.getRowOffset(), cell.getRowLength()));
525         } while (scanner.next());
526       }
527       Assert.assertEquals(3000, keys);
528       Assert.assertEquals(4498500, sum);
529     } finally {
530       HTU.deleteNumericRows(table, HConstants.CATALOG_FAMILY, 0, 1000);
531       closeRegion(hriSecondary);
532     }
533   }
534 }