View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.client;
20  
21  import static org.mockito.Matchers.anyObject;
22  import static org.mockito.Mockito.doReturn;
23  import static org.mockito.Mockito.mock;
24  import static org.mockito.Mockito.reset;
25  import static org.mockito.Mockito.times;
26  import static org.mockito.Mockito.verify;
27  
28  import java.math.BigDecimal;
29  import java.util.List;
30  import java.util.NavigableMap;
31  import java.util.Random;
32  
33  import org.apache.commons.logging.Log;
34  import org.apache.commons.logging.LogFactory;
35  import org.apache.hadoop.conf.Configuration;
36  import org.apache.hadoop.hbase.MetaTableAccessor;
37  import org.apache.hadoop.hbase.TableName;
38  import org.apache.hadoop.hbase.HBaseTestingUtility;
39  import org.apache.hadoop.hbase.HConstants;
40  import org.apache.hadoop.hbase.HRegionInfo;
41  import org.apache.hadoop.hbase.testclassification.MediumTests;
42  import org.apache.hadoop.hbase.ServerName;
43  import org.apache.hadoop.hbase.util.Bytes;
44  import org.apache.hadoop.hbase.util.StoppableImplementation;
45  import org.apache.hadoop.hbase.util.Threads;
46  import org.apache.hadoop.util.StringUtils;
47  import org.junit.After;
48  import org.junit.Assert;
49  import org.junit.Test;
50  import org.junit.experimental.categories.Category;
51  
52  @Category(MediumTests.class)
53  public class TestMetaScanner {
54    final Log LOG = LogFactory.getLog(getClass());
55    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
56    private Connection connection;
57  
58    public void setUp() throws Exception {
59      TEST_UTIL.startMiniCluster(1);
60      this.connection = TEST_UTIL.getConnection();
61    }
62  
63    @After
64    public void tearDown() throws Exception {
65      TEST_UTIL.shutdownMiniCluster();
66    }
67  
68    @Test
69    public void testMetaScanner() throws Exception {
70      LOG.info("Starting testMetaScanner");
71  
72      setUp();
73      final TableName TABLENAME = TableName.valueOf("testMetaScanner");
74      final byte[] FAMILY = Bytes.toBytes("family");
75      TEST_UTIL.createTable(TABLENAME, FAMILY);
76      Configuration conf = TEST_UTIL.getConfiguration();
77      HTable table = (HTable) connection.getTable(TABLENAME);
78      TEST_UTIL.createMultiRegions(conf, table, FAMILY,
79          new byte[][]{
80            HConstants.EMPTY_START_ROW,
81            Bytes.toBytes("region_a"),
82            Bytes.toBytes("region_b")});
83      // Make sure all the regions are deployed
84      TEST_UTIL.countRows(table);
85  
86      MetaScanner.MetaScannerVisitor visitor =
87        mock(MetaScanner.MetaScannerVisitor.class);
88      doReturn(true).when(visitor).processRow((Result)anyObject());
89  
90      // Scanning the entire table should give us three rows
91      MetaScanner.metaScan(connection, visitor, TABLENAME);
92      verify(visitor, times(3)).processRow((Result)anyObject());
93  
94      // Scanning the table with a specified empty start row should also
95      // give us three hbase:meta rows
96      reset(visitor);
97      doReturn(true).when(visitor).processRow((Result)anyObject());
98      MetaScanner.metaScan(connection, visitor, TABLENAME, HConstants.EMPTY_BYTE_ARRAY, 1000);
99      verify(visitor, times(3)).processRow((Result)anyObject());
100 
101     // Scanning the table starting in the middle should give us two rows:
102     // region_a and region_b
103     reset(visitor);
104     doReturn(true).when(visitor).processRow((Result)anyObject());
105     MetaScanner.metaScan(connection, visitor, TABLENAME, Bytes.toBytes("region_ac"), 1000);
106     verify(visitor, times(2)).processRow((Result)anyObject());
107 
108     // Scanning with a limit of 1 should only give us one row
109     reset(visitor);
110     doReturn(true).when(visitor).processRow((Result) anyObject());
111     MetaScanner.metaScan(connection, visitor, TABLENAME, Bytes.toBytes("region_ac"), 1);
112     verify(visitor, times(1)).processRow((Result) anyObject());
113     table.close();
114   }
115 
116   @Test
117   public void testConcurrentMetaScannerAndCatalogJanitor() throws Throwable {
118     /* TEST PLAN: start with only one region in a table. Have a splitter
119      * thread  and metascanner threads that continously scan the meta table for regions.
120      * CatalogJanitor from master will run frequently to clean things up
121      */
122     TEST_UTIL.getConfiguration().setLong("hbase.catalogjanitor.interval", 500);
123     setUp();
124 
125     final long runtime = 30 * 1000; //30 sec
126     LOG.info("Starting testConcurrentMetaScannerAndCatalogJanitor");
127     final TableName TABLENAME =
128         TableName.valueOf("testConcurrentMetaScannerAndCatalogJanitor");
129     final byte[] FAMILY = Bytes.toBytes("family");
130     TEST_UTIL.createTable(TABLENAME, FAMILY);
131 
132     class RegionMetaSplitter extends StoppableImplementation implements Runnable {
133       Random random = new Random();
134       Throwable ex = null;
135       @Override
136       public void run() {
137         while (!isStopped()) {
138           try {
139             List<HRegionInfo> regions = MetaScanner.listAllRegions(TEST_UTIL.getConfiguration(),
140                 connection, false);
141 
142             //select a random region
143             HRegionInfo parent = regions.get(random.nextInt(regions.size()));
144             if (parent == null || !TABLENAME.equals(parent.getTable())) {
145               continue;
146             }
147 
148             long startKey = 0, endKey = Long.MAX_VALUE;
149             byte[] start = parent.getStartKey();
150             byte[] end = parent.getEndKey();
151             if (!Bytes.equals(HConstants.EMPTY_START_ROW, parent.getStartKey())) {
152               startKey = Bytes.toLong(parent.getStartKey());
153             }
154             if (!Bytes.equals(HConstants.EMPTY_END_ROW, parent.getEndKey())) {
155               endKey = Bytes.toLong(parent.getEndKey());
156             }
157             if (startKey == endKey) {
158               continue;
159             }
160 
161             long midKey = BigDecimal.valueOf(startKey).add(BigDecimal.valueOf(endKey))
162                 .divideToIntegralValue(BigDecimal.valueOf(2)).longValue();
163 
164             HRegionInfo splita = new HRegionInfo(TABLENAME,
165               start,
166               Bytes.toBytes(midKey));
167             HRegionInfo splitb = new HRegionInfo(TABLENAME,
168               Bytes.toBytes(midKey),
169               end);
170 
171             MetaTableAccessor.splitRegion(connection,
172               parent, splita, splitb, ServerName.valueOf("fooserver", 1, 0));
173 
174             Threads.sleep(random.nextInt(200));
175           } catch (Throwable e) {
176             ex = e;
177             Assert.fail(StringUtils.stringifyException(e));
178           }
179         }
180       }
181       void rethrowExceptionIfAny() throws Throwable {
182         if (ex != null) { throw ex; }
183       }
184     }
185 
186     class MetaScannerVerifier extends StoppableImplementation implements Runnable {
187       Random random = new Random();
188       Throwable ex = null;
189       @Override
190       public void run() {
191          while(!isStopped()) {
192            try {
193             NavigableMap<HRegionInfo, ServerName> regions =
194                 MetaScanner.allTableRegions(connection, TABLENAME);
195 
196             LOG.info("-------");
197             byte[] lastEndKey = HConstants.EMPTY_START_ROW;
198             for (HRegionInfo hri: regions.navigableKeySet()) {
199               long startKey = 0, endKey = Long.MAX_VALUE;
200               if (!Bytes.equals(HConstants.EMPTY_START_ROW, hri.getStartKey())) {
201                 startKey = Bytes.toLong(hri.getStartKey());
202               }
203               if (!Bytes.equals(HConstants.EMPTY_END_ROW, hri.getEndKey())) {
204                 endKey = Bytes.toLong(hri.getEndKey());
205               }
206               LOG.info("start:" + startKey + " end:" + endKey + " hri:" + hri);
207               Assert.assertTrue("lastEndKey=" + Bytes.toString(lastEndKey) + ", startKey=" +
208                 Bytes.toString(hri.getStartKey()), Bytes.equals(lastEndKey, hri.getStartKey()));
209               lastEndKey = hri.getEndKey();
210             }
211             Assert.assertTrue(Bytes.equals(lastEndKey, HConstants.EMPTY_END_ROW));
212             LOG.info("-------");
213             Threads.sleep(10 + random.nextInt(50));
214           } catch (Throwable e) {
215             ex = e;
216             Assert.fail(StringUtils.stringifyException(e));
217           }
218          }
219       }
220       void rethrowExceptionIfAny() throws Throwable {
221         if (ex != null) { throw ex; }
222       }
223     }
224 
225     RegionMetaSplitter regionMetaSplitter = new RegionMetaSplitter();
226     MetaScannerVerifier metaScannerVerifier = new MetaScannerVerifier();
227 
228     Thread regionMetaSplitterThread = new Thread(regionMetaSplitter);
229     Thread metaScannerVerifierThread = new Thread(metaScannerVerifier);
230 
231     regionMetaSplitterThread.start();
232     metaScannerVerifierThread.start();
233 
234     Threads.sleep(runtime);
235 
236     regionMetaSplitter.stop("test finished");
237     metaScannerVerifier.stop("test finished");
238 
239     regionMetaSplitterThread.join();
240     metaScannerVerifierThread.join();
241 
242     regionMetaSplitter.rethrowExceptionIfAny();
243     metaScannerVerifier.rethrowExceptionIfAny();
244   }
245 
246 }