1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.client;
21
22 import static org.mockito.Matchers.anyObject;
23 import static org.mockito.Mockito.doReturn;
24 import static org.mockito.Mockito.mock;
25 import static org.mockito.Mockito.reset;
26 import static org.mockito.Mockito.times;
27 import static org.mockito.Mockito.verify;
28 import static org.mockito.Mockito.when;
29
30 import java.math.BigDecimal;
31 import java.util.List;
32 import java.util.NavigableMap;
33 import java.util.Random;
34
35 import org.apache.commons.logging.Log;
36 import org.apache.commons.logging.LogFactory;
37 import org.apache.hadoop.conf.Configuration;
38 import org.apache.hadoop.hbase.HBaseTestingUtility;
39 import org.apache.hadoop.hbase.HConstants;
40 import org.apache.hadoop.hbase.HRegionInfo;
41 import org.apache.hadoop.hbase.MediumTests;
42 import org.apache.hadoop.hbase.ServerName;
43 import org.apache.hadoop.hbase.catalog.CatalogTracker;
44 import org.apache.hadoop.hbase.catalog.MetaEditor;
45 import org.apache.hadoop.hbase.util.Bytes;
46 import org.apache.hadoop.hbase.util.StoppableImplementation;
47 import org.apache.hadoop.hbase.util.Threads;
48 import org.apache.hadoop.util.StringUtils;
49 import org.junit.After;
50 import org.junit.Assert;
51 import org.junit.Test;
52 import org.junit.experimental.categories.Category;
53 @Category(MediumTests.class)
54 public class TestMetaScanner {
55 final Log LOG = LogFactory.getLog(getClass());
56 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
57
58 public void setUp() throws Exception {
59 TEST_UTIL.startMiniCluster(1);
60 }
61
62 @After
63 public void tearDown() throws Exception {
64 TEST_UTIL.shutdownMiniCluster();
65 }
66
67 @Test
68 public void testMetaScanner() throws Exception {
69 LOG.info("Starting testMetaScanner");
70 setUp();
71 final byte[] TABLENAME = Bytes.toBytes("testMetaScanner");
72 final byte[] FAMILY = Bytes.toBytes("family");
73 TEST_UTIL.createTable(TABLENAME, FAMILY);
74 Configuration conf = TEST_UTIL.getConfiguration();
75 HTable table = new HTable(conf, TABLENAME);
76 TEST_UTIL.createMultiRegions(conf, table, FAMILY,
77 new byte[][]{
78 HConstants.EMPTY_START_ROW,
79 Bytes.toBytes("region_a"),
80 Bytes.toBytes("region_b")});
81
82 TEST_UTIL.countRows(table);
83
84 MetaScanner.MetaScannerVisitor visitor =
85 mock(MetaScanner.MetaScannerVisitor.class);
86 doReturn(true).when(visitor).processRow((Result)anyObject());
87
88
89 MetaScanner.metaScan(conf, visitor, TABLENAME);
90 verify(visitor, times(3)).processRow((Result)anyObject());
91
92
93
94 reset(visitor);
95 doReturn(true).when(visitor).processRow((Result)anyObject());
96 MetaScanner.metaScan(conf, visitor, TABLENAME, HConstants.EMPTY_BYTE_ARRAY, 1000);
97 verify(visitor, times(3)).processRow((Result)anyObject());
98
99
100
101 reset(visitor);
102 doReturn(true).when(visitor).processRow((Result)anyObject());
103 MetaScanner.metaScan(conf, visitor, TABLENAME, Bytes.toBytes("region_ac"), 1000);
104 verify(visitor, times(2)).processRow((Result)anyObject());
105
106
107 reset(visitor);
108 doReturn(true).when(visitor).processRow((Result)anyObject());
109 MetaScanner.metaScan(conf, visitor, TABLENAME, Bytes.toBytes("region_ac"), 1);
110 verify(visitor, times(1)).processRow((Result)anyObject());
111 table.close();
112 }
113
114 @Test
115 public void testConcurrentMetaScannerAndCatalogJanitor() throws Throwable {
116
117
118
119
120 TEST_UTIL.getConfiguration().setLong("hbase.catalogjanitor.interval", 500);
121 setUp();
122
123 final long runtime = 30 * 1000;
124 LOG.info("Starting testConcurrentMetaScannerAndCatalogJanitor");
125 final byte[] TABLENAME = Bytes.toBytes("testConcurrentMetaScannerAndCatalogJanitor");
126 final byte[] FAMILY = Bytes.toBytes("family");
127 TEST_UTIL.createTable(TABLENAME, FAMILY);
128 final CatalogTracker catalogTracker = mock(CatalogTracker.class);
129 when(catalogTracker.getConnection()).thenReturn(TEST_UTIL.getHBaseAdmin().getConnection());
130
131 class RegionMetaSplitter extends StoppableImplementation implements Runnable {
132 Random random = new Random();
133 Throwable ex = null;
134 @Override
135 public void run() {
136 while (!isStopped()) {
137 try {
138 List<HRegionInfo> regions = MetaScanner.listAllRegions(
139 TEST_UTIL.getConfiguration(), false);
140
141
142 HRegionInfo parent = regions.get(random.nextInt(regions.size()));
143 if (parent == null || !Bytes.equals(TABLENAME, parent.getTableName())) {
144 continue;
145 }
146
147 long startKey = 0, endKey = Long.MAX_VALUE;
148 byte[] start = parent.getStartKey();
149 byte[] end = parent.getEndKey();
150 if (!Bytes.equals(HConstants.EMPTY_START_ROW, parent.getStartKey())) {
151 startKey = Bytes.toLong(parent.getStartKey());
152 }
153 if (!Bytes.equals(HConstants.EMPTY_END_ROW, parent.getEndKey())) {
154 endKey = Bytes.toLong(parent.getEndKey());
155 }
156 if (startKey == endKey) {
157 continue;
158 }
159
160 long midKey = BigDecimal.valueOf(startKey).add(BigDecimal.valueOf(endKey))
161 .divideToIntegralValue(BigDecimal.valueOf(2)).longValue();
162
163 HRegionInfo splita = new HRegionInfo(TABLENAME,
164 start,
165 Bytes.toBytes(midKey));
166 HRegionInfo splitb = new HRegionInfo(TABLENAME,
167 Bytes.toBytes(midKey),
168 end);
169
170 MetaEditor.offlineParentInMeta(catalogTracker, parent, splita, splitb);
171 Threads.sleep(100);
172 MetaEditor.addDaughter(catalogTracker, splitb, null);
173 MetaEditor.addDaughter(catalogTracker, splita, null);
174
175 Threads.sleep(random.nextInt(200));
176 } catch (Throwable e) {
177 ex = e;
178 Assert.fail(StringUtils.stringifyException(e));
179 }
180 }
181 }
182 void rethrowExceptionIfAny() throws Throwable {
183 if (ex != null) { throw ex; }
184 }
185 }
186
187 class MetaScannerVerifier extends StoppableImplementation implements Runnable {
188 Random random = new Random();
189 Throwable ex = null;
190 @Override
191 public void run() {
192 while(!isStopped()) {
193 try {
194 NavigableMap<HRegionInfo, ServerName> regions =
195 MetaScanner.allTableRegions(TEST_UTIL.getConfiguration(), TABLENAME, false);
196
197 LOG.info("-------");
198 byte[] lastEndKey = HConstants.EMPTY_START_ROW;
199 for (HRegionInfo hri: regions.navigableKeySet()) {
200 long startKey = 0, endKey = Long.MAX_VALUE;
201 if (!Bytes.equals(HConstants.EMPTY_START_ROW, hri.getStartKey())) {
202 startKey = Bytes.toLong(hri.getStartKey());
203 }
204 if (!Bytes.equals(HConstants.EMPTY_END_ROW, hri.getEndKey())) {
205 endKey = Bytes.toLong(hri.getEndKey());
206 }
207 LOG.info("start:" + startKey + " end:" + endKey + " hri:" + hri);
208 Assert.assertTrue(Bytes.equals(lastEndKey, hri.getStartKey()));
209 lastEndKey = hri.getEndKey();
210 }
211 Assert.assertTrue(Bytes.equals(lastEndKey, HConstants.EMPTY_END_ROW));
212 LOG.info("-------");
213 Threads.sleep(10 + random.nextInt(50));
214 } catch (Throwable e) {
215 ex = e;
216 Assert.fail(StringUtils.stringifyException(e));
217 }
218 }
219 }
220 void rethrowExceptionIfAny() throws Throwable {
221 if (ex != null) { throw ex; }
222 }
223 }
224
225 RegionMetaSplitter regionMetaSplitter = new RegionMetaSplitter();
226 MetaScannerVerifier metaScannerVerifier = new MetaScannerVerifier();
227
228 Thread regionMetaSplitterThread = new Thread(regionMetaSplitter);
229 Thread metaScannerVerifierThread = new Thread(metaScannerVerifier);
230
231 regionMetaSplitterThread.start();
232 metaScannerVerifierThread.start();
233
234 Threads.sleep(runtime);
235
236 regionMetaSplitter.stop("test finished");
237 metaScannerVerifier.stop("test finished");
238
239 regionMetaSplitterThread.join();
240 metaScannerVerifierThread.join();
241
242 regionMetaSplitter.rethrowExceptionIfAny();
243 metaScannerVerifier.rethrowExceptionIfAny();
244 }
245
246 @org.junit.Rule
247 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
248 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
249 }
250