1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.catalog;
20
21 import static org.junit.Assert.assertTrue;
22
23 import java.io.IOException;
24 import java.net.ConnectException;
25 import java.util.concurrent.atomic.AtomicInteger;
26
27 import junit.framework.Assert;
28
29 import org.apache.commons.logging.Log;
30 import org.apache.commons.logging.LogFactory;
31 import org.apache.hadoop.conf.Configuration;
32 import org.apache.hadoop.hbase.Abortable;
33 import org.apache.hadoop.hbase.TableName;
34 import org.apache.hadoop.hbase.HBaseTestingUtility;
35 import org.apache.hadoop.hbase.HConstants;
36 import org.apache.hadoop.hbase.HRegionInfo;
37 import org.apache.hadoop.hbase.HRegionLocation;
38 import org.apache.hadoop.hbase.MediumTests;
39 import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
40 import org.apache.hadoop.hbase.ServerName;
41 import org.apache.hadoop.hbase.client.HConnection;
42 import org.apache.hadoop.hbase.client.HConnectionManager;
43 import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
44 import org.apache.hadoop.hbase.client.Result;
45 import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
46 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
47 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoRequest;
48 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
49 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetRequest;
50 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetResponse;
51 import org.apache.hadoop.hbase.util.Threads;
52 import org.apache.hadoop.hbase.zookeeper.MetaRegionTracker;
53 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
54 import org.apache.hadoop.util.Progressable;
55 import org.apache.zookeeper.KeeperException;
56 import org.junit.After;
57 import org.junit.AfterClass;
58 import org.junit.Before;
59 import org.junit.BeforeClass;
60 import org.junit.Test;
61 import org.junit.experimental.categories.Category;
62 import org.mockito.Mockito;
63
64 import com.google.protobuf.RpcController;
65 import com.google.protobuf.ServiceException;
66
67
68
69
70 @Category(MediumTests.class)
71 public class TestCatalogTracker {
72 private static final Log LOG = LogFactory.getLog(TestCatalogTracker.class);
73 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
74 private static final ServerName SN =
75 new ServerName("example.org", 1234, System.currentTimeMillis());
76 private ZooKeeperWatcher watcher;
77 private Abortable abortable;
78
79 @BeforeClass public static void beforeClass() throws Exception {
80
81 UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 3);
82 UTIL.startMiniZKCluster();
83 }
84
85 @AfterClass public static void afterClass() throws IOException {
86 UTIL.getZkCluster().shutdown();
87 }
88
89 @Before public void before() throws IOException {
90 this.abortable = new Abortable() {
91 @Override
92 public void abort(String why, Throwable e) {
93 LOG.info(why, e);
94 }
95
96 @Override
97 public boolean isAborted() {
98 return false;
99 }
100 };
101 this.watcher = new ZooKeeperWatcher(UTIL.getConfiguration(),
102 this.getClass().getSimpleName(), this.abortable, true);
103 }
104
105 @After public void after() {
106 this.watcher.close();
107 }
108
109 private CatalogTracker constructAndStartCatalogTracker(final HConnection c)
110 throws IOException, InterruptedException {
111 CatalogTracker ct = new CatalogTracker(this.watcher, UTIL.getConfiguration(),
112 c, this.abortable);
113 ct.start();
114 return ct;
115 }
116
117
118
119
120
121
122
123 @Test public void testThatIfMETAMovesWeAreNotified()
124 throws IOException, InterruptedException, KeeperException {
125 HConnection connection = Mockito.mock(HConnection.class);
126 constructAndStartCatalogTracker(connection);
127 try {
128 MetaRegionTracker.setMetaLocation(this.watcher,
129 new ServerName("example.com", 1234, System.currentTimeMillis()));
130 } finally {
131
132
133 MetaRegionTracker.deleteMetaLocation(this.watcher);
134 }
135 }
136
137
138
139
140
141
142
143 @Test public void testInterruptWaitOnMeta()
144 throws IOException, InterruptedException, ServiceException {
145 final ClientProtos.ClientService.BlockingInterface client =
146 Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
147 HConnection connection = mockConnection(null, client);
148 try {
149 Mockito.when(client.get((RpcController)Mockito.any(), (GetRequest)Mockito.any())).
150 thenReturn(GetResponse.newBuilder().build());
151 final CatalogTracker ct = constructAndStartCatalogTracker(connection);
152 ServerName meta = ct.getMetaLocation();
153 Assert.assertNull(meta);
154 Thread t = new Thread() {
155 @Override
156 public void run() {
157 try {
158 ct.waitForMeta();
159 } catch (InterruptedException e) {
160 throw new RuntimeException("Interrupted", e);
161 }
162 }
163 };
164 t.start();
165 while (!t.isAlive())
166 Threads.sleep(1);
167 Threads.sleep(1);
168 assertTrue(t.isAlive());
169 ct.stop();
170
171 t.join();
172 } finally {
173 HConnectionManager.deleteConnection(UTIL.getConfiguration());
174 }
175 }
176
177 private void testVerifyMetaRegionLocationWithException(Exception ex)
178 throws IOException, InterruptedException, KeeperException, ServiceException {
179
180 final ClientProtos.ClientService.BlockingInterface implementation =
181 Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
182 HConnection connection = mockConnection(null, implementation);
183 try {
184
185 Mockito.when(implementation.get((RpcController) Mockito.any(), (GetRequest) Mockito.any())).
186 thenThrow(new ServiceException(ex));
187
188 final CatalogTracker ct = constructAndStartCatalogTracker(connection);
189 try {
190 MetaRegionTracker.setMetaLocation(this.watcher, SN);
191 long timeout = UTIL.getConfiguration().
192 getLong("hbase.catalog.verification.timeout", 1000);
193 Assert.assertFalse(ct.verifyMetaRegionLocation(timeout));
194 } finally {
195
196
197 MetaRegionTracker.deleteMetaLocation(this.watcher);
198 }
199 } finally {
200
201 HConnectionManager.deleteConnection(UTIL.getConfiguration());
202 }
203 }
204
205
206
207
208
209
210
211
212 @Test
213 public void testGetMetaServerConnectionFails()
214 throws IOException, InterruptedException, KeeperException, ServiceException {
215 testVerifyMetaRegionLocationWithException(new ConnectException("Connection refused"));
216 }
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232 @Test
233 public void testVerifyMetaRegionServerNotRunning()
234 throws IOException, InterruptedException, KeeperException, ServiceException {
235 testVerifyMetaRegionLocationWithException(new ServerNotRunningYetException("mock"));
236 }
237
238
239
240
241
242
243
244
245 @Test
246 public void testVerifyMetaRegionLocationFails()
247 throws IOException, InterruptedException, KeeperException, ServiceException {
248 HConnection connection = Mockito.mock(HConnection.class);
249 ServiceException connectException =
250 new ServiceException(new ConnectException("Connection refused"));
251 final AdminProtos.AdminService.BlockingInterface implementation =
252 Mockito.mock(AdminProtos.AdminService.BlockingInterface.class);
253 Mockito.when(implementation.getRegionInfo((RpcController)Mockito.any(),
254 (GetRegionInfoRequest)Mockito.any())).thenThrow(connectException);
255 Mockito.when(connection.getAdmin(Mockito.any(ServerName.class), Mockito.anyBoolean())).
256 thenReturn(implementation);
257 final CatalogTracker ct = constructAndStartCatalogTracker(connection);
258 try {
259 MetaRegionTracker.setMetaLocation(this.watcher,
260 new ServerName("example.com", 1234, System.currentTimeMillis()));
261 Assert.assertFalse(ct.verifyMetaRegionLocation(100));
262 } finally {
263
264
265 MetaRegionTracker.deleteMetaLocation(this.watcher);
266 }
267 }
268
269 @Test (expected = NotAllMetaRegionsOnlineException.class)
270 public void testTimeoutWaitForMeta()
271 throws IOException, InterruptedException {
272 HConnection connection = Mockito.mock(HConnection.class);
273 final CatalogTracker ct = constructAndStartCatalogTracker(connection);
274 ct.waitForMeta(100);
275 }
276
277
278
279
280
281
282
283 @Test public void testNoTimeoutWaitForMeta()
284 throws IOException, InterruptedException, KeeperException {
285 HConnection connection = Mockito.mock(HConnection.class);
286 final CatalogTracker ct = constructAndStartCatalogTracker(connection);
287 ServerName hsa = ct.getMetaLocation();
288 Assert.assertNull(hsa);
289
290
291 Thread t = new WaitOnMetaThread(ct);
292 startWaitAliveThenWaitItLives(t, 1);
293
294 hsa = setMetaLocation();
295
296 t.join();
297
298 Assert.assertTrue(ct.getMetaLocation().equals(hsa));
299 }
300
301 private ServerName setMetaLocation() throws KeeperException {
302 MetaRegionTracker.setMetaLocation(this.watcher, SN);
303 return SN;
304 }
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321 private HConnection mockConnection(final AdminProtos.AdminService.BlockingInterface admin,
322 final ClientProtos.ClientService.BlockingInterface client)
323 throws IOException {
324 HConnection connection =
325 HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration());
326 Mockito.doNothing().when(connection).close();
327
328 final HRegionLocation anyLocation =
329 new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, SN);
330 Mockito.when(connection.getRegionLocation((TableName) Mockito.any(),
331 (byte[]) Mockito.any(), Mockito.anyBoolean())).
332 thenReturn(anyLocation);
333 Mockito.when(connection.locateRegion((TableName) Mockito.any(),
334 (byte[]) Mockito.any())).
335 thenReturn(anyLocation);
336 if (admin != null) {
337
338 Mockito.when(connection.getAdmin(Mockito.any(ServerName.class))).
339 thenReturn(admin);
340 }
341 if (client != null) {
342
343 Mockito.when(connection.getClient(Mockito.any(ServerName.class))).
344 thenReturn(client);
345 }
346 return connection;
347 }
348
349
350
351
352
353
354 private Result getMetaTableRowResult() throws IOException {
355 return MetaMockingUtil.getMetaTableRowResult(HRegionInfo.FIRST_META_REGIONINFO, SN);
356 }
357
358 private void startWaitAliveThenWaitItLives(final Thread t, final int ms) {
359 t.start();
360 while(!t.isAlive()) {
361
362 }
363
364 Threads.sleep(ms);
365 Assert.assertTrue("Assert " + t.getName() + " still waiting", t.isAlive());
366 }
367
368 class CountingProgressable implements Progressable {
369 final AtomicInteger counter = new AtomicInteger(0);
370 @Override
371 public void progress() {
372 this.counter.incrementAndGet();
373 }
374 }
375
376
377
378
379 class WaitOnMetaThread extends Thread {
380 final CatalogTracker ct;
381
382 WaitOnMetaThread(final CatalogTracker ct) {
383 super("WaitOnMeta");
384 this.ct = ct;
385 }
386
387 @Override
388 public void run() {
389 try {
390 doWaiting();
391 } catch (InterruptedException e) {
392 throw new RuntimeException("Failed wait", e);
393 }
394 LOG.info("Exiting " + getName());
395 }
396
397 void doWaiting() throws InterruptedException {
398 try {
399 while (this.ct.waitForMeta(100) == null);
400 } catch (NotAllMetaRegionsOnlineException e) {
401
402 }
403 }
404 }
405
406 }