1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.master;
21
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertFalse;
24 import static org.junit.Assert.assertTrue;
25 import static org.junit.Assert.fail;
26
27 import java.io.IOException;
28 import java.util.List;
29 import java.util.Random;
30 import java.util.concurrent.Callable;
31 import java.util.concurrent.CountDownLatch;
32 import java.util.concurrent.ExecutionException;
33 import java.util.concurrent.ExecutorService;
34 import java.util.concurrent.Executors;
35 import java.util.concurrent.Future;
36
37 import org.apache.commons.logging.Log;
38 import org.apache.commons.logging.LogFactory;
39 import org.apache.hadoop.conf.Configuration;
40 import org.apache.hadoop.hbase.Chore;
41 import org.apache.hadoop.hbase.TableName;
42 import org.apache.hadoop.hbase.HBaseTestingUtility;
43 import org.apache.hadoop.hbase.HColumnDescriptor;
44 import org.apache.hadoop.hbase.HRegionInfo;
45 import org.apache.hadoop.hbase.HTableDescriptor;
46 import org.apache.hadoop.hbase.testclassification.LargeTests;
47 import org.apache.hadoop.hbase.NotServingRegionException;
48 import org.apache.hadoop.hbase.ServerName;
49 import org.apache.hadoop.hbase.TableNotDisabledException;
50 import org.apache.hadoop.hbase.Waiter;
51 import org.apache.hadoop.hbase.client.HBaseAdmin;
52 import org.apache.hadoop.hbase.coprocessor.BaseMasterObserver;
53 import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
54 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
55 import org.apache.hadoop.hbase.exceptions.LockTimeoutException;
56 import org.apache.hadoop.hbase.regionserver.HRegion;
57 import org.apache.hadoop.hbase.util.Bytes;
58 import org.apache.hadoop.hbase.util.LoadTestTool;
59 import org.apache.hadoop.hbase.util.StoppableImplementation;
60 import org.apache.hadoop.hbase.util.Threads;
61 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
62 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
63 import org.junit.After;
64 import org.junit.Test;
65 import org.junit.experimental.categories.Category;
66
67
68
69
70 @Category(LargeTests.class)
71 public class TestTableLockManager {
72
73 private static final Log LOG =
74 LogFactory.getLog(TestTableLockManager.class);
75
76 private static final TableName TABLE_NAME =
77 TableName.valueOf("TestTableLevelLocks");
78
79 private static final String FAMILY1 = "f1";
80 private static final byte[] FAMILY = Bytes.toBytes(FAMILY1);
81
82 private static final byte[] NEW_FAMILY = Bytes.toBytes("f2");
83
84 private final HBaseTestingUtility TEST_UTIL =
85 new HBaseTestingUtility();
86
87 private static final CountDownLatch deleteColumn = new CountDownLatch(1);
88 private static final CountDownLatch addColumn = new CountDownLatch(1);
89
90 public void prepareMiniCluster() throws Exception {
91 TEST_UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable", true);
92 TEST_UTIL.startMiniCluster(2);
93 TEST_UTIL.createTable(TABLE_NAME, new String[] { FAMILY1, "f3" });
94 }
95
96 public void prepareMiniZkCluster() throws Exception {
97 TEST_UTIL.startMiniZKCluster(1);
98 }
99
100 @After
101 public void tearDown() throws Exception {
102 TEST_UTIL.shutdownMiniCluster();
103 }
104
105 @Test(timeout = 600000)
106 public void testLockTimeoutException() throws Exception {
107 Configuration conf = TEST_UTIL.getConfiguration();
108 conf.setInt(TableLockManager.TABLE_WRITE_LOCK_TIMEOUT_MS, 3000);
109 prepareMiniCluster();
110 HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
111 master.getCoprocessorHost().load(TestLockTimeoutExceptionMasterObserver.class,
112 0, TEST_UTIL.getConfiguration());
113
114 ExecutorService executor = Executors.newSingleThreadExecutor();
115 Future<Object> shouldFinish = executor.submit(new Callable<Object>() {
116 @Override
117 public Object call() throws Exception {
118 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
119 admin.deleteColumn(TABLE_NAME, FAMILY);
120 return null;
121 }
122 });
123
124 deleteColumn.await();
125
126 try {
127 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
128 admin.addColumn(TABLE_NAME, new HColumnDescriptor(NEW_FAMILY));
129 fail("Was expecting TableLockTimeoutException");
130 } catch (LockTimeoutException ex) {
131
132 }
133 shouldFinish.get();
134 }
135
136 public static class TestLockTimeoutExceptionMasterObserver extends BaseMasterObserver {
137 @Override
138 public void preDeleteColumnHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
139 TableName tableName, byte[] c) throws IOException {
140 deleteColumn.countDown();
141 }
142 @Override
143 public void postDeleteColumnHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
144 TableName tableName, byte[] c) throws IOException {
145 Threads.sleep(10000);
146 }
147
148 @Override
149 public void preAddColumnHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
150 TableName tableName, HColumnDescriptor column) throws IOException {
151 fail("Add column should have timeouted out for acquiring the table lock");
152 }
153 }
154
155 @Test(timeout = 600000)
156 public void testAlterAndDisable() throws Exception {
157 prepareMiniCluster();
158
159
160
161
162 HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
163 master.getCoprocessorHost().load(TestAlterAndDisableMasterObserver.class,
164 0, TEST_UTIL.getConfiguration());
165
166 ExecutorService executor = Executors.newFixedThreadPool(2);
167 Future<Object> alterTableFuture = executor.submit(new Callable<Object>() {
168 @Override
169 public Object call() throws Exception {
170 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
171 admin.addColumn(TABLE_NAME, new HColumnDescriptor(NEW_FAMILY));
172 LOG.info("Added new column family");
173 HTableDescriptor tableDesc = admin.getTableDescriptor(TABLE_NAME);
174 assertTrue(tableDesc.getFamiliesKeys().contains(NEW_FAMILY));
175 return null;
176 }
177 });
178 Future<Object> disableTableFuture = executor.submit(new Callable<Object>() {
179 @Override
180 public Object call() throws Exception {
181 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
182 admin.disableTable(TABLE_NAME);
183 assertTrue(admin.isTableDisabled(TABLE_NAME));
184 admin.deleteTable(TABLE_NAME);
185 assertFalse(admin.tableExists(TABLE_NAME));
186 return null;
187 }
188 });
189
190 try {
191 disableTableFuture.get();
192 alterTableFuture.get();
193 } catch (ExecutionException e) {
194 if (e.getCause() instanceof AssertionError) {
195 throw (AssertionError) e.getCause();
196 }
197 throw e;
198 }
199 }
200
201 public static class TestAlterAndDisableMasterObserver extends BaseMasterObserver {
202 @Override
203 public void preAddColumnHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
204 TableName tableName, HColumnDescriptor column) throws IOException {
205 LOG.debug("addColumn called");
206 addColumn.countDown();
207 }
208
209 @Override
210 public void postAddColumnHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
211 TableName tableName, HColumnDescriptor column) throws IOException {
212 Threads.sleep(6000);
213 try {
214 ctx.getEnvironment().getMasterServices().checkTableModifiable(tableName);
215 } catch(TableNotDisabledException expected) {
216
217 return;
218 } catch(IOException ex) {
219 }
220 fail("was expecting the table to be enabled");
221 }
222
223 @Override
224 public void preDisableTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
225 TableName tableName) throws IOException {
226 try {
227 LOG.debug("Waiting for addColumn to be processed first");
228
229 addColumn.await();
230 LOG.debug("addColumn started, we can continue");
231 } catch (InterruptedException ex) {
232 LOG.warn("Sleep interrupted while waiting for addColumn countdown");
233 }
234 }
235
236 @Override
237 public void postDisableTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
238 TableName tableName) throws IOException {
239 Threads.sleep(3000);
240 }
241 }
242
243 @Test(timeout = 600000)
244 public void testDelete() throws Exception {
245 prepareMiniCluster();
246
247 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
248 admin.disableTable(TABLE_NAME);
249 admin.deleteTable(TABLE_NAME);
250
251
252 final ZooKeeperWatcher zkWatcher = TEST_UTIL.getZooKeeperWatcher();
253 final String znode = ZKUtil.joinZNode(zkWatcher.tableLockZNode, TABLE_NAME.getNameAsString());
254
255 TEST_UTIL.waitFor(5000, new Waiter.Predicate<Exception>() {
256 @Override
257 public boolean evaluate() throws Exception {
258 int ver = ZKUtil.checkExists(zkWatcher, znode);
259 return ver < 0;
260 }
261 });
262 int ver = ZKUtil.checkExists(zkWatcher,
263 ZKUtil.joinZNode(zkWatcher.tableLockZNode, TABLE_NAME.getNameAsString()));
264 assertTrue("Unexpected znode version " + ver, ver < 0);
265
266 }
267
268
269 @Test(timeout = 600000)
270 public void testReapAllTableLocks() throws Exception {
271 prepareMiniZkCluster();
272 ServerName serverName = ServerName.valueOf("localhost:10000", 0);
273 final TableLockManager lockManager = TableLockManager.createTableLockManager(
274 TEST_UTIL.getConfiguration(), TEST_UTIL.getZooKeeperWatcher(), serverName);
275
276 String tables[] = {"table1", "table2", "table3", "table4"};
277 ExecutorService executor = Executors.newFixedThreadPool(6);
278
279 final CountDownLatch writeLocksObtained = new CountDownLatch(4);
280 final CountDownLatch writeLocksAttempted = new CountDownLatch(10);
281
282
283
284 for (int i = 0; i < tables.length; i++) {
285 final String table = tables[i];
286 for (int j = 0; j < i+1; j++) {
287 executor.submit(new Callable<Void>() {
288 @Override
289 public Void call() throws Exception {
290 writeLocksAttempted.countDown();
291 lockManager.writeLock(TableName.valueOf(table),
292 "testReapAllTableLocks").acquire();
293 writeLocksObtained.countDown();
294 return null;
295 }
296 });
297 }
298 }
299
300 writeLocksObtained.await();
301 writeLocksAttempted.await();
302
303
304 lockManager.reapWriteLocks();
305
306 TEST_UTIL.getConfiguration().setInt(TableLockManager.TABLE_WRITE_LOCK_TIMEOUT_MS, 0);
307 TableLockManager zeroTimeoutLockManager = TableLockManager.createTableLockManager(
308 TEST_UTIL.getConfiguration(), TEST_UTIL.getZooKeeperWatcher(), serverName);
309
310
311 zeroTimeoutLockManager.writeLock(
312 TableName.valueOf(tables[tables.length - 1]),
313 "zero timeout")
314 .acquire();
315
316 executor.shutdownNow();
317 }
318
319 @Test(timeout = 600000)
320 public void testTableReadLock() throws Exception {
321
322
323
324
325
326 prepareMiniCluster();
327 LoadTestTool loadTool = new LoadTestTool();
328 loadTool.setConf(TEST_UTIL.getConfiguration());
329 int numKeys = 10000;
330 final TableName tableName = TableName.valueOf("testTableReadLock");
331 final HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
332 final HTableDescriptor desc = new HTableDescriptor(tableName);
333 final byte[] family = Bytes.toBytes("test_cf");
334 desc.addFamily(new HColumnDescriptor(family));
335 admin.createTable(desc);
336
337
338 int ret = loadTool.run(new String[] { "-tn", tableName.getNameAsString(), "-write",
339 String.format("%d:%d:%d", 1, 10, 10), "-num_keys", String.valueOf(numKeys), "-skip_init" });
340 if (0 != ret) {
341 String errorMsg = "Load failed with error code " + ret;
342 LOG.error(errorMsg);
343 fail(errorMsg);
344 }
345
346 int familyValues = admin.getTableDescriptor(tableName).getFamily(family).getValues().size();
347 StoppableImplementation stopper = new StoppableImplementation();
348
349
350 Chore alterThread = new Chore("Alter Chore", 10000, stopper) {
351 @Override
352 protected void chore() {
353 Random random = new Random();
354 try {
355 HTableDescriptor htd = admin.getTableDescriptor(tableName);
356 String val = String.valueOf(random.nextInt());
357 htd.getFamily(family).setValue(val, val);
358 desc.getFamily(family).setValue(val, val);
359
360 admin.modifyTable(tableName, htd);
361 } catch (Exception ex) {
362 LOG.warn("Caught exception", ex);
363 fail(ex.getMessage());
364 }
365 }
366 };
367
368
369 Chore splitThread = new Chore("Split thread", 5000, stopper) {
370 @Override
371 public void chore() {
372 try {
373 HRegion region = TEST_UTIL.getSplittableRegion(tableName, -1);
374 if (region != null) {
375 byte[] regionName = region.getRegionName();
376 admin.flush(regionName);
377 admin.compact(regionName);
378 admin.split(regionName);
379 } else {
380 LOG.warn("Could not find suitable region for the table. Possibly the " +
381 "region got closed and the attempts got over before " +
382 "the region could have got reassigned.");
383 }
384 } catch (NotServingRegionException nsre) {
385
386 LOG.warn("Caught exception", nsre);
387 } catch (Exception ex) {
388 LOG.warn("Caught exception", ex);
389 fail(ex.getMessage());
390 }
391 }
392 };
393
394 alterThread.start();
395 splitThread.start();
396 while (true) {
397 List<HRegionInfo> regions = admin.getTableRegions(tableName);
398 LOG.info(String.format("Table #regions: %d regions: %s:", regions.size(), regions));
399 assertEquals(admin.getTableDescriptor(tableName), desc);
400 for (HRegion region : TEST_UTIL.getMiniHBaseCluster().getRegions(tableName)) {
401 assertEquals(desc, region.getTableDesc());
402 }
403 if (regions.size() >= 5) {
404 break;
405 }
406 Threads.sleep(1000);
407 }
408 stopper.stop("test finished");
409
410 int newFamilyValues = admin.getTableDescriptor(tableName).getFamily(family).getValues().size();
411 LOG.info(String.format("Altered the table %d times", newFamilyValues - familyValues));
412 assertTrue(newFamilyValues > familyValues);
413
414
415 ret = loadTool.run(new String[] { "-tn", tableName.getNameAsString(), "-read", "100:10",
416 "-num_keys", String.valueOf(numKeys), "-skip_init" });
417 if (0 != ret) {
418 String errorMsg = "Verify failed with error code " + ret;
419 LOG.error(errorMsg);
420 fail(errorMsg);
421 }
422
423 admin.close();
424 }
425
426 }