View Javadoc

1   /*
2    * Copyright The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.master;
21  
22  import static org.junit.Assert.assertEquals;
23  import static org.junit.Assert.assertFalse;
24  import static org.junit.Assert.assertTrue;
25  import static org.junit.Assert.fail;
26  
27  import java.io.IOException;
28  import java.util.List;
29  import java.util.Random;
30  import java.util.concurrent.Callable;
31  import java.util.concurrent.CountDownLatch;
32  import java.util.concurrent.ExecutionException;
33  import java.util.concurrent.ExecutorService;
34  import java.util.concurrent.Executors;
35  import java.util.concurrent.Future;
36  
37  import org.apache.commons.logging.Log;
38  import org.apache.commons.logging.LogFactory;
39  import org.apache.hadoop.conf.Configuration;
40  import org.apache.hadoop.hbase.Chore;
41  import org.apache.hadoop.hbase.TableName;
42  import org.apache.hadoop.hbase.HBaseTestingUtility;
43  import org.apache.hadoop.hbase.HColumnDescriptor;
44  import org.apache.hadoop.hbase.HRegionInfo;
45  import org.apache.hadoop.hbase.HTableDescriptor;
46  import org.apache.hadoop.hbase.testclassification.LargeTests;
47  import org.apache.hadoop.hbase.NotServingRegionException;
48  import org.apache.hadoop.hbase.ServerName;
49  import org.apache.hadoop.hbase.TableNotDisabledException;
50  import org.apache.hadoop.hbase.Waiter;
51  import org.apache.hadoop.hbase.client.HBaseAdmin;
52  import org.apache.hadoop.hbase.coprocessor.BaseMasterObserver;
53  import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
54  import org.apache.hadoop.hbase.coprocessor.ObserverContext;
55  import org.apache.hadoop.hbase.exceptions.LockTimeoutException;
56  import org.apache.hadoop.hbase.regionserver.HRegion;
57  import org.apache.hadoop.hbase.util.Bytes;
58  import org.apache.hadoop.hbase.util.LoadTestTool;
59  import org.apache.hadoop.hbase.util.StoppableImplementation;
60  import org.apache.hadoop.hbase.util.Threads;
61  import org.apache.hadoop.hbase.zookeeper.ZKUtil;
62  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
63  import org.junit.After;
64  import org.junit.Test;
65  import org.junit.experimental.categories.Category;
66  
67  /**
68   * Tests the default table lock manager
69   */
70  @Category(LargeTests.class)
71  public class TestTableLockManager {
72  
73    private static final Log LOG =
74      LogFactory.getLog(TestTableLockManager.class);
75  
76    private static final TableName TABLE_NAME =
77        TableName.valueOf("TestTableLevelLocks");
78  
79    private static final String FAMILY1 = "f1";
80    private static final byte[] FAMILY = Bytes.toBytes(FAMILY1);
81  
82    private static final byte[] NEW_FAMILY = Bytes.toBytes("f2");
83  
84    private final HBaseTestingUtility TEST_UTIL =
85      new HBaseTestingUtility();
86  
87    private static final CountDownLatch deleteColumn = new CountDownLatch(1);
88    private static final CountDownLatch addColumn = new CountDownLatch(1);
89  
90    public void prepareMiniCluster() throws Exception {
91      TEST_UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable", true);
92      TEST_UTIL.startMiniCluster(2);
93      TEST_UTIL.createTable(TABLE_NAME, new String[] { FAMILY1, "f3" });
94    }
95  
96    public void prepareMiniZkCluster() throws Exception {
97      TEST_UTIL.startMiniZKCluster(1);
98    }
99  
100   @After
101   public void tearDown() throws Exception {
102     TEST_UTIL.shutdownMiniCluster();
103   }
104 
105   @Test(timeout = 600000)
106   public void testLockTimeoutException() throws Exception {
107     Configuration conf = TEST_UTIL.getConfiguration();
108     conf.setInt(TableLockManager.TABLE_WRITE_LOCK_TIMEOUT_MS, 3000);
109     prepareMiniCluster();
110     HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
111     master.getCoprocessorHost().load(TestLockTimeoutExceptionMasterObserver.class,
112         0, TEST_UTIL.getConfiguration());
113 
114     ExecutorService executor = Executors.newSingleThreadExecutor();
115     Future<Object> shouldFinish = executor.submit(new Callable<Object>() {
116       @Override
117       public Object call() throws Exception {
118         HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
119         admin.deleteColumn(TABLE_NAME, FAMILY);
120         return null;
121       }
122     });
123 
124     deleteColumn.await();
125 
126     try {
127       HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
128       admin.addColumn(TABLE_NAME, new HColumnDescriptor(NEW_FAMILY));
129       fail("Was expecting TableLockTimeoutException");
130     } catch (LockTimeoutException ex) {
131       //expected
132     }
133     shouldFinish.get();
134   }
135 
136   public static class TestLockTimeoutExceptionMasterObserver extends BaseMasterObserver {
137     @Override
138     public void preDeleteColumnHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
139         TableName tableName, byte[] c) throws IOException {
140       deleteColumn.countDown();
141     }
142     @Override
143     public void postDeleteColumnHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
144         TableName tableName, byte[] c) throws IOException {
145       Threads.sleep(10000);
146     }
147 
148     @Override
149     public void preAddColumnHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
150         TableName tableName, HColumnDescriptor column) throws IOException {
151       fail("Add column should have timeouted out for acquiring the table lock");
152     }
153   }
154 
155   @Test(timeout = 600000)
156   public void testAlterAndDisable() throws Exception {
157     prepareMiniCluster();
158     // Send a request to alter a table, then sleep during
159     // the alteration phase. In the mean time, from another
160     // thread, send a request to disable, and then delete a table.
161 
162     HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
163     master.getCoprocessorHost().load(TestAlterAndDisableMasterObserver.class,
164         0, TEST_UTIL.getConfiguration());
165 
166     ExecutorService executor = Executors.newFixedThreadPool(2);
167     Future<Object> alterTableFuture = executor.submit(new Callable<Object>() {
168       @Override
169       public Object call() throws Exception {
170         HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
171         admin.addColumn(TABLE_NAME, new HColumnDescriptor(NEW_FAMILY));
172         LOG.info("Added new column family");
173         HTableDescriptor tableDesc = admin.getTableDescriptor(TABLE_NAME);
174         assertTrue(tableDesc.getFamiliesKeys().contains(NEW_FAMILY));
175         return null;
176       }
177     });
178     Future<Object> disableTableFuture = executor.submit(new Callable<Object>() {
179       @Override
180       public Object call() throws Exception {
181         HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
182         admin.disableTable(TABLE_NAME);
183         assertTrue(admin.isTableDisabled(TABLE_NAME));
184         admin.deleteTable(TABLE_NAME);
185         assertFalse(admin.tableExists(TABLE_NAME));
186         return null;
187       }
188     });
189 
190     try {
191       disableTableFuture.get();
192       alterTableFuture.get();
193     } catch (ExecutionException e) {
194       if (e.getCause() instanceof AssertionError) {
195         throw (AssertionError) e.getCause();
196       }
197       throw e;
198     }
199   }
200 
201   public static class TestAlterAndDisableMasterObserver extends BaseMasterObserver {
202     @Override
203     public void preAddColumnHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
204         TableName tableName, HColumnDescriptor column) throws IOException {
205       LOG.debug("addColumn called");
206       addColumn.countDown();
207     }
208 
209     @Override
210     public void postAddColumnHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
211         TableName tableName, HColumnDescriptor column) throws IOException {
212       Threads.sleep(6000);
213       try {
214         ctx.getEnvironment().getMasterServices().checkTableModifiable(tableName);
215       } catch(TableNotDisabledException expected) {
216         //pass
217         return;
218       } catch(IOException ex) {
219       }
220       fail("was expecting the table to be enabled");
221     }
222 
223     @Override
224     public void preDisableTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
225                                 TableName tableName) throws IOException {
226       try {
227         LOG.debug("Waiting for addColumn to be processed first");
228         //wait for addColumn to be processed first
229         addColumn.await();
230         LOG.debug("addColumn started, we can continue");
231       } catch (InterruptedException ex) {
232         LOG.warn("Sleep interrupted while waiting for addColumn countdown");
233       }
234     }
235 
236     @Override
237     public void postDisableTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
238                                         TableName tableName) throws IOException {
239       Threads.sleep(3000);
240     }
241   }
242 
243   @Test(timeout = 600000)
244   public void testDelete() throws Exception {
245     prepareMiniCluster();
246 
247     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
248     admin.disableTable(TABLE_NAME);
249     admin.deleteTable(TABLE_NAME);
250 
251     //ensure that znode for the table node has been deleted
252     final ZooKeeperWatcher zkWatcher = TEST_UTIL.getZooKeeperWatcher();
253     final String znode = ZKUtil.joinZNode(zkWatcher.tableLockZNode, TABLE_NAME.getNameAsString());
254     
255     TEST_UTIL.waitFor(5000, new Waiter.Predicate<Exception>() {
256       @Override
257       public boolean evaluate() throws Exception {
258         int ver = ZKUtil.checkExists(zkWatcher, znode);
259         return ver < 0;
260       }
261     });
262     int ver = ZKUtil.checkExists(zkWatcher,
263       ZKUtil.joinZNode(zkWatcher.tableLockZNode, TABLE_NAME.getNameAsString()));
264     assertTrue("Unexpected znode version " + ver, ver < 0);
265 
266   }
267 
268 
269   @Test(timeout = 600000)
270   public void testReapAllTableLocks() throws Exception {
271     prepareMiniZkCluster();
272     ServerName serverName = ServerName.valueOf("localhost:10000", 0);
273     final TableLockManager lockManager = TableLockManager.createTableLockManager(
274         TEST_UTIL.getConfiguration(), TEST_UTIL.getZooKeeperWatcher(), serverName);
275 
276     String tables[] = {"table1", "table2", "table3", "table4"};
277     ExecutorService executor = Executors.newFixedThreadPool(6);
278 
279     final CountDownLatch writeLocksObtained = new CountDownLatch(4);
280     final CountDownLatch writeLocksAttempted = new CountDownLatch(10);
281     //TODO: read lock tables
282 
283     //6 threads will be stuck waiting for the table lock
284     for (int i = 0; i < tables.length; i++) {
285       final String table = tables[i];
286       for (int j = 0; j < i+1; j++) { //i+1 write locks attempted for table[i]
287         executor.submit(new Callable<Void>() {
288           @Override
289           public Void call() throws Exception {
290             writeLocksAttempted.countDown();
291             lockManager.writeLock(TableName.valueOf(table),
292                 "testReapAllTableLocks").acquire();
293             writeLocksObtained.countDown();
294             return null;
295           }
296         });
297       }
298     }
299 
300     writeLocksObtained.await();
301     writeLocksAttempted.await();
302 
303     //now reap all table locks
304     lockManager.reapWriteLocks();
305 
306     TEST_UTIL.getConfiguration().setInt(TableLockManager.TABLE_WRITE_LOCK_TIMEOUT_MS, 0);
307     TableLockManager zeroTimeoutLockManager = TableLockManager.createTableLockManager(
308           TEST_UTIL.getConfiguration(), TEST_UTIL.getZooKeeperWatcher(), serverName);
309 
310     //should not throw table lock timeout exception
311     zeroTimeoutLockManager.writeLock(
312         TableName.valueOf(tables[tables.length - 1]),
313         "zero timeout")
314       .acquire();
315 
316     executor.shutdownNow();
317   }
318 
319   @Test(timeout = 600000)
320   public void testTableReadLock() throws Exception {
321     // test plan: write some data to the table. Continuously alter the table and
322     // force splits
323     // concurrently until we have 5 regions. verify the data just in case.
324     // Every region should contain the same table descriptor
325     // This is not an exact test
326     prepareMiniCluster();
327     LoadTestTool loadTool = new LoadTestTool();
328     loadTool.setConf(TEST_UTIL.getConfiguration());
329     int numKeys = 10000;
330     final TableName tableName = TableName.valueOf("testTableReadLock");
331     final HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
332     final HTableDescriptor desc = new HTableDescriptor(tableName);
333     final byte[] family = Bytes.toBytes("test_cf");
334     desc.addFamily(new HColumnDescriptor(family));
335     admin.createTable(desc); // create with one region
336 
337     // write some data, not much
338     int ret = loadTool.run(new String[] { "-tn", tableName.getNameAsString(), "-write",
339         String.format("%d:%d:%d", 1, 10, 10), "-num_keys", String.valueOf(numKeys), "-skip_init" });
340     if (0 != ret) {
341       String errorMsg = "Load failed with error code " + ret;
342       LOG.error(errorMsg);
343       fail(errorMsg);
344     }
345 
346     int familyValues = admin.getTableDescriptor(tableName).getFamily(family).getValues().size();
347     StoppableImplementation stopper = new StoppableImplementation();
348 
349     //alter table every 10 sec
350     Chore alterThread = new Chore("Alter Chore", 10000, stopper) {
351       @Override
352       protected void chore() {
353         Random random = new Random();
354         try {
355           HTableDescriptor htd = admin.getTableDescriptor(tableName);
356           String val = String.valueOf(random.nextInt());
357           htd.getFamily(family).setValue(val, val);
358           desc.getFamily(family).setValue(val, val); // save it for later
359                                                      // control
360           admin.modifyTable(tableName, htd);
361         } catch (Exception ex) {
362           LOG.warn("Caught exception", ex);
363           fail(ex.getMessage());
364         }
365       }
366     };
367 
368     //split table every 5 sec
369     Chore splitThread = new Chore("Split thread", 5000, stopper) {
370       @Override
371       public void chore() {
372         try {
373           HRegion region = TEST_UTIL.getSplittableRegion(tableName, -1);
374           if (region != null) {
375             byte[] regionName = region.getRegionName();
376             admin.flush(regionName);
377             admin.compact(regionName);
378             admin.split(regionName);
379           } else {
380             LOG.warn("Could not find suitable region for the table.  Possibly the " +
381               "region got closed and the attempts got over before " +
382               "the region could have got reassigned.");
383           }
384         } catch (NotServingRegionException nsre) {
385           // the region may be in transition
386           LOG.warn("Caught exception", nsre);
387         } catch (Exception ex) {
388           LOG.warn("Caught exception", ex);
389           fail(ex.getMessage());
390         }
391       }
392     };
393 
394     alterThread.start();
395     splitThread.start();
396     while (true) {
397       List<HRegionInfo> regions = admin.getTableRegions(tableName);
398       LOG.info(String.format("Table #regions: %d regions: %s:", regions.size(), regions));
399       assertEquals(admin.getTableDescriptor(tableName), desc);
400       for (HRegion region : TEST_UTIL.getMiniHBaseCluster().getRegions(tableName)) {
401         assertEquals(desc, region.getTableDesc());
402       }
403       if (regions.size() >= 5) {
404         break;
405       }
406       Threads.sleep(1000);
407     }
408     stopper.stop("test finished");
409 
410     int newFamilyValues = admin.getTableDescriptor(tableName).getFamily(family).getValues().size();
411     LOG.info(String.format("Altered the table %d times", newFamilyValues - familyValues));
412     assertTrue(newFamilyValues > familyValues); // at least one alter went
413                                                 // through
414 
415     ret = loadTool.run(new String[] { "-tn", tableName.getNameAsString(), "-read", "100:10",
416         "-num_keys", String.valueOf(numKeys), "-skip_init" });
417     if (0 != ret) {
418       String errorMsg = "Verify failed with error code " + ret;
419       LOG.error(errorMsg);
420       fail(errorMsg);
421     }
422 
423     admin.close();
424   }
425 
426 }