View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase;
20  
21  import com.google.common.collect.Sets;
22  import org.apache.commons.logging.Log;
23  import org.apache.commons.logging.LogFactory;
24  import org.apache.hadoop.fs.FileSystem;
25  import org.apache.hadoop.fs.Path;
26  import org.apache.hadoop.hbase.client.Get;
27  import org.apache.hadoop.hbase.client.HBaseAdmin;
28  import org.apache.hadoop.hbase.client.HTable;
29  import org.apache.hadoop.hbase.client.Put;
30  import org.apache.hadoop.hbase.constraint.ConstraintException;
31  import org.apache.hadoop.hbase.master.HMaster;
32  import org.apache.hadoop.hbase.migration.NamespaceUpgrade;
33  import org.apache.hadoop.hbase.util.Bytes;
34  import org.apache.hadoop.hbase.util.FSUtils;
35  import org.apache.hadoop.hbase.zookeeper.ZKUtil;
36  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
37  import org.junit.AfterClass;
38  import org.junit.Before;
39  import org.junit.BeforeClass;
40  import org.junit.Test;
41  import org.junit.experimental.categories.Category;
42  
43  import java.io.IOException;
44  import java.util.Set;
45  
46  import static org.junit.Assert.*;
47  
48  @Category(MediumTests.class)
49  public class TestNamespace {
50    protected static final Log LOG = LogFactory.getLog(TestNamespace.class);
51    private static HMaster master;
52    protected final static int NUM_SLAVES_BASE = 4;
53    private static HBaseTestingUtility TEST_UTIL;
54    protected static HBaseAdmin admin;
55    protected static HBaseCluster cluster;
56    private static ZKNamespaceManager zkNamespaceManager;
57    private String prefix = "TestNamespace";
58  
59  
60    @BeforeClass
61    public static void setUp() throws Exception {
62      TEST_UTIL = new HBaseTestingUtility();
63      TEST_UTIL.getConfiguration().setInt("hbase.namespacejanitor.interval", 5000);
64      TEST_UTIL.startMiniCluster(NUM_SLAVES_BASE);
65      admin = TEST_UTIL.getHBaseAdmin();
66      cluster = TEST_UTIL.getHBaseCluster();
67      master = ((MiniHBaseCluster)cluster).getMaster();
68      zkNamespaceManager =
69          new ZKNamespaceManager(master.getZooKeeperWatcher());
70      zkNamespaceManager.start();
71      LOG.info("Done initializing cluster");
72    }
73  
74    @AfterClass
75    public static void tearDown() throws Exception {
76      TEST_UTIL.shutdownMiniCluster();
77    }
78  
79    @Before
80    public void beforeMethod() throws IOException {
81      for (HTableDescriptor desc : admin.listTables(prefix+".*")) {
82        admin.disableTable(desc.getTableName());
83        admin.deleteTable(desc.getTableName());
84      }
85      for (NamespaceDescriptor ns : admin.listNamespaceDescriptors()) {
86        if (ns.getName().startsWith(prefix)) {
87          admin.deleteNamespace(ns.getName());
88        }
89      }
90    }
91  
92    @Test
93    public void verifyReservedNS() throws IOException {
94      //verify existence of reserved namespaces
95      NamespaceDescriptor ns =
96          admin.getNamespaceDescriptor(NamespaceDescriptor.DEFAULT_NAMESPACE.getName());
97      assertNotNull(ns);
98      assertEquals(ns.getName(), NamespaceDescriptor.DEFAULT_NAMESPACE.getName());
99      assertNotNull(zkNamespaceManager.get(NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR));
100 
101     ns = admin.getNamespaceDescriptor(NamespaceDescriptor.SYSTEM_NAMESPACE.getName());
102     assertNotNull(ns);
103     assertEquals(ns.getName(), NamespaceDescriptor.SYSTEM_NAMESPACE.getName());
104     assertNotNull(zkNamespaceManager.get(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR));
105 
106     assertEquals(2, admin.listNamespaceDescriptors().length);
107 
108     //verify existence of system tables
109     Set<TableName> systemTables = Sets.newHashSet(
110         TableName.META_TABLE_NAME,
111         TableName.NAMESPACE_TABLE_NAME);
112     HTableDescriptor[] descs =
113         admin.listTableDescriptorsByNamespace(NamespaceDescriptor.SYSTEM_NAMESPACE.getName());
114     assertEquals(systemTables.size(), descs.length);
115     for (HTableDescriptor desc : descs) {
116       assertTrue(systemTables.contains(desc.getTableName()));
117     }
118     //verify system tables aren't listed
119     assertEquals(0, admin.listTables().length);
120     
121     //Try creating default and system namespaces. 
122     boolean exceptionCaught = false;
123     try {
124       admin.createNamespace(NamespaceDescriptor.DEFAULT_NAMESPACE);
125     } catch (IOException exp) {
126       LOG.warn(exp);
127       exceptionCaught = true;
128     } finally {
129       assertTrue(exceptionCaught);
130     }
131 
132     exceptionCaught = false;
133     try {
134       admin.createNamespace(NamespaceDescriptor.SYSTEM_NAMESPACE);
135     } catch (IOException exp) {
136       LOG.warn(exp);
137       exceptionCaught = true;
138     } finally {
139       assertTrue(exceptionCaught);
140     }
141   }
142   
143   @Test
144   public void testDeleteReservedNS() throws Exception {
145     boolean exceptionCaught = false;
146     try {
147       admin.deleteNamespace(NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR);
148     } catch (IOException exp) {
149       LOG.warn(exp);
150       exceptionCaught = true;
151     } finally {
152       assertTrue(exceptionCaught);
153     }
154 
155     try {
156       admin.deleteNamespace(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR);
157     } catch (IOException exp) {
158       LOG.warn(exp);
159       exceptionCaught = true;
160     } finally {
161       assertTrue(exceptionCaught);
162     }
163   }
164 
165   @Test
166   public void createRemoveTest() throws Exception {
167     String testName = "createRemoveTest";
168     String nsName = prefix+"_"+testName;
169     LOG.info(testName);
170 
171     //create namespace and verify
172     admin.createNamespace(NamespaceDescriptor.create(nsName).build());
173     assertEquals(3, admin.listNamespaceDescriptors().length);
174     TEST_UTIL.waitFor(60000, new Waiter.Predicate<Exception>() {
175       @Override
176       public boolean evaluate() throws Exception {
177         return zkNamespaceManager.list().size() == 3;
178       }
179     });
180     assertNotNull(zkNamespaceManager.get(nsName));
181     //remove namespace and verify
182     admin.deleteNamespace(nsName);
183     assertEquals(2, admin.listNamespaceDescriptors().length);
184     assertEquals(2, zkNamespaceManager.list().size());
185     assertNull(zkNamespaceManager.get(nsName));
186   }
187 
188   @Test
189   public void createDoubleTest() throws IOException, InterruptedException {
190     String testName = "createDoubleTest";
191     String nsName = prefix+"_"+testName;
192     LOG.info(testName);
193 
194     byte[] tableName = Bytes.toBytes("my_table");
195     byte[] tableNameFoo = Bytes.toBytes(nsName+".my_table");
196     //create namespace and verify
197     admin.createNamespace(NamespaceDescriptor.create(nsName).build());
198     TEST_UTIL.createTable(tableName, Bytes.toBytes(nsName));
199     TEST_UTIL.createTable(tableNameFoo,Bytes.toBytes(nsName));
200     assertEquals(2, admin.listTables().length);
201     assertNotNull(admin
202         .getTableDescriptor(tableName));
203     assertNotNull(admin
204         .getTableDescriptor(tableNameFoo));
205     //remove namespace and verify
206     admin.disableTable(tableName);
207     admin.deleteTable(tableName);
208     assertEquals(1, admin.listTables().length);
209   }
210 
211   @Test
212   public void createTableTest() throws IOException, InterruptedException {
213     String testName = "createTableTest";
214     String nsName = prefix+"_"+testName;
215     LOG.info(testName);
216 
217     HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(nsName+":my_table"));
218     HColumnDescriptor colDesc = new HColumnDescriptor("my_cf");
219     desc.addFamily(colDesc);
220     try {
221       admin.createTable(desc);
222       fail("Expected no namespace constraint exception");
223     } catch (ConstraintException ex) {
224     }
225     //create table and in new namespace
226     admin.createNamespace(NamespaceDescriptor.create(nsName).build());
227     admin.createTable(desc);
228     TEST_UTIL.waitTableAvailable(desc.getTableName().getName(), 10000);
229     FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
230     assertTrue(fs.exists(
231         new Path(master.getMasterFileSystem().getRootDir(),
232             new Path(HConstants.BASE_NAMESPACE_DIR,
233                 new Path(nsName, desc.getTableName().getQualifierAsString())))));
234     assertEquals(1, admin.listTables().length);
235 
236     //verify non-empty namespace can't be removed
237     try {
238       admin.deleteNamespace(nsName);
239       fail("Expected non-empty namespace constraint exception");
240     } catch (Exception ex) {
241       LOG.info("Caught expected exception: " + ex);
242     }
243 
244     //sanity check try to write and read from table
245     HTable table = new HTable(TEST_UTIL.getConfiguration(), desc.getTableName());
246     Put p = new Put(Bytes.toBytes("row1"));
247     p.add(Bytes.toBytes("my_cf"),Bytes.toBytes("my_col"),Bytes.toBytes("value1"));
248     table.put(p);
249     //flush and read from disk to make sure directory changes are working
250     admin.flush(desc.getTableName().getName());
251     Get g = new Get(Bytes.toBytes("row1"));
252     assertTrue(table.exists(g));
253 
254     //normal case of removing namespace
255     TEST_UTIL.deleteTable(desc.getTableName());
256     admin.deleteNamespace(nsName);
257   }
258 
259   @Test
260   public void createTableInDefaultNamespace() throws Exception {
261     HTableDescriptor desc = new HTableDescriptor(TableName.valueOf("default_table"));
262     HColumnDescriptor colDesc = new HColumnDescriptor("cf1");
263     desc.addFamily(colDesc);
264     admin.createTable(desc);
265     assertTrue(admin.listTables().length == 1);  
266     admin.disableTable(desc.getTableName());
267     admin.deleteTable(desc.getTableName());
268   }
269 
270   @Test
271   public void createTableInSystemNamespace() throws Exception {
272     String tableName = "hbase:createTableInSystemNamespace";
273     HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
274     HColumnDescriptor colDesc = new HColumnDescriptor("cf1");
275     desc.addFamily(colDesc);
276     admin.createTable(desc);
277     assertEquals(0, admin.listTables().length);
278     assertTrue(admin.tableExists(Bytes.toBytes(tableName)));
279     admin.disableTable(desc.getTableName());
280     admin.deleteTable(desc.getTableName());
281   }
282 
283   @Test
284   public void testNamespaceJanitor() throws Exception {
285     FileSystem fs = TEST_UTIL.getTestFileSystem();
286 
287     int fsCount = fs.listStatus(new Path(FSUtils.getRootDir(TEST_UTIL.getConfiguration()),
288         HConstants.BASE_NAMESPACE_DIR)).length;
289     Path fakeNSPath =
290         FSUtils.getNamespaceDir(FSUtils.getRootDir(TEST_UTIL.getConfiguration()), "foo");
291     assertTrue(fs.mkdirs(fakeNSPath));
292 
293     String fakeZnode = ZKUtil.joinZNode(ZooKeeperWatcher.namespaceZNode, "foo");
294     int zkCount = ZKUtil.listChildrenNoWatch(TEST_UTIL.getZooKeeperWatcher(),
295         ZooKeeperWatcher.namespaceZNode).size();
296     ZKUtil.createWithParents(TEST_UTIL.getZooKeeperWatcher(), fakeZnode);
297     Thread.sleep(10000);
298 
299     //verify namespace count is the same and orphan is removed
300     assertFalse(fs.exists(fakeNSPath));
301     assertEquals(fsCount, fs.listStatus(new Path(FSUtils.getRootDir(TEST_UTIL.getConfiguration()),
302             HConstants.BASE_NAMESPACE_DIR)).length);
303 
304     assertEquals(-1, ZKUtil.checkExists(TEST_UTIL.getZooKeeperWatcher(), fakeZnode));
305     assertEquals(zkCount,
306         ZKUtil.listChildrenNoWatch(TEST_UTIL.getZooKeeperWatcher(),
307             ZooKeeperWatcher.namespaceZNode).size());
308   }
309 
310 }