View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.master;
20  
21  import java.io.FileNotFoundException;
22  import java.io.IOException;
23  import java.io.InterruptedIOException;
24  import java.util.NavigableSet;
25  
26  import org.apache.commons.logging.Log;
27  import org.apache.commons.logging.LogFactory;
28  import org.apache.hadoop.hbase.classification.InterfaceAudience;
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.fs.FileStatus;
31  import org.apache.hadoop.fs.FileSystem;
32  import org.apache.hadoop.hbase.CellUtil;
33  import org.apache.hadoop.hbase.HConstants;
34  import org.apache.hadoop.hbase.HRegionInfo;
35  import org.apache.hadoop.hbase.HTableDescriptor;
36  import org.apache.hadoop.hbase.NamespaceDescriptor;
37  import org.apache.hadoop.hbase.NamespaceExistException;
38  import org.apache.hadoop.hbase.NamespaceNotFoundException;
39  import org.apache.hadoop.hbase.TableName;
40  import org.apache.hadoop.hbase.ZKNamespaceManager;
41  import org.apache.hadoop.hbase.MetaTableAccessor;
42  import org.apache.hadoop.hbase.client.Delete;
43  import org.apache.hadoop.hbase.client.Get;
44  import org.apache.hadoop.hbase.client.Put;
45  import org.apache.hadoop.hbase.client.Result;
46  import org.apache.hadoop.hbase.client.ResultScanner;
47  import org.apache.hadoop.hbase.client.Table;
48  import org.apache.hadoop.hbase.constraint.ConstraintException;
49  import org.apache.hadoop.hbase.master.handler.CreateTableHandler;
50  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
51  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
52  import org.apache.hadoop.hbase.util.Bytes;
53  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
54  import org.apache.hadoop.hbase.util.FSUtils;
55  
56  import com.google.common.collect.Sets;
57  
58  /**
59   * This is a helper class used to manage the namespace
60   * metadata that is stored in TableName.NAMESPACE_TABLE_NAME
61   * It also mirrors updates to the ZK store by forwarding updates to
62   * {@link org.apache.hadoop.hbase.ZKNamespaceManager}
63   */
64  @InterfaceAudience.Private
65  public class TableNamespaceManager {
66    private static final Log LOG = LogFactory.getLog(TableNamespaceManager.class);
67  
68    private Configuration conf;
69    private MasterServices masterServices;
70    private Table nsTable;
71    private ZKNamespaceManager zkNamespaceManager;
72    private boolean initialized;
73  
74    static final String NS_INIT_TIMEOUT = "hbase.master.namespace.init.timeout";
75    static final int DEFAULT_NS_INIT_TIMEOUT = 300000;
76  
77    public TableNamespaceManager(MasterServices masterServices) {
78      this.masterServices = masterServices;
79      this.conf = masterServices.getConfiguration();
80    }
81  
82    public void start() throws IOException {
83      if (!MetaTableAccessor.tableExists(masterServices.getConnection(),
84          TableName.NAMESPACE_TABLE_NAME)) {
85        LOG.info("Namespace table not found. Creating...");
86        createNamespaceTable(masterServices);
87      }
88  
89      try {
90        // Wait for the namespace table to be assigned.
91        // If timed out, we will move ahead without initializing it.
92        // So that it should be initialized later on lazily.
93        long startTime = EnvironmentEdgeManager.currentTime();
94        int timeout = conf.getInt(NS_INIT_TIMEOUT, DEFAULT_NS_INIT_TIMEOUT);
95        while (!isTableAssigned()) {
96          if (EnvironmentEdgeManager.currentTime() - startTime + 100 > timeout) {
97            // We can't do anything if ns is not online.
98            throw new IOException("Timedout " + timeout + "ms waiting for namespace table to " +
99              "be assigned");
100         }
101         Thread.sleep(100);
102       }
103     } catch (InterruptedException e) {
104       throw (InterruptedIOException)new InterruptedIOException().initCause(e);
105     }
106 
107     // initialize namespace table
108     isTableAvailableAndInitialized();
109   }
110 
111   private synchronized Table getNamespaceTable() throws IOException {
112     if (!isTableAvailableAndInitialized()) {
113       throw new IOException(this.getClass().getName() + " isn't ready to serve");
114     }
115     return nsTable;
116   }
117 
118 
119   public synchronized NamespaceDescriptor get(String name) throws IOException {
120     if (!isTableAvailableAndInitialized()) return null;
121     return zkNamespaceManager.get(name);
122   }
123 
124   public synchronized void create(NamespaceDescriptor ns) throws IOException {
125     create(getNamespaceTable(), ns);
126   }
127 
128   public synchronized void update(NamespaceDescriptor ns) throws IOException {
129     Table table = getNamespaceTable();
130     if (get(table, ns.getName()) == null) {
131       throw new NamespaceNotFoundException(ns.getName());
132     }
133     upsert(table, ns);
134   }
135 
136   private NamespaceDescriptor get(Table table, String name) throws IOException {
137     Result res = table.get(new Get(Bytes.toBytes(name)));
138     if (res.isEmpty()) {
139       return null;
140     }
141     byte[] val = CellUtil.cloneValue(res.getColumnLatestCell(
142         HTableDescriptor.NAMESPACE_FAMILY_INFO_BYTES, HTableDescriptor.NAMESPACE_COL_DESC_BYTES));
143     return
144         ProtobufUtil.toNamespaceDescriptor(
145             HBaseProtos.NamespaceDescriptor.parseFrom(val));
146   }
147 
148   private void create(Table table, NamespaceDescriptor ns) throws IOException {
149     if (get(table, ns.getName()) != null) {
150       throw new NamespaceExistException(ns.getName());
151     }
152     FileSystem fs = masterServices.getMasterFileSystem().getFileSystem();
153     fs.mkdirs(FSUtils.getNamespaceDir(
154         masterServices.getMasterFileSystem().getRootDir(), ns.getName()));
155     upsert(table, ns);
156   }
157 
158   private void upsert(Table table, NamespaceDescriptor ns) throws IOException {
159     Put p = new Put(Bytes.toBytes(ns.getName()));
160     p.addImmutable(HTableDescriptor.NAMESPACE_FAMILY_INFO_BYTES,
161         HTableDescriptor.NAMESPACE_COL_DESC_BYTES,
162         ProtobufUtil.toProtoNamespaceDescriptor(ns).toByteArray());
163     table.put(p);
164     try {
165       zkNamespaceManager.update(ns);
166     } catch(IOException ex) {
167       String msg = "Failed to update namespace information in ZK. Aborting.";
168       LOG.fatal(msg, ex);
169       masterServices.abort(msg, ex);
170     }
171   }
172 
173   public synchronized void remove(String name) throws IOException {
174     if (get(name) == null) {
175       throw new NamespaceNotFoundException(name);
176     }
177     if (NamespaceDescriptor.RESERVED_NAMESPACES.contains(name)) {
178       throw new ConstraintException("Reserved namespace "+name+" cannot be removed.");
179     }
180     int tableCount;
181     try {
182       tableCount = masterServices.listTableDescriptorsByNamespace(name).size();
183     } catch (FileNotFoundException fnfe) {
184       throw new NamespaceNotFoundException(name);
185     }
186     if (tableCount > 0) {
187       throw new ConstraintException("Only empty namespaces can be removed. " +
188           "Namespace "+name+" has "+tableCount+" tables");
189     }
190     Delete d = new Delete(Bytes.toBytes(name));
191     getNamespaceTable().delete(d);
192     //don't abort if cleanup isn't complete
193     //it will be replaced on new namespace creation
194     zkNamespaceManager.remove(name);
195     FileSystem fs = masterServices.getMasterFileSystem().getFileSystem();
196     for(FileStatus status :
197             fs.listStatus(FSUtils.getNamespaceDir(
198                 masterServices.getMasterFileSystem().getRootDir(), name))) {
199       if (!HConstants.HBASE_NON_TABLE_DIRS.contains(status.getPath().getName())) {
200         throw new IOException("Namespace directory contains table dir: "+status.getPath());
201       }
202     }
203     if (!fs.delete(FSUtils.getNamespaceDir(
204         masterServices.getMasterFileSystem().getRootDir(), name), true)) {
205       throw new IOException("Failed to remove namespace: "+name);
206     }
207   }
208 
209   public synchronized NavigableSet<NamespaceDescriptor> list() throws IOException {
210     NavigableSet<NamespaceDescriptor> ret =
211         Sets.newTreeSet(NamespaceDescriptor.NAMESPACE_DESCRIPTOR_COMPARATOR);
212     ResultScanner scanner = getNamespaceTable().getScanner(HTableDescriptor.NAMESPACE_FAMILY_INFO_BYTES);
213     try {
214       for(Result r : scanner) {
215         byte[] val = CellUtil.cloneValue(r.getColumnLatestCell(
216           HTableDescriptor.NAMESPACE_FAMILY_INFO_BYTES,
217           HTableDescriptor.NAMESPACE_COL_DESC_BYTES));
218         ret.add(ProtobufUtil.toNamespaceDescriptor(
219             HBaseProtos.NamespaceDescriptor.parseFrom(val)));
220       }
221     } finally {
222       scanner.close();
223     }
224     return ret;
225   }
226 
227   private void createNamespaceTable(MasterServices masterServices) throws IOException {
228     HRegionInfo newRegions[] = new HRegionInfo[]{
229         new HRegionInfo(HTableDescriptor.NAMESPACE_TABLEDESC.getTableName(), null, null)};
230 
231     //we need to create the table this way to bypass
232     //checkInitialized
233     masterServices.getExecutorService()
234         .submit(new CreateTableHandler(masterServices,
235             masterServices.getMasterFileSystem(),
236             HTableDescriptor.NAMESPACE_TABLEDESC,
237             masterServices.getConfiguration(),
238             newRegions,
239             masterServices).prepare());
240   }
241 
242   /**
243    * This method checks if the namespace table is assigned and then
244    * tries to create its HTable. If it was already created before, it also makes
245    * sure that the connection isn't closed.
246    * @return true if the namespace table manager is ready to serve, false
247    * otherwise
248    * @throws IOException
249    */
250   @SuppressWarnings("deprecation")
251   public synchronized boolean isTableAvailableAndInitialized() throws IOException {
252     // Did we already get a table? If so, still make sure it's available
253     if (initialized) {
254       this.nsTable = this.masterServices.getConnection().getTable(TableName.NAMESPACE_TABLE_NAME);
255       return true;
256     }
257 
258     // Now check if the table is assigned, if not then fail fast
259     if (isTableAssigned()) {
260       try {
261         nsTable = this.masterServices.getConnection().getTable(TableName.NAMESPACE_TABLE_NAME);
262         zkNamespaceManager = new ZKNamespaceManager(masterServices.getZooKeeper());
263         zkNamespaceManager.start();
264 
265         if (get(nsTable, NamespaceDescriptor.DEFAULT_NAMESPACE.getName()) == null) {
266           create(nsTable, NamespaceDescriptor.DEFAULT_NAMESPACE);
267         }
268         if (get(nsTable, NamespaceDescriptor.SYSTEM_NAMESPACE.getName()) == null) {
269           create(nsTable, NamespaceDescriptor.SYSTEM_NAMESPACE);
270         }
271 
272         ResultScanner scanner = nsTable.getScanner(HTableDescriptor.NAMESPACE_FAMILY_INFO_BYTES);
273         try {
274           for (Result result : scanner) {
275             byte[] val =  CellUtil.cloneValue(result.getColumnLatest(
276                 HTableDescriptor.NAMESPACE_FAMILY_INFO_BYTES,
277                 HTableDescriptor.NAMESPACE_COL_DESC_BYTES));
278             NamespaceDescriptor ns =
279                 ProtobufUtil.toNamespaceDescriptor(
280                     HBaseProtos.NamespaceDescriptor.parseFrom(val));
281             zkNamespaceManager.update(ns);
282           }
283         } finally {
284           scanner.close();
285         }
286         initialized = true;
287         return true;
288       } catch (IOException ie) {
289         LOG.warn("Caught exception in initializing namespace table manager", ie);
290         if (nsTable != null) {
291           nsTable.close();
292         }
293         throw ie;
294       }
295     }
296     return false;
297   }
298 
299   private boolean isTableAssigned() {
300     return !masterServices.getAssignmentManager().getRegionStates().
301       getRegionsOfTable(TableName.NAMESPACE_TABLE_NAME).isEmpty();
302   }
303 }