View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.master;
20  
21  import java.io.FileNotFoundException;
22  import java.io.IOException;
23  import java.util.NavigableSet;
24  
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.classification.InterfaceAudience;
28  import org.apache.hadoop.conf.Configuration;
29  import org.apache.hadoop.fs.FileStatus;
30  import org.apache.hadoop.fs.FileSystem;
31  import org.apache.hadoop.hbase.CellUtil;
32  import org.apache.hadoop.hbase.HConstants;
33  import org.apache.hadoop.hbase.HRegionInfo;
34  import org.apache.hadoop.hbase.HTableDescriptor;
35  import org.apache.hadoop.hbase.NamespaceDescriptor;
36  import org.apache.hadoop.hbase.NamespaceExistException;
37  import org.apache.hadoop.hbase.NamespaceNotFoundException;
38  import org.apache.hadoop.hbase.TableName;
39  import org.apache.hadoop.hbase.ZKNamespaceManager;
40  import org.apache.hadoop.hbase.catalog.MetaReader;
41  import org.apache.hadoop.hbase.client.Delete;
42  import org.apache.hadoop.hbase.client.Get;
43  import org.apache.hadoop.hbase.client.HTable;
44  import org.apache.hadoop.hbase.client.Put;
45  import org.apache.hadoop.hbase.client.Result;
46  import org.apache.hadoop.hbase.client.ResultScanner;
47  import org.apache.hadoop.hbase.constraint.ConstraintException;
48  import org.apache.hadoop.hbase.master.handler.CreateTableHandler;
49  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
50  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
51  import org.apache.hadoop.hbase.util.Bytes;
52  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
53  import org.apache.hadoop.hbase.util.FSUtils;
54  
55  import com.google.common.collect.Sets;
56  
57  /**
58   * This is a helper class used to manage the namespace
59   * metadata that is stored in TableName.NAMESPACE_TABLE_NAME
60   * It also mirrors updates to the ZK store by forwarding updates to
61   * {@link org.apache.hadoop.hbase.ZKNamespaceManager}
62   */
63  @InterfaceAudience.Private
64  public class TableNamespaceManager {
65    private static final Log LOG = LogFactory.getLog(TableNamespaceManager.class);
66  
67    private Configuration conf;
68    private MasterServices masterServices;
69    private HTable nsTable;
70    private ZKNamespaceManager zkNamespaceManager;
71    private boolean initialized;
72  
73    static final String NS_INIT_TIMEOUT = "hbase.master.namespace.init.timeout";
74    static final int DEFAULT_NS_INIT_TIMEOUT = 60000;
75  
76    public TableNamespaceManager(MasterServices masterServices) {
77      this.masterServices = masterServices;
78      this.conf = masterServices.getConfiguration();
79    }
80  
81    public void start() throws IOException {
82      if (!MetaReader.tableExists(masterServices.getCatalogTracker(),
83          TableName.NAMESPACE_TABLE_NAME)) {
84        LOG.info("Namespace table not found. Creating...");
85        createNamespaceTable(masterServices);
86      }
87  
88      try {
89        // Wait for the namespace table to be assigned.
90        // If timed out, we will move ahead without initializing it.
91        // So that it should be initialized later on lazily.
92        long startTime = EnvironmentEdgeManager.currentTimeMillis();
93        int timeout = conf.getInt(NS_INIT_TIMEOUT, DEFAULT_NS_INIT_TIMEOUT);
94        while (!isTableAssigned()) {
95          if (EnvironmentEdgeManager.currentTimeMillis() - startTime + 100 > timeout) {
96            LOG.warn("Timedout waiting for namespace table to be assigned.");
97            return;
98          }
99          Thread.sleep(100);
100       }
101     } catch (InterruptedException e) {
102       throw new IOException("Wait for namespace table assignment interrupted", e);
103     }
104 
105     // initialize namespace table
106     isTableAvailableAndInitialized();
107   }
108 
109   private synchronized HTable getNamespaceTable() throws IOException {
110     if (!isTableAvailableAndInitialized()) {
111       throw new IOException(this.getClass().getName() + " isn't ready to serve");
112     }
113     return nsTable;
114   }
115 
116 
117   public synchronized NamespaceDescriptor get(String name) throws IOException {
118     return get(getNamespaceTable(), name);
119   }
120 
121   public synchronized void create(NamespaceDescriptor ns) throws IOException {
122     create(getNamespaceTable(), ns);
123   }
124 
125   public synchronized void update(NamespaceDescriptor ns) throws IOException {
126     HTable table = getNamespaceTable();
127     if (get(table, ns.getName()) == null) {
128       throw new NamespaceNotFoundException(ns.getName());
129     }
130     upsert(table, ns);
131   }
132 
133   private NamespaceDescriptor get(HTable table, String name) throws IOException {
134     Result res = table.get(new Get(Bytes.toBytes(name)));
135     if (res.isEmpty()) {
136       return null;
137     }
138     byte[] val = CellUtil.cloneValue(res.getColumnLatestCell(
139         HTableDescriptor.NAMESPACE_FAMILY_INFO_BYTES, HTableDescriptor.NAMESPACE_COL_DESC_BYTES));
140     return
141         ProtobufUtil.toNamespaceDescriptor(
142             HBaseProtos.NamespaceDescriptor.parseFrom(val));
143   }
144 
145   private void create(HTable table, NamespaceDescriptor ns) throws IOException {
146     if (get(table, ns.getName()) != null) {
147       throw new NamespaceExistException(ns.getName());
148     }
149     FileSystem fs = masterServices.getMasterFileSystem().getFileSystem();
150     fs.mkdirs(FSUtils.getNamespaceDir(
151         masterServices.getMasterFileSystem().getRootDir(), ns.getName()));
152     upsert(table, ns);
153   }
154 
155   private void upsert(HTable table, NamespaceDescriptor ns) throws IOException {
156     Put p = new Put(Bytes.toBytes(ns.getName()));
157     p.addImmutable(HTableDescriptor.NAMESPACE_FAMILY_INFO_BYTES,
158         HTableDescriptor.NAMESPACE_COL_DESC_BYTES,
159         ProtobufUtil.toProtoNamespaceDescriptor(ns).toByteArray());
160     table.put(p);
161     try {
162       zkNamespaceManager.update(ns);
163     } catch(IOException ex) {
164       String msg = "Failed to update namespace information in ZK. Aborting.";
165       LOG.fatal(msg, ex);
166       masterServices.abort(msg, ex);
167     }
168   }
169 
170   public synchronized void remove(String name) throws IOException {
171     if (get(name) == null) {
172       throw new NamespaceNotFoundException(name);
173     }
174     if (NamespaceDescriptor.RESERVED_NAMESPACES.contains(name)) {
175       throw new ConstraintException("Reserved namespace "+name+" cannot be removed.");
176     }
177     int tableCount;
178     try {
179       tableCount = masterServices.listTableDescriptorsByNamespace(name).size();
180     } catch (FileNotFoundException fnfe) {
181       throw new NamespaceNotFoundException(name);
182     }
183     if (tableCount > 0) {
184       throw new ConstraintException("Only empty namespaces can be removed. " +
185           "Namespace "+name+" has "+tableCount+" tables");
186     }
187     Delete d = new Delete(Bytes.toBytes(name));
188     getNamespaceTable().delete(d);
189     //don't abort if cleanup isn't complete
190     //it will be replaced on new namespace creation
191     zkNamespaceManager.remove(name);
192     FileSystem fs = masterServices.getMasterFileSystem().getFileSystem();
193     for(FileStatus status :
194             fs.listStatus(FSUtils.getNamespaceDir(
195                 masterServices.getMasterFileSystem().getRootDir(), name))) {
196       if (!HConstants.HBASE_NON_TABLE_DIRS.contains(status.getPath().getName())) {
197         throw new IOException("Namespace directory contains table dir: "+status.getPath());
198       }
199     }
200     if (!fs.delete(FSUtils.getNamespaceDir(
201         masterServices.getMasterFileSystem().getRootDir(), name), true)) {
202       throw new IOException("Failed to remove namespace: "+name);
203     }
204   }
205 
206   public synchronized NavigableSet<NamespaceDescriptor> list() throws IOException {
207     NavigableSet<NamespaceDescriptor> ret =
208         Sets.newTreeSet(NamespaceDescriptor.NAMESPACE_DESCRIPTOR_COMPARATOR);
209     ResultScanner scanner = getNamespaceTable().getScanner(HTableDescriptor.NAMESPACE_FAMILY_INFO_BYTES);
210     try {
211       for(Result r : scanner) {
212         byte[] val = CellUtil.cloneValue(r.getColumnLatestCell(
213           HTableDescriptor.NAMESPACE_FAMILY_INFO_BYTES,
214           HTableDescriptor.NAMESPACE_COL_DESC_BYTES));
215         ret.add(ProtobufUtil.toNamespaceDescriptor(
216             HBaseProtos.NamespaceDescriptor.parseFrom(val)));
217       }
218     } finally {
219       scanner.close();
220     }
221     return ret;
222   }
223 
224   private void createNamespaceTable(MasterServices masterServices) throws IOException {
225     HRegionInfo newRegions[] = new HRegionInfo[]{
226         new HRegionInfo(HTableDescriptor.NAMESPACE_TABLEDESC.getTableName(), null, null)};
227 
228     //we need to create the table this way to bypass
229     //checkInitialized
230     masterServices.getExecutorService()
231         .submit(new CreateTableHandler(masterServices,
232             masterServices.getMasterFileSystem(),
233             HTableDescriptor.NAMESPACE_TABLEDESC,
234             masterServices.getConfiguration(),
235             newRegions,
236             masterServices).prepare());
237   }
238 
239   /**
240    * This method checks if the namespace table is assigned and then
241    * tries to create its HTable. If it was already created before, it also makes
242    * sure that the connection isn't closed.
243    * @return true if the namespace table manager is ready to serve, false
244    * otherwise
245    * @throws IOException
246    */
247   @SuppressWarnings("deprecation")
248   public synchronized boolean isTableAvailableAndInitialized() throws IOException {
249     // Did we already get a table? If so, still make sure it's available
250     if (initialized) {
251       if (nsTable.getConnection().isClosed()) {
252         nsTable = new HTable(conf, TableName.NAMESPACE_TABLE_NAME);
253       }
254       return true;
255     }
256 
257     // Now check if the table is assigned, if not then fail fast
258     if (isTableAssigned()) {
259       try {
260         nsTable = new HTable(conf, TableName.NAMESPACE_TABLE_NAME);
261         zkNamespaceManager = new ZKNamespaceManager(masterServices.getZooKeeper());
262         zkNamespaceManager.start();
263 
264         if (get(nsTable, NamespaceDescriptor.DEFAULT_NAMESPACE.getName()) == null) {
265           create(nsTable, NamespaceDescriptor.DEFAULT_NAMESPACE);
266         }
267         if (get(nsTable, NamespaceDescriptor.SYSTEM_NAMESPACE.getName()) == null) {
268           create(nsTable, NamespaceDescriptor.SYSTEM_NAMESPACE);
269         }
270 
271         ResultScanner scanner = nsTable.getScanner(HTableDescriptor.NAMESPACE_FAMILY_INFO_BYTES);
272         try {
273           for (Result result : scanner) {
274             byte[] val =  CellUtil.cloneValue(result.getColumnLatest(
275                 HTableDescriptor.NAMESPACE_FAMILY_INFO_BYTES,
276                 HTableDescriptor.NAMESPACE_COL_DESC_BYTES));
277             NamespaceDescriptor ns =
278                 ProtobufUtil.toNamespaceDescriptor(
279                     HBaseProtos.NamespaceDescriptor.parseFrom(val));
280             zkNamespaceManager.update(ns);
281           }
282         } finally {
283           scanner.close();
284         }
285         initialized = true;
286         return true;
287       } catch (IOException ie) {
288         LOG.warn("Caught exception in initializing namespace table manager", ie);
289         if (nsTable != null) {
290           nsTable.close();
291         }
292         throw ie;
293       }
294     }
295     return false;
296   }
297 
298   private boolean isTableAssigned() {
299     return !masterServices.getAssignmentManager()
300         .getRegionStates().getRegionsOfTable(TableName.NAMESPACE_TABLE_NAME).isEmpty();
301   }
302 }