View Javadoc

1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.master.handler;
21  
22  import java.io.IOException;
23  import java.util.List;
24  import java.util.Map;
25  import java.util.NavigableMap;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.hbase.HConstants;
30  import org.apache.hadoop.hbase.HRegionInfo;
31  import org.apache.hadoop.hbase.HServerInfo;
32  import org.apache.hadoop.hbase.Server;
33  import org.apache.hadoop.hbase.catalog.CatalogTracker;
34  import org.apache.hadoop.hbase.catalog.MetaEditor;
35  import org.apache.hadoop.hbase.catalog.MetaReader;
36  import org.apache.hadoop.hbase.client.Result;
37  import org.apache.hadoop.hbase.executor.EventHandler;
38  import org.apache.hadoop.hbase.master.AssignmentManager;
39  import org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
40  import org.apache.hadoop.hbase.master.DeadServer;
41  import org.apache.hadoop.hbase.master.MasterServices;
42  import org.apache.hadoop.hbase.master.ServerManager;
43  import org.apache.hadoop.hbase.util.Bytes;
44  import org.apache.hadoop.hbase.util.Writables;
45  import org.apache.zookeeper.KeeperException;
46  
47  /**
48   * Process server shutdown.
49   * Server-to-handle must be already in the deadservers lists.  See
50   * {@link ServerManager#expireServer(HServerInfo)}.
51   */
52  public class ServerShutdownHandler extends EventHandler {
53    private static final Log LOG = LogFactory.getLog(ServerShutdownHandler.class);
54    private final HServerInfo hsi;
55    private final Server server;
56    private final MasterServices services;
57    private final DeadServer deadServers;
58  
59    public ServerShutdownHandler(final Server server, final MasterServices services,
60        final DeadServer deadServers, final HServerInfo hsi) {
61      this(server, services, deadServers, hsi, EventType.M_SERVER_SHUTDOWN);
62    }
63  
64    ServerShutdownHandler(final Server server, final MasterServices services,
65        final DeadServer deadServers, final HServerInfo hsi, EventType type) {
66      super(server, type);
67      this.hsi = hsi;
68      this.server = server;
69      this.services = services;
70      this.deadServers = deadServers;
71      if (!this.deadServers.contains(hsi.getServerName())) {
72        LOG.warn(hsi.getServerName() + " is NOT in deadservers; it should be!");
73      }
74    }
75  
76    /**
77     * Before assign the ROOT region, ensure it haven't 
78     *  been assigned by other place
79     * <p>
80     * Under some scenarios, the ROOT region can be opened twice, so it seemed online
81     * in two regionserver at the same time.
82     * If the ROOT region has been assigned, so the operation can be canceled. 
83     * @throws InterruptedException
84     * @throws IOException
85     * @throws KeeperException
86     */
87    private void verifyAndAssignRoot() 
88    throws InterruptedException, IOException, KeeperException {
89      long timeout = this.server.getConfiguration().
90        getLong("hbase.catalog.verification.timeout", 1000);
91      if (!this.server.getCatalogTracker().verifyRootRegionLocation(timeout)) {
92        this.services.getAssignmentManager().assignRoot();     
93      }
94    }
95    
96    /**
97     * Failed many times, shutdown processing
98     * @throws IOException
99     */
100   private void verifyAndAssignRootWithRetries() throws IOException {
101     int iTimes = this.server.getConfiguration().getInt(
102         "hbase.catalog.verification.retries", 10);
103 
104     long waitTime = this.server.getConfiguration().getLong(
105         "hbase.catalog.verification.timeout", 1000);
106 
107     int iFlag = 0;
108     while (true) {
109       try {
110         verifyAndAssignRoot();
111         break;
112       } catch (KeeperException e) {
113         this.server.abort("In server shutdown processing, assigning root", e);
114         throw new IOException("Aborting", e);
115       } catch (Exception e) {
116         if (iFlag >= iTimes) {
117           this.server.abort("verifyAndAssignRoot failed after" + iTimes
118               + " times retries, aborting", e);
119           throw new IOException("Aborting", e);
120         }
121         try {
122           Thread.sleep(waitTime);
123         } catch (InterruptedException e1) {
124           LOG.warn("Interrupted when is the thread sleep", e1);
125           Thread.currentThread().interrupt();
126           throw new IOException("Interrupted", e1);
127         }
128         iFlag++;
129       }
130     }
131   }
132   
133   /**
134    * @return True if the server we are processing was carrying <code>-ROOT-</code>
135    */
136   boolean isCarryingRoot() {
137     return false;
138   }
139 
140   /**
141    * @return True if the server we are processing was carrying <code>.META.</code>
142    */
143   boolean isCarryingMeta() {
144     return false;
145   }
146 
147   @Override
148   public void process() throws IOException {
149     final String serverName = this.hsi.getServerName();
150 
151     LOG.info("Splitting logs for " + serverName);
152     this.services.getMasterFileSystem().splitLog(serverName);
153 
154     // Clean out anything in regions in transition.  Being conservative and
155     // doing after log splitting.  Could do some states before -- OPENING?
156     // OFFLINE? -- and then others after like CLOSING that depend on log
157     // splitting.
158     List<RegionState> regionsInTransition =
159       this.services.getAssignmentManager().processServerShutdown(this.hsi);
160 
161     // Assign root and meta if we were carrying them.
162     if (isCarryingRoot()) { // -ROOT-
163       verifyAndAssignRootWithRetries();
164     }
165 
166     // Carrying meta?
167     if (isCarryingMeta()) this.services.getAssignmentManager().assignMeta();
168 
169     // Wait on meta to come online; we need it to progress.
170     // TODO: Best way to hold strictly here?  We should build this retry logic
171     //       into the MetaReader operations themselves.
172     NavigableMap<HRegionInfo, Result> hris = null;
173     while (!this.server.isStopped()) {
174       try {
175         this.server.getCatalogTracker().waitForMeta();
176         hris = MetaReader.getServerUserRegions(this.server.getCatalogTracker(),
177             this.hsi);
178         break;
179       } catch (InterruptedException e) {
180         Thread.currentThread().interrupt();
181         throw new IOException("Interrupted", e);
182       } catch (IOException ioe) {
183         LOG.info("Received exception accessing META during server shutdown of " +
184             serverName + ", retrying META read", ioe);
185       }
186     }
187 
188     // Skip regions that were in transition unless CLOSING or PENDING_CLOSE
189     for (RegionState rit : regionsInTransition) {
190       if (!rit.isClosing() && !rit.isPendingClose()) {
191         LOG.debug("Removed " + rit.getRegion().getRegionNameAsString() +
192           " from list of regions to assign because in RIT");
193         hris.remove(rit.getRegion());
194       }
195     }
196 
197     LOG.info("Reassigning " + (hris == null? 0: hris.size()) +
198       " region(s) that " + serverName +
199       " was carrying (skipping " + regionsInTransition.size() +
200       " regions(s) that are already in transition)");
201 
202     // Iterate regions that were on this server and assign them
203     for (Map.Entry<HRegionInfo, Result> e: hris.entrySet()) {
204       if (processDeadRegion(e.getKey(), e.getValue(),
205           this.services.getAssignmentManager(),
206           this.server.getCatalogTracker())) {
207         this.services.getAssignmentManager().assign(e.getKey(), true);
208       }
209     }
210     this.deadServers.finish(serverName);
211     LOG.info("Finished processing of shutdown of " + serverName);
212   }
213 
214   /**
215    * Process a dead region from a dead RS.  Checks if the region is disabled
216    * or if the region has a partially completed split.
217    * @param hri
218    * @param result
219    * @param assignmentManager
220    * @param catalogTracker
221    * @return Returns true if specified region should be assigned, false if not.
222    * @throws IOException
223    */
224   public static boolean processDeadRegion(HRegionInfo hri, Result result,
225       AssignmentManager assignmentManager, CatalogTracker catalogTracker)
226   throws IOException {
227     // If table is not disabled but the region is offlined,
228     boolean disabled = assignmentManager.getZKTable().isDisabledTable(
229         hri.getTableDesc().getNameAsString());
230     if (disabled) return false;
231     if (hri.isOffline() && hri.isSplit()) {
232       LOG.debug("Offlined and split region " + hri.getRegionNameAsString() +
233         "; checking daughter presence");
234       fixupDaughters(result, assignmentManager, catalogTracker);
235       return false;
236     }
237     return true;
238   }
239 
240   /**
241    * Check that daughter regions are up in .META. and if not, add them.
242    * @param hris All regions for this server in meta.
243    * @param result The contents of the parent row in .META.
244    * @throws IOException
245    */
246   static void fixupDaughters(final Result result,
247       final AssignmentManager assignmentManager,
248       final CatalogTracker catalogTracker)
249   throws IOException {
250     fixupDaughter(result, HConstants.SPLITA_QUALIFIER, assignmentManager,
251       catalogTracker);
252     fixupDaughter(result, HConstants.SPLITB_QUALIFIER, assignmentManager,
253       catalogTracker);
254   }
255 
256   /**
257    * Check individual daughter is up in .META.; fixup if its not.
258    * @param result The contents of the parent row in .META.
259    * @param qualifier Which daughter to check for.
260    * @throws IOException
261    */
262   static void fixupDaughter(final Result result, final byte [] qualifier,
263       final AssignmentManager assignmentManager,
264       final CatalogTracker catalogTracker)
265   throws IOException {
266     HRegionInfo daughter = getHRegionInfo(result, qualifier);
267     if (daughter == null) return;
268     if (isDaughterMissing(catalogTracker, daughter)) {
269       LOG.info("Fixup; missing daughter " + daughter.getRegionNameAsString());
270       MetaEditor.addDaughter(catalogTracker, daughter, null);
271       // And assign it.
272       assignmentManager.assign(daughter, true);
273     } else {
274       LOG.debug("Daughter " + daughter.getRegionNameAsString() + " present");
275     }
276   }
277 
278   /**
279    * Interpret the content of the cell at {@link HConstants#CATALOG_FAMILY} and
280    * <code>qualifier</code> as an HRegionInfo and return it, or null.
281    * @param r Result instance to pull from.
282    * @param qualifier Column family qualifier
283    * @return An HRegionInfo instance or null.
284    * @throws IOException
285    */
286   private static HRegionInfo getHRegionInfo(final Result r, byte [] qualifier)
287   throws IOException {
288     byte [] bytes = r.getValue(HConstants.CATALOG_FAMILY, qualifier);
289     if (bytes == null || bytes.length <= 0) return null;
290     return Writables.getHRegionInfoOrNull(bytes);
291   }
292 
293   /**
294    * Look for presence of the daughter OR of a split of the daughter in .META.
295    * Daughter could have been split over on regionserver before a run of the
296    * catalogJanitor had chance to clear reference from parent.
297    * @param daughter Daughter region to search for.
298    * @throws IOException 
299    */
300   private static boolean isDaughterMissing(final CatalogTracker catalogTracker,
301       final HRegionInfo daughter) throws IOException {
302     FindDaughterVisitor visitor = new FindDaughterVisitor(daughter);
303     // Start the scan at what should be the daughter's row in the .META.
304     // We will either 1., find the daughter or some derivative split of the
305     // daughter (will have same table name and start row at least but will sort
306     // after because has larger regionid -- the regionid is timestamp of region
307     // creation), OR, we will not find anything with same table name and start
308     // row.  If the latter, then assume daughter missing and do fixup.
309     byte [] startrow = daughter.getRegionName();
310     MetaReader.fullScan(catalogTracker, visitor, startrow);
311     return !visitor.foundDaughter();
312   }
313 
314   /**
315    * Looks for daughter.  Sets a flag if daughter or some progeny of daughter
316    * is found up in <code>.META.</code>.
317    */
318   static class FindDaughterVisitor implements MetaReader.Visitor {
319     private final HRegionInfo daughter;
320     private boolean found = false;
321 
322     FindDaughterVisitor(final HRegionInfo daughter) {
323       this.daughter = daughter;
324     }
325 
326     /**
327      * @return True if we found a daughter region during our visiting.
328      */
329     boolean foundDaughter() {
330       return this.found;
331     }
332 
333     @Override
334     public boolean visit(Result r) throws IOException {
335       HRegionInfo hri = getHRegionInfo(r, HConstants.REGIONINFO_QUALIFIER);
336       if (hri == null) {
337         LOG.warn("No serialized HRegionInfo in " + r);
338         return true;
339       }
340       byte [] value = r.getValue(HConstants.CATALOG_FAMILY,
341           HConstants.SERVER_QUALIFIER);
342       // See if daughter is assigned to some server
343       if (value == null) return false;
344 
345       // Now see if we have gone beyond the daughter's startrow.
346       if (!Bytes.equals(daughter.getTableDesc().getName(),
347           hri.getTableDesc().getName())) {
348         // We fell into another table.  Stop scanning.
349         return false;
350       }
351       // If our start rows do not compare, move on.
352       if (!Bytes.equals(daughter.getStartKey(), hri.getStartKey())) {
353         return false;
354       }
355       // Else, table name and start rows compare.  It means that the daughter
356       // or some derivative split of the daughter is up in .META.  Daughter
357       // exists.
358       this.found = true;
359       return false;
360     }
361   }
362 }