View Javadoc

1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.master.handler;
21  
22  import java.io.IOException;
23  import java.util.List;
24  import java.util.Map;
25  import java.util.NavigableMap;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.hbase.HConstants;
30  import org.apache.hadoop.hbase.HRegionInfo;
31  import org.apache.hadoop.hbase.HServerInfo;
32  import org.apache.hadoop.hbase.Server;
33  import org.apache.hadoop.hbase.catalog.CatalogTracker;
34  import org.apache.hadoop.hbase.catalog.MetaEditor;
35  import org.apache.hadoop.hbase.catalog.MetaReader;
36  import org.apache.hadoop.hbase.client.Result;
37  import org.apache.hadoop.hbase.executor.EventHandler;
38  import org.apache.hadoop.hbase.master.AssignmentManager;
39  import org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
40  import org.apache.hadoop.hbase.master.DeadServer;
41  import org.apache.hadoop.hbase.master.MasterServices;
42  import org.apache.hadoop.hbase.master.ServerManager;
43  import org.apache.hadoop.hbase.util.Bytes;
44  import org.apache.hadoop.hbase.util.Writables;
45  import org.apache.zookeeper.KeeperException;
46  
47  /**
48   * Process server shutdown.
49   * Server-to-handle must be already in the deadservers lists.  See
50   * {@link ServerManager#expireServer(HServerInfo)}.
51   */
52  public class ServerShutdownHandler extends EventHandler {
53    private static final Log LOG = LogFactory.getLog(ServerShutdownHandler.class);
54    private final HServerInfo hsi;
55    private final Server server;
56    private final MasterServices services;
57    private final DeadServer deadServers;
58  
59    public ServerShutdownHandler(final Server server, final MasterServices services,
60        final DeadServer deadServers, final HServerInfo hsi) {
61      this(server, services, deadServers, hsi, EventType.M_SERVER_SHUTDOWN);
62    }
63  
64    ServerShutdownHandler(final Server server, final MasterServices services,
65        final DeadServer deadServers, final HServerInfo hsi, EventType type) {
66      super(server, type);
67      this.hsi = hsi;
68      this.server = server;
69      this.services = services;
70      this.deadServers = deadServers;
71      if (!this.deadServers.contains(hsi.getServerName())) {
72        LOG.warn(hsi.getServerName() + " is NOT in deadservers; it should be!");
73      }
74    }
75  
76    /**
77     * @return True if the server we are processing was carrying <code>-ROOT-</code>
78     */
79    boolean isCarryingRoot() {
80      return false;
81    }
82  
83    /**
84     * @return True if the server we are processing was carrying <code>.META.</code>
85     */
86    boolean isCarryingMeta() {
87      return false;
88    }
89  
90    @Override
91    public void process() throws IOException {
92      final String serverName = this.hsi.getServerName();
93  
94      LOG.info("Splitting logs for " + serverName);
95      this.services.getMasterFileSystem().splitLog(serverName);
96  
97      // Clean out anything in regions in transition.  Being conservative and
98      // doing after log splitting.  Could do some states before -- OPENING?
99      // OFFLINE? -- and then others after like CLOSING that depend on log
100     // splitting.
101     List<RegionState> regionsInTransition =
102       this.services.getAssignmentManager().processServerShutdown(this.hsi);
103 
104     // Assign root and meta if we were carrying them.
105     if (isCarryingRoot()) { // -ROOT-
106       try {
107         this.services.getAssignmentManager().assignRoot();
108       } catch (KeeperException e) {
109         this.server.abort("In server shutdown processing, assigning root", e);
110         throw new IOException("Aborting", e);
111       }
112     }
113 
114     // Carrying meta?
115     if (isCarryingMeta()) this.services.getAssignmentManager().assignMeta();
116 
117     // Wait on meta to come online; we need it to progress.
118     // TODO: Best way to hold strictly here?  We should build this retry logic
119     //       into the MetaReader operations themselves.
120     NavigableMap<HRegionInfo, Result> hris = null;
121     while (!this.server.isStopped()) {
122       try {
123         this.server.getCatalogTracker().waitForMeta();
124         hris = MetaReader.getServerUserRegions(this.server.getCatalogTracker(),
125             this.hsi);
126         break;
127       } catch (InterruptedException e) {
128         Thread.currentThread().interrupt();
129         throw new IOException("Interrupted", e);
130       } catch (IOException ioe) {
131         LOG.info("Received exception accessing META during server shutdown of " +
132             serverName + ", retrying META read");
133       }
134     }
135 
136     // Skip regions that were in transition unless CLOSING or PENDING_CLOSE
137     for (RegionState rit : regionsInTransition) {
138       if (!rit.isClosing() && !rit.isPendingClose()) {
139         LOG.debug("Removed " + rit.getRegion().getRegionNameAsString() +
140           " from list of regions to assign because in RIT");
141         hris.remove(rit.getRegion());
142       }
143     }
144 
145     LOG.info("Reassigning " + hris.size() + " region(s) that " + serverName +
146       " was carrying (skipping " + regionsInTransition.size() +
147       " regions(s) that are already in transition)");
148 
149     // Iterate regions that were on this server and assign them
150     for (Map.Entry<HRegionInfo, Result> e: hris.entrySet()) {
151       if (processDeadRegion(e.getKey(), e.getValue(),
152           this.services.getAssignmentManager(),
153           this.server.getCatalogTracker())) {
154         this.services.getAssignmentManager().assign(e.getKey(), true);
155       }
156     }
157     this.deadServers.finish(serverName);
158     LOG.info("Finished processing of shutdown of " + serverName);
159   }
160 
161   /**
162    * Process a dead region from a dead RS.  Checks if the region is disabled
163    * or if the region has a partially completed split.
164    * @param hri
165    * @param result
166    * @param assignmentManager
167    * @param catalogTracker
168    * @return Returns true if specified region should be assigned, false if not.
169    * @throws IOException
170    */
171   public static boolean processDeadRegion(HRegionInfo hri, Result result,
172       AssignmentManager assignmentManager, CatalogTracker catalogTracker)
173   throws IOException {
174     // If table is not disabled but the region is offlined,
175     boolean disabled = assignmentManager.getZKTable().isDisabledTable(
176         hri.getTableDesc().getNameAsString());
177     if (disabled) return false;
178     if (hri.isOffline() && hri.isSplit()) {
179       LOG.debug("Offlined and split region " + hri.getRegionNameAsString() +
180         "; checking daughter presence");
181       fixupDaughters(result, assignmentManager, catalogTracker);
182       return false;
183     }
184     return true;
185   }
186 
187   /**
188    * Check that daughter regions are up in .META. and if not, add them.
189    * @param hris All regions for this server in meta.
190    * @param result The contents of the parent row in .META.
191    * @throws IOException
192    */
193   static void fixupDaughters(final Result result,
194       final AssignmentManager assignmentManager,
195       final CatalogTracker catalogTracker) throws IOException {
196     fixupDaughter(result, HConstants.SPLITA_QUALIFIER, assignmentManager,
197         catalogTracker);
198     fixupDaughter(result, HConstants.SPLITB_QUALIFIER, assignmentManager,
199         catalogTracker);
200   }
201 
202   /**
203    * Check individual daughter is up in .META.; fixup if its not.
204    * @param result The contents of the parent row in .META.
205    * @param qualifier Which daughter to check for.
206    * @throws IOException
207    */
208   static void fixupDaughter(final Result result, final byte [] qualifier,
209       final AssignmentManager assignmentManager,
210       final CatalogTracker catalogTracker)
211   throws IOException {
212     HRegionInfo daughter = getHRegionInfo(result, qualifier);
213     if (daughter == null) return;
214     if (isDaughterMissing(catalogTracker, daughter)) {
215       LOG.info("Fixup; missing daughter " + daughter.getRegionNameAsString());
216       MetaEditor.addDaughter(catalogTracker, daughter, null);
217       // And assign it.
218       assignmentManager.assign(daughter, true);
219     } else {
220       LOG.debug("Daughter " + daughter.getRegionNameAsString() + " present");
221     }
222   }
223 
224   /**
225    * Interpret the content of the cell at {@link HConstants#CATALOG_FAMILY} and
226    * <code>qualifier</code> as an HRegionInfo and return it, or null.
227    * @param r Result instance to pull from.
228    * @param qualifier Column family qualifier
229    * @return An HRegionInfo instance or null.
230    * @throws IOException
231    */
232   private static HRegionInfo getHRegionInfo(final Result r, byte [] qualifier)
233   throws IOException {
234     byte [] bytes = r.getValue(HConstants.CATALOG_FAMILY, qualifier);
235     if (bytes == null || bytes.length <= 0) return null;
236     return Writables.getHRegionInfoOrNull(bytes);
237   }
238 
239   /**
240    * Look for presence of the daughter OR of a split of the daughter. Daughter
241    * could have been split over on regionserver before a run of the
242    * catalogJanitor had chance to clear reference from parent.
243    * @param daughter Daughter region to search for.
244    * @throws IOException 
245    */
246   private static boolean isDaughterMissing(final CatalogTracker catalogTracker,
247       final HRegionInfo daughter) throws IOException {
248     FindDaughterVisitor visitor = new FindDaughterVisitor(daughter);
249     // Start the scan at what should be the daughter's row in the .META.
250     // We will either 1., find the daughter or some derivative split of the
251     // daughter (will have same table name and start row at least but will sort
252     // after because has larger regionid -- the regionid is timestamp of region
253     // creation), OR, we will not find anything with same table name and start
254     // row.  If the latter, then assume daughter missing and do fixup.
255     byte [] startrow = daughter.getRegionName();
256     MetaReader.fullScan(catalogTracker, visitor, startrow);
257     return !visitor.foundDaughter();
258   }
259 
260   /**
261    * Looks for daughter.  Sets a flag if daughter or some progeny of daughter
262    * is found up in <code>.META.</code>.
263    */
264   static class FindDaughterVisitor implements MetaReader.Visitor {
265     private final HRegionInfo daughter;
266     private boolean found = false;
267 
268     FindDaughterVisitor(final HRegionInfo daughter) {
269       this.daughter = daughter;
270     }
271 
272     /**
273      * @return True if we found a daughter region during our visiting.
274      */
275     boolean foundDaughter() {
276       return this.found;
277     }
278 
279     @Override
280     public boolean visit(Result r) throws IOException {
281       HRegionInfo hri = getHRegionInfo(r, HConstants.REGIONINFO_QUALIFIER);
282       if (hri == null) {
283         LOG.warn("No serialized HRegionInfo in " + r);
284         return true;
285       }
286       // Now see if we have gone beyond the daughter's startrow.
287       if (!Bytes.equals(daughter.getTableDesc().getName(),
288           hri.getTableDesc().getName())) {
289         // We fell into another table.  Stop scanning.
290         return false;
291       }
292       // If our start rows do not compare, move on.
293       if (!Bytes.equals(daughter.getStartKey(), hri.getStartKey())) {
294         return false;
295       }
296       // Else, table name and start rows compare.  It means that the daughter
297       // or some derivative split of the daughter is up in .META.  Daughter
298       // exists.
299       this.found = true;
300       return false;
301     }
302   }
303 }