View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.master;
19  
20  import java.lang.Thread.UncaughtExceptionHandler;
21  import java.util.ArrayList;
22  import java.util.HashSet;
23  import java.util.Iterator;
24  import java.util.List;
25  import java.util.Map;
26  import java.util.Set;
27  import java.util.concurrent.ConcurrentHashMap;
28  import java.util.concurrent.ExecutorService;
29  import java.util.concurrent.TimeUnit;
30  
31  import org.apache.commons.logging.Log;
32  import org.apache.commons.logging.LogFactory;
33  import org.apache.hadoop.classification.InterfaceAudience;
34  import org.apache.hadoop.conf.Configuration;
35  import org.apache.hadoop.hbase.HRegionInfo;
36  import org.apache.hadoop.hbase.Server;
37  import org.apache.hadoop.hbase.ServerName;
38  
39  /**
40   * Run bulk assign.  Does one RCP per regionserver passing a
41   * batch of regions using {@link GeneralBulkAssigner.SingleServerBulkAssigner}.
42   */
43  @InterfaceAudience.Private
44  public class GeneralBulkAssigner extends BulkAssigner {
45    private static final Log LOG = LogFactory.getLog(GeneralBulkAssigner.class);
46  
47    private Map<ServerName, List<HRegionInfo>> failedPlans
48      = new ConcurrentHashMap<ServerName, List<HRegionInfo>>();
49    private ExecutorService pool;
50  
51    final Map<ServerName, List<HRegionInfo>> bulkPlan;
52    final AssignmentManager assignmentManager;
53    final boolean waitTillAllAssigned;
54  
55    GeneralBulkAssigner(final Server server,
56        final Map<ServerName, List<HRegionInfo>> bulkPlan,
57        final AssignmentManager am, final boolean waitTillAllAssigned) {
58      super(server);
59      this.bulkPlan = bulkPlan;
60      this.assignmentManager = am;
61      this.waitTillAllAssigned = waitTillAllAssigned;
62    }
63  
64    @Override
65    protected String getThreadNamePrefix() {
66      return this.server.getServerName() + "-GeneralBulkAssigner";
67    }
68  
69    @Override
70    protected void populatePool(ExecutorService pool) {
71      this.pool = pool; // shut it down later in case some assigner hangs
72      for (Map.Entry<ServerName, List<HRegionInfo>> e: this.bulkPlan.entrySet()) {
73        pool.execute(new SingleServerBulkAssigner(e.getKey(), e.getValue(),
74          this.assignmentManager, this.failedPlans));
75      }
76    }
77  
78    /**
79     *
80     * @param timeout How long to wait.
81     * @return true if done.
82     */
83    @Override
84    protected boolean waitUntilDone(final long timeout)
85    throws InterruptedException {
86      Set<HRegionInfo> regionSet = new HashSet<HRegionInfo>();
87      for (List<HRegionInfo> regionList : bulkPlan.values()) {
88        regionSet.addAll(regionList);
89      }
90  
91      pool.shutdown(); // no more task allowed
92      int serverCount = bulkPlan.size();
93      int regionCount = regionSet.size();
94      long startTime = System.currentTimeMillis();
95      long rpcWaitTime = startTime + timeout;
96      while (!server.isStopped() && !pool.isTerminated()
97          && rpcWaitTime > System.currentTimeMillis()) {
98        if (failedPlans.isEmpty()) {
99          pool.awaitTermination(100, TimeUnit.MILLISECONDS);
100       } else {
101         reassignFailedPlans();
102       }
103     }
104     if (!pool.isTerminated()) {
105       LOG.warn("bulk assigner is still running after "
106         + (System.currentTimeMillis() - startTime) + "ms, shut it down now");
107       // some assigner hangs, can't wait any more, shutdown the pool now
108       List<Runnable> notStarted = pool.shutdownNow();
109       if (notStarted != null && !notStarted.isEmpty()) {
110         server.abort("some single server assigner hasn't started yet"
111           + " when the bulk assigner timed out", null);
112         return false;
113       }
114     }
115 
116     int reassigningRegions = 0;
117     if (!failedPlans.isEmpty() && !server.isStopped()) {
118       reassigningRegions = reassignFailedPlans();
119     }
120 
121     Configuration conf = server.getConfiguration();
122     long perRegionOpenTimeGuesstimate =
123       conf.getLong("hbase.bulk.assignment.perregion.open.time", 1000);
124     long endTime = Math.max(System.currentTimeMillis(), rpcWaitTime)
125       + perRegionOpenTimeGuesstimate * (reassigningRegions + 1);
126     RegionStates regionStates = assignmentManager.getRegionStates();
127     // We're not synchronizing on regionsInTransition now because we don't use any iterator.
128     while (!regionSet.isEmpty() && !server.isStopped() && endTime > System.currentTimeMillis()) {
129       Iterator<HRegionInfo> regionInfoIterator = regionSet.iterator();
130       while (regionInfoIterator.hasNext()) {
131         HRegionInfo hri = regionInfoIterator.next();
132         RegionState state = regionStates.getRegionState(hri);
133         if ((!regionStates.isRegionInTransition(hri) && regionStates.isRegionAssigned(hri))
134             || state.isSplitting() || state.isMerging()) {
135           regionInfoIterator.remove();
136         }
137       }
138       if (!waitTillAllAssigned) {
139         // No need to wait, let assignment going on asynchronously
140         break;
141       }
142       if (!regionSet.isEmpty()) {
143         regionStates.waitForUpdate(100);
144       }
145     }
146 
147     if (LOG.isDebugEnabled()) {
148       long elapsedTime = System.currentTimeMillis() - startTime;
149       String status = "successfully";
150       if (!regionSet.isEmpty()) {
151         status = "with " + regionSet.size() + " regions still in transition";
152       }
153       LOG.debug("bulk assigning total " + regionCount + " regions to "
154         + serverCount + " servers, took " + elapsedTime + "ms, " + status);
155     }
156     return regionSet.isEmpty();
157   }
158 
159   @Override
160   protected long getTimeoutOnRIT() {
161     // Guess timeout.  Multiply the max number of regions on a server
162     // by how long we think one region takes opening.
163     Configuration conf = server.getConfiguration();
164     long perRegionOpenTimeGuesstimate =
165       conf.getLong("hbase.bulk.assignment.perregion.open.time", 1000);
166     int maxRegionsPerServer = 1;
167     for (List<HRegionInfo> regionList : bulkPlan.values()) {
168       int size = regionList.size();
169       if (size > maxRegionsPerServer) {
170         maxRegionsPerServer = size;
171       }
172     }
173     long timeout = perRegionOpenTimeGuesstimate * maxRegionsPerServer
174       + conf.getLong("hbase.regionserver.rpc.startup.waittime", 60000)
175       + conf.getLong("hbase.bulk.assignment.perregionserver.rpc.waittime",
176         30000) * bulkPlan.size();
177     LOG.debug("Timeout-on-RIT=" + timeout);
178     return timeout;
179   }
180 
181   @Override
182   protected UncaughtExceptionHandler getUncaughtExceptionHandler() {
183     return new UncaughtExceptionHandler() {
184       @Override
185       public void uncaughtException(Thread t, Throwable e) {
186         LOG.warn("Assigning regions in " + t.getName(), e);
187       }
188     };
189   }
190 
191   private int reassignFailedPlans() {
192     List<HRegionInfo> reassigningRegions = new ArrayList<HRegionInfo>();
193     for (Map.Entry<ServerName, List<HRegionInfo>> e : failedPlans.entrySet()) {
194       LOG.info("Failed assigning " + e.getValue().size()
195           + " regions to server " + e.getKey() + ", reassigning them");
196       reassigningRegions.addAll(failedPlans.remove(e.getKey()));
197     }
198     for (HRegionInfo region : reassigningRegions) {
199       assignmentManager.invokeAssign(region);
200     }
201     return reassigningRegions.size();
202   }
203 
204   /**
205    * Manage bulk assigning to a server.
206    */
207   static class SingleServerBulkAssigner implements Runnable {
208     private final ServerName regionserver;
209     private final List<HRegionInfo> regions;
210     private final AssignmentManager assignmentManager;
211     private final Map<ServerName, List<HRegionInfo>> failedPlans;
212 
213     SingleServerBulkAssigner(final ServerName regionserver,
214         final List<HRegionInfo> regions, final AssignmentManager am,
215         final Map<ServerName, List<HRegionInfo>> failedPlans) {
216       this.regionserver = regionserver;
217       this.regions = regions;
218       this.assignmentManager = am;
219       this.failedPlans = failedPlans;
220     }
221 
222     @Override
223     public void run() {
224       try {
225        if (!assignmentManager.assign(regionserver, regions)) {
226          failedPlans.put(regionserver, regions);
227        }
228       } catch (Throwable t) {
229         LOG.warn("Failed bulking assigning " + regions.size()
230             + " region(s) to " + regionserver.getServerName()
231             + ", and continue to bulk assign others", t);
232         failedPlans.put(regionserver, regions);
233       }
234     }
235   }
236 }