1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.master;
19
20 import java.lang.Thread.UncaughtExceptionHandler;
21 import java.util.ArrayList;
22 import java.util.HashSet;
23 import java.util.Iterator;
24 import java.util.List;
25 import java.util.Map;
26 import java.util.Set;
27 import java.util.concurrent.ConcurrentHashMap;
28 import java.util.concurrent.ExecutorService;
29 import java.util.concurrent.TimeUnit;
30
31 import org.apache.commons.logging.Log;
32 import org.apache.commons.logging.LogFactory;
33 import org.apache.hadoop.classification.InterfaceAudience;
34 import org.apache.hadoop.conf.Configuration;
35 import org.apache.hadoop.hbase.HRegionInfo;
36 import org.apache.hadoop.hbase.Server;
37 import org.apache.hadoop.hbase.ServerName;
38
39
40
41
42
43 @InterfaceAudience.Private
44 public class GeneralBulkAssigner extends BulkAssigner {
45 private static final Log LOG = LogFactory.getLog(GeneralBulkAssigner.class);
46
47 private Map<ServerName, List<HRegionInfo>> failedPlans
48 = new ConcurrentHashMap<ServerName, List<HRegionInfo>>();
49 private ExecutorService pool;
50
51 final Map<ServerName, List<HRegionInfo>> bulkPlan;
52 final AssignmentManager assignmentManager;
53 final boolean waitTillAllAssigned;
54
55 GeneralBulkAssigner(final Server server,
56 final Map<ServerName, List<HRegionInfo>> bulkPlan,
57 final AssignmentManager am, final boolean waitTillAllAssigned) {
58 super(server);
59 this.bulkPlan = bulkPlan;
60 this.assignmentManager = am;
61 this.waitTillAllAssigned = waitTillAllAssigned;
62 }
63
64 @Override
65 protected String getThreadNamePrefix() {
66 return this.server.getServerName() + "-GeneralBulkAssigner";
67 }
68
69 @Override
70 protected void populatePool(ExecutorService pool) {
71 this.pool = pool;
72 for (Map.Entry<ServerName, List<HRegionInfo>> e: this.bulkPlan.entrySet()) {
73 pool.execute(new SingleServerBulkAssigner(e.getKey(), e.getValue(),
74 this.assignmentManager, this.failedPlans));
75 }
76 }
77
78
79
80
81
82
83 @Override
84 protected boolean waitUntilDone(final long timeout)
85 throws InterruptedException {
86 Set<HRegionInfo> regionSet = new HashSet<HRegionInfo>();
87 for (List<HRegionInfo> regionList : bulkPlan.values()) {
88 regionSet.addAll(regionList);
89 }
90
91 pool.shutdown();
92 int serverCount = bulkPlan.size();
93 int regionCount = regionSet.size();
94 long startTime = System.currentTimeMillis();
95 long rpcWaitTime = startTime + timeout;
96 while (!server.isStopped() && !pool.isTerminated()
97 && rpcWaitTime > System.currentTimeMillis()) {
98 if (failedPlans.isEmpty()) {
99 pool.awaitTermination(100, TimeUnit.MILLISECONDS);
100 } else {
101 reassignFailedPlans();
102 }
103 }
104 if (!pool.isTerminated()) {
105 LOG.warn("bulk assigner is still running after "
106 + (System.currentTimeMillis() - startTime) + "ms, shut it down now");
107
108 List<Runnable> notStarted = pool.shutdownNow();
109 if (notStarted != null && !notStarted.isEmpty()) {
110 server.abort("some single server assigner hasn't started yet"
111 + " when the bulk assigner timed out", null);
112 return false;
113 }
114 }
115
116 int reassigningRegions = 0;
117 if (!failedPlans.isEmpty() && !server.isStopped()) {
118 reassigningRegions = reassignFailedPlans();
119 }
120
121 Configuration conf = server.getConfiguration();
122 long perRegionOpenTimeGuesstimate =
123 conf.getLong("hbase.bulk.assignment.perregion.open.time", 1000);
124 long endTime = Math.max(System.currentTimeMillis(), rpcWaitTime)
125 + perRegionOpenTimeGuesstimate * (reassigningRegions + 1);
126 RegionStates regionStates = assignmentManager.getRegionStates();
127
128 while (!regionSet.isEmpty() && !server.isStopped() && endTime > System.currentTimeMillis()) {
129 Iterator<HRegionInfo> regionInfoIterator = regionSet.iterator();
130 while (regionInfoIterator.hasNext()) {
131 HRegionInfo hri = regionInfoIterator.next();
132 RegionState state = regionStates.getRegionState(hri);
133 if ((!regionStates.isRegionInTransition(hri) && regionStates.isRegionAssigned(hri))
134 || state.isSplitting() || state.isMerging()) {
135 regionInfoIterator.remove();
136 }
137 }
138 if (!waitTillAllAssigned) {
139
140 break;
141 }
142 if (!regionSet.isEmpty()) {
143 regionStates.waitForUpdate(100);
144 }
145 }
146
147 if (LOG.isDebugEnabled()) {
148 long elapsedTime = System.currentTimeMillis() - startTime;
149 String status = "successfully";
150 if (!regionSet.isEmpty()) {
151 status = "with " + regionSet.size() + " regions still in transition";
152 }
153 LOG.debug("bulk assigning total " + regionCount + " regions to "
154 + serverCount + " servers, took " + elapsedTime + "ms, " + status);
155 }
156 return regionSet.isEmpty();
157 }
158
159 @Override
160 protected long getTimeoutOnRIT() {
161
162
163 Configuration conf = server.getConfiguration();
164 long perRegionOpenTimeGuesstimate =
165 conf.getLong("hbase.bulk.assignment.perregion.open.time", 1000);
166 int maxRegionsPerServer = 1;
167 for (List<HRegionInfo> regionList : bulkPlan.values()) {
168 int size = regionList.size();
169 if (size > maxRegionsPerServer) {
170 maxRegionsPerServer = size;
171 }
172 }
173 long timeout = perRegionOpenTimeGuesstimate * maxRegionsPerServer
174 + conf.getLong("hbase.regionserver.rpc.startup.waittime", 60000)
175 + conf.getLong("hbase.bulk.assignment.perregionserver.rpc.waittime",
176 30000) * bulkPlan.size();
177 LOG.debug("Timeout-on-RIT=" + timeout);
178 return timeout;
179 }
180
181 @Override
182 protected UncaughtExceptionHandler getUncaughtExceptionHandler() {
183 return new UncaughtExceptionHandler() {
184 @Override
185 public void uncaughtException(Thread t, Throwable e) {
186 LOG.warn("Assigning regions in " + t.getName(), e);
187 }
188 };
189 }
190
191 private int reassignFailedPlans() {
192 List<HRegionInfo> reassigningRegions = new ArrayList<HRegionInfo>();
193 for (Map.Entry<ServerName, List<HRegionInfo>> e : failedPlans.entrySet()) {
194 LOG.info("Failed assigning " + e.getValue().size()
195 + " regions to server " + e.getKey() + ", reassigning them");
196 reassigningRegions.addAll(failedPlans.remove(e.getKey()));
197 }
198 for (HRegionInfo region : reassigningRegions) {
199 assignmentManager.invokeAssign(region);
200 }
201 return reassigningRegions.size();
202 }
203
204
205
206
207 static class SingleServerBulkAssigner implements Runnable {
208 private final ServerName regionserver;
209 private final List<HRegionInfo> regions;
210 private final AssignmentManager assignmentManager;
211 private final Map<ServerName, List<HRegionInfo>> failedPlans;
212
213 SingleServerBulkAssigner(final ServerName regionserver,
214 final List<HRegionInfo> regions, final AssignmentManager am,
215 final Map<ServerName, List<HRegionInfo>> failedPlans) {
216 this.regionserver = regionserver;
217 this.regions = regions;
218 this.assignmentManager = am;
219 this.failedPlans = failedPlans;
220 }
221
222 @Override
223 public void run() {
224 try {
225 if (!assignmentManager.assign(regionserver, regions)) {
226 failedPlans.put(regionserver, regions);
227 }
228 } catch (Throwable t) {
229 LOG.warn("Failed bulking assigning " + regions.size()
230 + " region(s) to " + regionserver.getServerName()
231 + ", and continue to bulk assign others", t);
232 failedPlans.put(regionserver, regions);
233 }
234 }
235 }
236 }