1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.master;
21
22 import java.io.DataInput;
23 import java.io.DataOutput;
24 import java.io.IOException;
25 import java.lang.Thread.UncaughtExceptionHandler;
26 import java.util.ArrayList;
27 import java.util.Arrays;
28 import java.util.Collections;
29 import java.util.Date;
30 import java.util.HashMap;
31 import java.util.HashSet;
32 import java.util.Iterator;
33 import java.util.List;
34 import java.util.Map;
35 import java.util.NavigableMap;
36 import java.util.Set;
37 import java.util.SortedMap;
38 import java.util.TreeMap;
39 import java.util.TreeSet;
40 import java.util.concurrent.ConcurrentSkipListMap;
41 import java.util.concurrent.ConcurrentSkipListSet;
42 import java.util.concurrent.Executors;
43 import java.util.concurrent.atomic.AtomicInteger;
44 import java.util.concurrent.atomic.AtomicLong;
45
46 import org.apache.commons.logging.Log;
47 import org.apache.commons.logging.LogFactory;
48 import org.apache.hadoop.conf.Configuration;
49 import org.apache.hadoop.hbase.Chore;
50 import org.apache.hadoop.hbase.HConstants;
51 import org.apache.hadoop.hbase.HRegionInfo;
52 import org.apache.hadoop.hbase.HServerLoad;
53 import org.apache.hadoop.hbase.NotServingRegionException;
54 import org.apache.hadoop.hbase.Server;
55 import org.apache.hadoop.hbase.ServerName;
56 import org.apache.hadoop.hbase.Stoppable;
57 import org.apache.hadoop.hbase.TableNotFoundException;
58 import org.apache.hadoop.hbase.catalog.CatalogTracker;
59 import org.apache.hadoop.hbase.catalog.MetaReader;
60 import org.apache.hadoop.hbase.catalog.RootLocationEditor;
61 import org.apache.hadoop.hbase.client.Result;
62 import org.apache.hadoop.hbase.executor.EventHandler;
63 import org.apache.hadoop.hbase.executor.EventHandler.EventType;
64 import org.apache.hadoop.hbase.executor.ExecutorService;
65 import org.apache.hadoop.hbase.executor.RegionTransitionData;
66 import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
67 import org.apache.hadoop.hbase.master.AssignmentManager.RegionState.State;
68 import org.apache.hadoop.hbase.master.handler.ClosedRegionHandler;
69 import org.apache.hadoop.hbase.master.handler.DisableTableHandler;
70 import org.apache.hadoop.hbase.master.handler.EnableTableHandler;
71 import org.apache.hadoop.hbase.master.handler.OpenedRegionHandler;
72 import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler;
73 import org.apache.hadoop.hbase.master.handler.SplitRegionHandler;
74 import org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException;
75 import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
76 import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException;
77 import org.apache.hadoop.hbase.util.Bytes;
78 import org.apache.hadoop.hbase.util.Pair;
79 import org.apache.hadoop.hbase.util.Threads;
80 import org.apache.hadoop.hbase.util.Writables;
81 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
82 import org.apache.hadoop.hbase.zookeeper.ZKTable;
83 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
84 import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
85 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
86 import org.apache.hadoop.ipc.RemoteException;
87 import org.apache.zookeeper.AsyncCallback;
88 import org.apache.zookeeper.KeeperException;
89 import org.apache.zookeeper.KeeperException.NoNodeException;
90 import org.apache.zookeeper.KeeperException.NodeExistsException;
91 import org.apache.zookeeper.data.Stat;
92
93
94
95
96
97
98
99
100 public class AssignmentManager extends ZooKeeperListener {
101
102 private static final Log LOG = LogFactory.getLog(AssignmentManager.class);
103
104 protected Server master;
105
106 private ServerManager serverManager;
107
108 private CatalogTracker catalogTracker;
109
110 private TimeoutMonitor timeoutMonitor;
111
112 private TimerUpdater timerUpdater;
113
114 private LoadBalancer balancer;
115
116
117
118
119
120 private final Map <String, HRegionInfo> regionsToReopen;
121
122
123
124
125 private final int maximumAssignmentAttempts;
126
127
128
129
130
131 final ConcurrentSkipListMap<String, RegionState> regionsInTransition =
132 new ConcurrentSkipListMap<String, RegionState>();
133
134
135
136
137
138 final NavigableMap<String, RegionPlan> regionPlans =
139 new TreeMap<String, RegionPlan>();
140
141 private final ZKTable zkTable;
142
143
144 Set<String> disablingTables = new HashSet<String>(1);
145
146
147
148 Map<String, List<HRegionInfo>> enablingTables = new HashMap<String, List<HRegionInfo>>(1);
149
150
151
152
153
154
155
156 private final NavigableMap<ServerName, Set<HRegionInfo>> servers =
157 new TreeMap<ServerName, Set<HRegionInfo>>();
158
159
160
161
162
163 private final ConcurrentSkipListSet<ServerName> serversInUpdatingTimer =
164 new ConcurrentSkipListSet<ServerName>();
165
166
167
168
169
170
171
172
173 private final SortedMap<HRegionInfo, ServerName> regions =
174 new TreeMap<HRegionInfo, ServerName>();
175
176 private final ExecutorService executorService;
177
178
179 private java.util.concurrent.ExecutorService threadPoolExecutorService;
180
181 private List<EventType> ignoreStatesRSOffline = Arrays.asList(new EventType[]{
182 EventType.RS_ZK_REGION_FAILED_OPEN, EventType.RS_ZK_REGION_CLOSED });
183
184
185
186
187
188 private volatile boolean failover = false;
189
190
191
192 private Map<String, HRegionInfo> failoverProcessedRegions =
193 new HashMap<String, HRegionInfo>();
194
195
196
197
198
199
200
201
202
203
204
205 public AssignmentManager(Server master, ServerManager serverManager,
206 CatalogTracker catalogTracker, final LoadBalancer balancer,
207 final ExecutorService service) throws KeeperException, IOException {
208 super(master.getZooKeeper());
209 this.master = master;
210 this.serverManager = serverManager;
211 this.catalogTracker = catalogTracker;
212 this.executorService = service;
213 this.regionsToReopen = Collections.synchronizedMap
214 (new HashMap<String, HRegionInfo> ());
215 Configuration conf = master.getConfiguration();
216 this.timeoutMonitor = new TimeoutMonitor(
217 conf.getInt("hbase.master.assignment.timeoutmonitor.period", 10000),
218 master, serverManager,
219 conf.getInt("hbase.master.assignment.timeoutmonitor.timeout", 1800000));
220 this.timerUpdater = new TimerUpdater(conf.getInt(
221 "hbase.master.assignment.timerupdater.period", 10000), master);
222 Threads.setDaemonThreadRunning(timerUpdater.getThread(),
223 master.getServerName() + ".timerUpdater");
224 this.zkTable = new ZKTable(this.master.getZooKeeper());
225 this.maximumAssignmentAttempts =
226 this.master.getConfiguration().getInt("hbase.assignment.maximum.attempts", 10);
227 this.balancer = balancer;
228 this.threadPoolExecutorService = Executors.newCachedThreadPool();
229 }
230
231 void startTimeOutMonitor() {
232 Threads.setDaemonThreadRunning(timeoutMonitor.getThread(), master.getServerName()
233 + ".timeoutMonitor");
234 }
235
236
237
238
239
240
241
242 double getAverageLoad() {
243 int totalLoad = 0;
244 int numServers = 0;
245
246
247 synchronized (this.regions) {
248 for (Map.Entry<ServerName, Set<HRegionInfo>> e: servers.entrySet()) {
249 numServers++;
250 totalLoad += e.getValue().size();
251 }
252 }
253 return (double)totalLoad / (double)numServers;
254 }
255
256
257
258
259 public ZKTable getZKTable() {
260
261
262 return this.zkTable;
263 }
264
265
266
267
268
269
270
271 public ServerName getRegionServerOfRegion(HRegionInfo hri) {
272 synchronized (this.regions ) {
273 return regions.get(hri);
274 }
275 }
276
277
278
279
280
281
282 public boolean isRegionAssigned(HRegionInfo hri) {
283 synchronized (this.regions ) {
284 return regions.containsKey(hri);
285 }
286 }
287
288
289
290
291
292
293
294 public List<HRegionInfo> getEnablingTableRegions(String tableName){
295 return this.enablingTables.get(tableName);
296 }
297
298
299
300
301
302
303 public void addPlan(String encodedName, RegionPlan plan) {
304 synchronized (regionPlans) {
305 regionPlans.put(encodedName, plan);
306 }
307 }
308
309
310
311
312 public void addPlans(Map<String, RegionPlan> plans) {
313 synchronized (regionPlans) {
314 regionPlans.putAll(plans);
315 }
316 }
317
318
319
320
321
322
323
324
325 public void setRegionsToReopen(List <HRegionInfo> regions) {
326 for(HRegionInfo hri : regions) {
327 regionsToReopen.put(hri.getEncodedName(), hri);
328 }
329 }
330
331
332
333
334
335
336
337
338 public Pair<Integer, Integer> getReopenStatus(byte[] tableName)
339 throws IOException {
340 List <HRegionInfo> hris =
341 MetaReader.getTableRegions(this.master.getCatalogTracker(), tableName);
342 Integer pending = 0;
343 for(HRegionInfo hri : hris) {
344 String name = hri.getEncodedName();
345 if (regionsToReopen.containsKey(name) || regionsInTransition.containsKey(name)) {
346 pending++;
347 }
348 }
349 return new Pair<Integer, Integer>(pending, hris.size());
350 }
351
352
353
354
355
356
357 void cleanoutUnassigned() throws IOException, KeeperException {
358
359 ZKAssign.deleteAllNodes(watcher);
360 ZKUtil.listChildrenAndWatchForNewChildren(this.watcher,
361 this.watcher.assignmentZNode);
362 }
363
364
365
366
367
368
369
370
371 void joinCluster() throws IOException,
372 KeeperException, InterruptedException {
373
374
375
376
377
378
379
380
381
382
383 Map<ServerName, List<Pair<HRegionInfo, Result>>> deadServers = rebuildUserRegions();
384
385 processDeadServersAndRegionsInTransition(deadServers);
386
387
388
389 boolean isWatcherCreated = recoverTableInDisablingState(this.disablingTables);
390 recoverTableInEnablingState(this.enablingTables.keySet(), isWatcherCreated);
391 this.enablingTables.clear();
392 this.disablingTables.clear();
393 }
394
395
396
397
398
399
400
401
402 void processDeadServersAndRegionsInTransition()
403 throws KeeperException, IOException, InterruptedException {
404
405 processDeadServersAndRegionsInTransition(null);
406 }
407
408
409
410
411
412
413
414
415
416
417
418 void processDeadServersAndRegionsInTransition(
419 final Map<ServerName, List<Pair<HRegionInfo, Result>>> deadServers)
420 throws KeeperException, IOException, InterruptedException {
421 List<String> nodes = ZKUtil.listChildrenAndWatchForNewChildren(watcher,
422 watcher.assignmentZNode);
423
424 if (nodes == null) {
425 String errorMessage = "Failed to get the children from ZK";
426 master.abort(errorMessage, new IOException(errorMessage));
427 return;
428 }
429
430
431 synchronized (this.regions) {
432 for (Map.Entry<HRegionInfo, ServerName> e : this.regions.entrySet()) {
433 if (!e.getKey().isMetaTable() && e.getValue() != null) {
434 LOG.debug("Found " + e + " out on cluster");
435 this.failover = true;
436 break;
437 }
438 if (nodes.contains(e.getKey().getEncodedName())) {
439 LOG.debug("Found " + e.getKey().getRegionNameAsString() + " in RITs");
440
441 this.failover = true;
442 break;
443 }
444 }
445 }
446
447
448
449 synchronized (regionsInTransition) {
450 nodes.removeAll(regionsInTransition.keySet());
451 }
452
453
454
455
456 if (!this.serverManager.getDeadServers().isEmpty()) {
457 this.failover = true;
458 }
459
460
461 if (this.failover) {
462 LOG.info("Found regions out on cluster or in RIT; failover");
463
464
465 processDeadServersAndRecoverLostRegions(deadServers, nodes);
466 this.failover = false;
467 failoverProcessedRegions.clear();
468 } else {
469
470 LOG.info("Clean cluster startup. Assigning userregions");
471 cleanoutUnassigned();
472 assignAllUserRegions();
473 }
474 }
475
476
477
478
479
480
481
482
483
484
485
486
487 boolean processRegionInTransitionAndBlockUntilAssigned(final HRegionInfo hri)
488 throws InterruptedException, KeeperException, IOException {
489 boolean intransistion =
490 processRegionInTransition(hri.getEncodedName(), hri, null);
491 if (!intransistion) return intransistion;
492 LOG.debug("Waiting on " + HRegionInfo.prettyPrint(hri.getEncodedName()));
493 synchronized(this.regionsInTransition) {
494 while (!this.master.isStopped() &&
495 this.regionsInTransition.containsKey(hri.getEncodedName())) {
496
497 this.regionsInTransition.wait(100);
498 }
499 }
500 return intransistion;
501 }
502
503
504
505
506
507
508
509
510
511
512
513 boolean processRegionInTransition(final String encodedRegionName,
514 final HRegionInfo regionInfo,
515 final Map<ServerName,List<Pair<HRegionInfo,Result>>> deadServers)
516 throws KeeperException, IOException {
517 Stat stat = new Stat();
518 RegionTransitionData data = ZKAssign.getDataAndWatch(watcher,
519 encodedRegionName, stat);
520 if (data == null) return false;
521 HRegionInfo hri = regionInfo;
522 if (hri == null) {
523 if ((hri = getHRegionInfo(data)) == null) return false;
524 }
525 processRegionsInTransition(data, hri, deadServers, stat.getVersion());
526 return true;
527 }
528
529 void processRegionsInTransition(final RegionTransitionData data,
530 final HRegionInfo regionInfo,
531 final Map<ServerName, List<Pair<HRegionInfo, Result>>> deadServers,
532 int expectedVersion)
533 throws KeeperException {
534 String encodedRegionName = regionInfo.getEncodedName();
535 LOG.info("Processing region " + regionInfo.getRegionNameAsString() +
536 " in state " + data.getEventType());
537 List<HRegionInfo> hris = this.enablingTables.get(regionInfo.getTableNameAsString());
538 if (hris != null && !hris.isEmpty()) {
539 hris.remove(regionInfo);
540 }
541 synchronized (regionsInTransition) {
542 RegionState regionState = regionsInTransition.get(encodedRegionName);
543 if (regionState != null ||
544 failoverProcessedRegions.containsKey(encodedRegionName)) {
545
546 return;
547 }
548 switch (data.getEventType()) {
549 case M_ZK_REGION_CLOSING:
550
551
552 if (isOnDeadServer(regionInfo, deadServers) &&
553 (data.getOrigin() == null || !serverManager.isServerOnline(data.getOrigin()))) {
554
555
556 forceOffline(regionInfo, data);
557 } else {
558
559
560 regionsInTransition.put(encodedRegionName, new RegionState(
561 regionInfo, RegionState.State.CLOSING,
562 data.getStamp(), data.getOrigin()));
563 }
564 failoverProcessedRegions.put(encodedRegionName, regionInfo);
565 break;
566
567 case RS_ZK_REGION_CLOSED:
568 case RS_ZK_REGION_FAILED_OPEN:
569
570 addToRITandCallClose(regionInfo, RegionState.State.CLOSED, data);
571 failoverProcessedRegions.put(encodedRegionName, regionInfo);
572 break;
573
574 case M_ZK_REGION_OFFLINE:
575
576
577 if (isOnDeadServer(regionInfo, deadServers) &&
578 (data.getOrigin() == null ||
579 !serverManager.isServerOnline(data.getOrigin()))) {
580
581 addToRITandCallClose(regionInfo, RegionState.State.OFFLINE, data);
582 } else if (data.getOrigin() != null &&
583 !serverManager.isServerOnline(data.getOrigin())) {
584
585
586 addToRITandCallClose(regionInfo, RegionState.State.OFFLINE, data);
587 } else {
588 regionsInTransition.put(encodedRegionName, new RegionState(
589 regionInfo, RegionState.State.PENDING_OPEN, data.getStamp(), data
590 .getOrigin()));
591 }
592 failoverProcessedRegions.put(encodedRegionName, regionInfo);
593 break;
594
595 case RS_ZK_REGION_OPENING:
596
597
598
599
600
601 if (regionInfo.isMetaTable()) {
602 regionsInTransition.put(encodedRegionName, new RegionState(
603 regionInfo, RegionState.State.OPENING, data.getStamp(), data
604 .getOrigin()));
605
606
607
608
609
610 processOpeningState(regionInfo);
611 break;
612 }
613 regionsInTransition.put(encodedRegionName, new RegionState(regionInfo,
614 RegionState.State.OPENING, data.getStamp(), data.getOrigin()));
615 failoverProcessedRegions.put(encodedRegionName, regionInfo);
616 break;
617
618 case RS_ZK_REGION_OPENED:
619
620 regionsInTransition.put(encodedRegionName, new RegionState(
621 regionInfo, RegionState.State.OPEN,
622 data.getStamp(), data.getOrigin()));
623 ServerName sn = data.getOrigin() == null? null: data.getOrigin();
624
625
626
627 if (sn == null) {
628 LOG.warn("Region in transition " + regionInfo.getEncodedName() +
629 " references a null server; letting RIT timeout so will be " +
630 "assigned elsewhere");
631 } else if (!serverManager.isServerOnline(sn)
632 && (isOnDeadServer(regionInfo, deadServers)
633 || regionInfo.isMetaRegion() || regionInfo.isRootRegion())) {
634 forceOffline(regionInfo, data);
635 } else {
636 new OpenedRegionHandler(master, this, regionInfo, sn, expectedVersion)
637 .process();
638 }
639 failoverProcessedRegions.put(encodedRegionName, regionInfo);
640 break;
641 }
642 }
643 }
644
645
646
647
648
649
650
651
652 private void forceOffline(final HRegionInfo hri,
653 final RegionTransitionData oldData)
654 throws KeeperException {
655
656
657 LOG.debug("RIT " + hri.getEncodedName() + " in state=" +
658 oldData.getEventType() + " was on deadserver; forcing offline");
659 ZKAssign.createOrForceNodeOffline(this.watcher, hri,
660 this.master.getServerName());
661 addToRITandCallClose(hri, RegionState.State.OFFLINE, oldData);
662 }
663
664
665
666
667
668
669
670
671 private void addToRITandCallClose(final HRegionInfo hri,
672 final RegionState.State state, final RegionTransitionData oldData) {
673 this.regionsInTransition.put(hri.getEncodedName(),
674 new RegionState(hri, state, oldData.getStamp(), oldData.getOrigin()));
675 new ClosedRegionHandler(this.master, this, hri).process();
676 }
677
678
679
680
681
682 public void removeClosedRegion(HRegionInfo hri) {
683 if (!regionsToReopen.isEmpty()) {
684 if (regionsToReopen.remove(hri.getEncodedName()) != null) {
685 LOG.debug("Removed region from reopening regions because it was closed");
686 }
687 }
688 }
689
690
691
692
693
694
695
696 private boolean isOnDeadServer(final HRegionInfo regionInfo,
697 final Map<ServerName, List<Pair<HRegionInfo, Result>>> deadServers) {
698 if (deadServers == null) return false;
699 for (Map.Entry<ServerName, List<Pair<HRegionInfo, Result>>> deadServer:
700 deadServers.entrySet()) {
701 for (Pair<HRegionInfo, Result> e: deadServer.getValue()) {
702 if (e.getFirst().equals(regionInfo)) return true;
703 }
704 }
705 return false;
706 }
707
708
709
710
711
712
713
714
715
716
717
718 private void handleRegion(final RegionTransitionData data, int expectedVersion) {
719 synchronized(regionsInTransition) {
720 HRegionInfo hri = null;
721 if (data == null || data.getOrigin() == null) {
722 LOG.warn("Unexpected NULL input " + data);
723 return;
724 }
725 ServerName sn = data.getOrigin();
726
727 if (sn.equals(HConstants.HBCK_CODE_SERVERNAME)) {
728 handleHBCK(data);
729 return;
730 }
731 String encodedName = HRegionInfo.encodeRegionName(data.getRegionName());
732 String prettyPrintedRegionName = HRegionInfo.prettyPrint(encodedName);
733
734 if (!serverManager.isServerOnline(sn) &&
735 !this.master.getServerName().equals(sn)
736 && !ignoreStatesRSOffline.contains(data.getEventType())) {
737 LOG.warn("Attempted to handle region transition for server but " +
738 "server is not online: " + prettyPrintedRegionName);
739 return;
740 }
741
742 boolean lateEvent = data.getStamp() <
743 (System.currentTimeMillis() - 15000);
744 LOG.debug("Handling transition=" + data.getEventType() +
745 ", server=" + data.getOrigin() + ", region=" +
746 (prettyPrintedRegionName == null? "null": prettyPrintedRegionName) +
747 (lateEvent? ", which is more than 15 seconds late" : ""));
748 RegionState regionState = regionsInTransition.get(encodedName);
749 switch (data.getEventType()) {
750 case M_ZK_REGION_OFFLINE:
751
752 break;
753
754 case RS_ZK_REGION_SPLITTING:
755 if (!isInStateForSplitting(regionState)) break;
756 addSplittingToRIT(sn, encodedName);
757 break;
758
759 case RS_ZK_REGION_SPLIT:
760
761 if (!isInStateForSplitting(regionState)) break;
762
763 if (regionState == null) {
764 regionState = addSplittingToRIT(sn, encodedName);
765 String message = "Received SPLIT for region " + prettyPrintedRegionName +
766 " from server " + sn;
767
768 if (regionState == null) {
769 LOG.warn(message + " but it doesn't exist anymore," +
770 " probably already processed its split");
771 break;
772 }
773 LOG.info(message +
774 " but region was not first in SPLITTING state; continuing");
775 }
776
777 byte [] payload = data.getPayload();
778 List<HRegionInfo> daughters = null;
779 try {
780 daughters = Writables.getHRegionInfos(payload, 0, payload.length);
781 } catch (IOException e) {
782 LOG.error("Dropped split! Failed reading split payload for " +
783 prettyPrintedRegionName);
784 break;
785 }
786 assert daughters.size() == 2;
787
788 if (!this.serverManager.isServerOnline(sn)) {
789 LOG.error("Dropped split! ServerName=" + sn + " unknown.");
790 break;
791 }
792
793 this.executorService.submit(new SplitRegionHandler(master, this,
794 regionState.getRegion(), sn, daughters));
795 break;
796
797 case M_ZK_REGION_CLOSING:
798 hri = checkIfInFailover(regionState, encodedName, data);
799 if (hri != null) {
800 regionState = new RegionState(hri, RegionState.State.CLOSING, data
801 .getStamp(), data.getOrigin());
802 regionsInTransition.put(encodedName, regionState);
803 failoverProcessedRegions.put(encodedName, hri);
804 break;
805 }
806
807
808 if (regionState == null ||
809 (!regionState.isPendingClose() && !regionState.isClosing())) {
810 LOG.warn("Received CLOSING for region " + prettyPrintedRegionName +
811 " from server " + data.getOrigin() + " but region was in " +
812 " the state " + regionState + " and not " +
813 "in expected PENDING_CLOSE or CLOSING states");
814 return;
815 }
816
817 regionState.update(RegionState.State.CLOSING,
818 data.getStamp(), data.getOrigin());
819 break;
820
821 case RS_ZK_REGION_CLOSED:
822 hri = checkIfInFailover(regionState, encodedName, data);
823 if (hri != null) {
824 regionState = new RegionState(hri, RegionState.State.CLOSED, data
825 .getStamp(), data.getOrigin());
826 regionsInTransition.put(encodedName, regionState);
827 removeClosedRegion(regionState.getRegion());
828 new ClosedRegionHandler(master, this, regionState.getRegion())
829 .process();
830 failoverProcessedRegions.put(encodedName, hri);
831 break;
832 }
833
834 if (regionState == null ||
835 (!regionState.isPendingClose() && !regionState.isClosing())) {
836 LOG.warn("Received CLOSED for region " + prettyPrintedRegionName +
837 " from server " + data.getOrigin() + " but region was in " +
838 " the state " + regionState + " and not " +
839 "in expected PENDING_CLOSE or CLOSING states");
840 return;
841 }
842
843
844
845 regionState.update(RegionState.State.CLOSED,
846 data.getStamp(), data.getOrigin());
847 removeClosedRegion(regionState.getRegion());
848 this.executorService.submit(new ClosedRegionHandler(master,
849 this, regionState.getRegion()));
850 break;
851
852 case RS_ZK_REGION_FAILED_OPEN:
853 hri = checkIfInFailover(regionState, encodedName, data);
854 if (hri != null) {
855 regionState = new RegionState(hri, RegionState.State.CLOSED, data
856 .getStamp(), data.getOrigin());
857 regionsInTransition.put(encodedName, regionState);
858 new ClosedRegionHandler(master, this, regionState.getRegion())
859 .process();
860 failoverProcessedRegions.put(encodedName, hri);
861 break;
862 }
863 if (regionState == null ||
864 (!regionState.isOffline() && !regionState.isPendingOpen() && !regionState.isOpening())) {
865 LOG.warn("Received FAILED_OPEN for region " + prettyPrintedRegionName +
866 " from server " + data.getOrigin() + " but region was in " +
867 " the state " + regionState + " and not in OFFLINE, PENDING_OPEN or OPENING");
868 return;
869 }
870
871 regionState.update(RegionState.State.CLOSED,
872 data.getStamp(), data.getOrigin());
873
874
875 getRegionPlan(regionState, sn, true);
876 this.executorService.submit(new ClosedRegionHandler(master,
877 this, regionState.getRegion()));
878 break;
879
880 case RS_ZK_REGION_OPENING:
881 hri = checkIfInFailover(regionState, encodedName, data);
882 if (hri != null) {
883 regionState = new RegionState(hri, RegionState.State.OPENING, data
884 .getStamp(), data.getOrigin());
885 regionsInTransition.put(encodedName, regionState);
886 failoverProcessedRegions.put(encodedName, hri);
887 break;
888 }
889 if (regionState == null ||
890 (!regionState.isOffline() && !regionState.isPendingOpen() &&
891 !regionState.isOpening())) {
892 LOG.warn("Received OPENING for region " + prettyPrintedRegionName + " from server " +
893 sn + " but region was in " + " the state " + regionState + " and not " +
894 "in expected OFFLINE, PENDING_OPEN or OPENING states");
895 return;
896 }
897
898 regionState.update(RegionState.State.OPENING,
899 data.getStamp(), data.getOrigin());
900 break;
901
902 case RS_ZK_REGION_OPENED:
903 hri = checkIfInFailover(regionState, encodedName, data);
904 if (hri != null) {
905 regionState = new RegionState(hri, RegionState.State.OPEN, data
906 .getStamp(), data.getOrigin());
907 regionsInTransition.put(encodedName, regionState);
908 new OpenedRegionHandler(master, this, regionState.getRegion(), data
909 .getOrigin(), expectedVersion).process();
910 failoverProcessedRegions.put(encodedName, hri);
911 break;
912 }
913
914 if (regionState == null ||
915 (!regionState.isOffline() && !regionState.isPendingOpen() && !regionState.isOpening())) {
916 LOG.warn("Received OPENED for region " +
917 prettyPrintedRegionName +
918 " from server " + data.getOrigin() + " but region was in " +
919 " the state " + regionState + " and not " +
920 "in expected OFFLINE, PENDING_OPEN or OPENING states");
921 return;
922 }
923
924 regionState.update(RegionState.State.OPEN,
925 data.getStamp(), data.getOrigin());
926 this.executorService.submit(
927 new OpenedRegionHandler(master, this, regionState.getRegion(),
928 data.getOrigin(), expectedVersion));
929 break;
930 }
931 }
932 }
933
934
935
936
937
938
939
940
941
942 private HRegionInfo checkIfInFailover(RegionState regionState,
943 String encodedName, RegionTransitionData data) {
944 if (regionState == null && this.failover &&
945 (failoverProcessedRegions.containsKey(encodedName) == false ||
946 failoverProcessedRegions.get(encodedName) == null)) {
947 HRegionInfo hri = this.failoverProcessedRegions.get(encodedName);
948 if (hri == null) hri = getHRegionInfo(data);
949 return hri;
950 }
951 return null;
952 }
953
954
955
956
957
958
959 private HRegionInfo getHRegionInfo(RegionTransitionData data) {
960 Pair<HRegionInfo, ServerName> p = null;
961 try {
962 p = MetaReader.getRegion(catalogTracker, data.getRegionName());
963 if (p == null) return null;
964 return p.getFirst();
965 } catch (IOException e) {
966 master.abort("Aborting because error occoured while reading "
967 + data.getRegionName() + " from .META.", e);
968 return null;
969 }
970 }
971
972
973
974
975
976
977 private boolean isInStateForSplitting(final RegionState rs) {
978 if (rs == null) return true;
979 if (rs.isSplitting()) return true;
980 if (convertPendingCloseToSplitting(rs)) return true;
981 LOG.warn("Dropped region split! Not in state good for SPLITTING; rs=" + rs);
982 return false;
983 }
984
985
986
987
988
989
990
991
992
993
994 private boolean convertPendingCloseToSplitting(final RegionState rs) {
995 if (!rs.isPendingClose()) return false;
996 LOG.debug("Converting PENDING_CLOSE to SPLITING; rs=" + rs);
997 rs.update(RegionState.State.SPLITTING);
998
999
1000 clearRegionPlan(rs.getRegion());
1001 return true;
1002 }
1003
1004
1005
1006
1007
1008
1009
1010 private RegionState addSplittingToRIT(final ServerName serverName,
1011 final String encodedName) {
1012 RegionState regionState = null;
1013 synchronized (this.regions) {
1014 regionState = findHRegionInfoThenAddToRIT(serverName, encodedName);
1015 if (regionState != null) {
1016 regionState.update(RegionState.State.SPLITTING,
1017 System.currentTimeMillis(), serverName);
1018 }
1019 }
1020 return regionState;
1021 }
1022
1023
1024
1025
1026
1027
1028
1029 private RegionState findHRegionInfoThenAddToRIT(final ServerName serverName,
1030 final String encodedName) {
1031 HRegionInfo hri = findHRegionInfo(serverName, encodedName);
1032 if (hri == null) {
1033 LOG.warn("Region " + encodedName + " not found on server " + serverName +
1034 "; failed processing");
1035 return null;
1036 }
1037
1038 return addToRegionsInTransition(hri);
1039 }
1040
1041
1042
1043
1044
1045
1046
1047 private HRegionInfo findHRegionInfo(final ServerName sn,
1048 final String encodedName) {
1049 if (!this.serverManager.isServerOnline(sn)) return null;
1050 Set<HRegionInfo> hris = this.servers.get(sn);
1051 HRegionInfo foundHri = null;
1052 for (HRegionInfo hri: hris) {
1053 if (hri.getEncodedName().equals(encodedName)) {
1054 foundHri = hri;
1055 break;
1056 }
1057 }
1058 return foundHri;
1059 }
1060
1061
1062
1063
1064
1065
1066
1067 private void handleHBCK(RegionTransitionData data) {
1068 String encodedName = HRegionInfo.encodeRegionName(data.getRegionName());
1069 LOG.info("Handling HBCK triggered transition=" + data.getEventType() +
1070 ", server=" + data.getOrigin() + ", region=" +
1071 HRegionInfo.prettyPrint(encodedName));
1072 RegionState regionState = regionsInTransition.get(encodedName);
1073 switch (data.getEventType()) {
1074 case M_ZK_REGION_OFFLINE:
1075 HRegionInfo regionInfo = null;
1076 if (regionState != null) {
1077 regionInfo = regionState.getRegion();
1078 } else {
1079 try {
1080 byte[] name = data.getRegionName();
1081 Pair<HRegionInfo, ServerName> p = MetaReader.getRegion(catalogTracker, name);
1082 regionInfo = p.getFirst();
1083 } catch (IOException e) {
1084 LOG.info("Exception reading META doing HBCK repair operation", e);
1085 return;
1086 }
1087 }
1088 LOG.info("HBCK repair is triggering assignment of region=" +
1089 regionInfo.getRegionNameAsString());
1090
1091 assign(regionInfo, false);
1092 break;
1093
1094 default:
1095 LOG.warn("Received unexpected region state from HBCK (" +
1096 data.getEventType() + ")");
1097 break;
1098 }
1099 }
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115 @Override
1116 public void nodeCreated(String path) {
1117 if(path.startsWith(watcher.assignmentZNode)) {
1118 try {
1119 Stat stat = new Stat();
1120 RegionTransitionData data = ZKAssign.getDataAndWatch(watcher, path, stat);
1121 if (data == null) {
1122 return;
1123 }
1124 handleRegion(data, stat.getVersion());
1125 } catch (KeeperException e) {
1126 master.abort("Unexpected ZK exception reading unassigned node data", e);
1127 }
1128 }
1129 }
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143 @Override
1144 public void nodeDataChanged(String path) {
1145 if(path.startsWith(watcher.assignmentZNode)) {
1146 try {
1147 Stat stat = new Stat();
1148 RegionTransitionData data = ZKAssign.getDataAndWatch(watcher, path, stat);
1149 if (data == null) {
1150 return;
1151 }
1152 handleRegion(data, stat.getVersion());
1153 } catch (KeeperException e) {
1154 master.abort("Unexpected ZK exception reading unassigned node data", e);
1155 }
1156 }
1157 }
1158
1159 @Override
1160 public void nodeDeleted(final String path) {
1161 if (path.startsWith(this.watcher.assignmentZNode)) {
1162 String regionName = ZKAssign.getRegionName(this.master.getZooKeeper(), path);
1163 RegionState rs = this.regionsInTransition.get(regionName);
1164 if (rs != null) {
1165 HRegionInfo regionInfo = rs.getRegion();
1166 if (rs.isSplit()) {
1167 LOG.debug("Ephemeral node deleted, regionserver crashed?, offlining the region"
1168 + rs.getRegion() + " clearing from RIT;");
1169 regionOffline(rs.getRegion());
1170 } else if (rs.isSplitting()) {
1171 LOG.debug("Ephemeral node deleted. Found in SPLITTING state. " + "Removing from RIT "
1172 + rs.getRegion());
1173 synchronized(this.regionsInTransition) {
1174 this.regionsInTransition.remove(regionName);
1175 }
1176 } else {
1177 LOG.debug("The znode of region " + regionInfo.getRegionNameAsString()
1178 + " has been deleted.");
1179 if (rs.isOpened()) {
1180 makeRegionOnline(rs, regionInfo);
1181 }
1182 }
1183 }
1184 }
1185 }
1186
1187 private void makeRegionOnline(RegionState rs, HRegionInfo regionInfo) {
1188 regionOnline(regionInfo, rs.serverName);
1189 LOG.info("The master has opened the region "
1190 + regionInfo.getRegionNameAsString() + " that was online on "
1191 + rs.serverName);
1192 if (this.getZKTable().isDisablingOrDisabledTable(
1193 regionInfo.getTableNameAsString())) {
1194 LOG.debug("Opened region "
1195 + regionInfo.getRegionNameAsString() + " but "
1196 + "this table is disabled, triggering close of region");
1197 unassign(regionInfo);
1198 }
1199 }
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213 @Override
1214 public void nodeChildrenChanged(String path) {
1215 if(path.equals(watcher.assignmentZNode)) {
1216 try {
1217 List<String> children = ZKUtil.listChildrenAndWatchForNewChildren(watcher,
1218 watcher.assignmentZNode);
1219 if (children != null) {
1220 Stat stat = new Stat();
1221 for (String child : children) {
1222 stat.setVersion(0);
1223 RegionTransitionData data = ZKAssign.getDataAndWatch(watcher,
1224 ZKUtil.joinZNode(watcher.assignmentZNode, child), stat);
1225
1226 if (stat.getVersion() > 0 && data.getEventType() == EventType.RS_ZK_REGION_SPLITTING) {
1227 handleRegion(data, stat.getVersion());
1228 }
1229 }
1230 }
1231 } catch(KeeperException e) {
1232 master.abort("Unexpected ZK exception reading unassigned children", e);
1233 }
1234 }
1235 }
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245 void regionOnline(HRegionInfo regionInfo, ServerName sn) {
1246 synchronized (this.regionsInTransition) {
1247 RegionState rs =
1248 this.regionsInTransition.remove(regionInfo.getEncodedName());
1249 if (rs != null) {
1250 this.regionsInTransition.notifyAll();
1251 }
1252 }
1253 synchronized (this.regions) {
1254
1255 ServerName oldSn = this.regions.get(regionInfo);
1256 if (oldSn != null) LOG.warn("Overwriting " + regionInfo.getEncodedName() +
1257 " on " + oldSn + " with " + sn);
1258
1259 if (isServerOnline(sn)) {
1260 this.regions.put(regionInfo, sn);
1261 addToServers(sn, regionInfo);
1262 this.regions.notifyAll();
1263 } else {
1264 LOG.info("The server is not in online servers, ServerName=" +
1265 sn.getServerName() + ", region=" + regionInfo.getEncodedName());
1266 }
1267 }
1268
1269 clearRegionPlan(regionInfo);
1270
1271 addToServersInUpdatingTimer(sn);
1272 }
1273
1274
1275
1276
1277
1278
1279 private void addToServersInUpdatingTimer(final ServerName sn) {
1280 this.serversInUpdatingTimer.add(sn);
1281 }
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296 private void updateTimers(final ServerName sn) {
1297
1298
1299
1300
1301 Map<String, RegionPlan> copy = new HashMap<String, RegionPlan>();
1302 synchronized(this.regionPlans) {
1303 copy.putAll(this.regionPlans);
1304 }
1305 for (Map.Entry<String, RegionPlan> e: copy.entrySet()) {
1306 if (e.getValue() == null || e.getValue().getDestination() == null) continue;
1307 if (!e.getValue().getDestination().equals(sn)) continue;
1308 RegionState rs = null;
1309 synchronized (this.regionsInTransition) {
1310 rs = this.regionsInTransition.get(e.getKey());
1311 }
1312 if (rs == null) continue;
1313 rs.updateTimestampToNow();
1314 }
1315 }
1316
1317
1318
1319
1320
1321
1322
1323
1324 public void regionOffline(final HRegionInfo regionInfo) {
1325
1326 clearRegionPlan(regionInfo);
1327 setOffline(regionInfo);
1328
1329 synchronized(this.regionsInTransition) {
1330 if (this.regionsInTransition.remove(regionInfo.getEncodedName()) != null) {
1331 this.regionsInTransition.notifyAll();
1332 }
1333 }
1334 }
1335
1336
1337
1338
1339
1340
1341
1342
1343 public void setOffline(HRegionInfo regionInfo) {
1344 synchronized (this.regions) {
1345 ServerName sn = this.regions.remove(regionInfo);
1346 if (sn == null) return;
1347 Set<HRegionInfo> serverRegions = this.servers.get(sn);
1348 if (!serverRegions.remove(regionInfo)) {
1349 LOG.warn("No " + regionInfo + " on " + sn);
1350 }
1351 }
1352 }
1353
1354 public void offlineDisabledRegion(HRegionInfo regionInfo) {
1355
1356 LOG.debug("Table being disabled so deleting ZK node and removing from " +
1357 "regions in transition, skipping assignment of region " +
1358 regionInfo.getRegionNameAsString());
1359 try {
1360 if (!ZKAssign.deleteClosedNode(watcher, regionInfo.getEncodedName())) {
1361
1362 ZKAssign.deleteOfflineNode(watcher, regionInfo.getEncodedName());
1363 }
1364 } catch (KeeperException.NoNodeException nne) {
1365 LOG.debug("Tried to delete closed node for " + regionInfo + " but it " +
1366 "does not exist so just offlining");
1367 } catch (KeeperException e) {
1368 this.master.abort("Error deleting CLOSED node in ZK", e);
1369 }
1370 regionOffline(regionInfo);
1371 }
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393 public void assign(HRegionInfo region, boolean setOfflineInZK) {
1394 assign(region, setOfflineInZK, false);
1395 }
1396
1397 public void assign(HRegionInfo region, boolean setOfflineInZK,
1398 boolean forceNewPlan) {
1399 assign(region, setOfflineInZK, forceNewPlan, false);
1400 }
1401
1402
1403
1404
1405
1406
1407
1408
1409 public void assign(HRegionInfo region, boolean setOfflineInZK,
1410 boolean forceNewPlan, boolean hijack) {
1411
1412
1413 if (!hijack && isDisabledorDisablingRegionInRIT(region)) {
1414 return;
1415 }
1416 if (this.serverManager.isClusterShutdown()) {
1417 LOG.info("Cluster shutdown is set; skipping assign of " +
1418 region.getRegionNameAsString());
1419 return;
1420 }
1421 RegionState state = addToRegionsInTransition(region,
1422 hijack);
1423 synchronized (state) {
1424 assign(region, state, setOfflineInZK, forceNewPlan, hijack);
1425 }
1426 }
1427
1428
1429
1430
1431
1432
1433 void assign(final ServerName destination,
1434 final List<HRegionInfo> regions) {
1435 if (regions.size() == 0) {
1436 return;
1437 }
1438 LOG.debug("Bulk assigning " + regions.size() + " region(s) to " +
1439 destination.toString());
1440
1441 List<RegionState> states = new ArrayList<RegionState>(regions.size());
1442 synchronized (this.regionsInTransition) {
1443 for (HRegionInfo region: regions) {
1444 states.add(forceRegionStateToOffline(region));
1445 }
1446 }
1447
1448
1449 Map<String, RegionPlan> plans=new HashMap<String, RegionPlan>();
1450 for (HRegionInfo region : regions) {
1451 plans.put(region.getEncodedName(), new RegionPlan(region, null,
1452 destination));
1453 }
1454 this.addPlans(plans);
1455
1456
1457
1458 AtomicInteger counter = new AtomicInteger(0);
1459 CreateUnassignedAsyncCallback cb =
1460 new CreateUnassignedAsyncCallback(this.watcher, destination, counter);
1461 for (RegionState state: states) {
1462 if (!asyncSetOfflineInZooKeeper(state, cb, state)) {
1463 return;
1464 }
1465 }
1466
1467 int total = regions.size();
1468 for (int oldCounter = 0; true;) {
1469 int count = counter.get();
1470 if (oldCounter != count) {
1471 LOG.info(destination.toString() + " unassigned znodes=" + count +
1472 " of total=" + total);
1473 oldCounter = count;
1474 }
1475 if (count == total) break;
1476 Threads.sleep(1);
1477 }
1478
1479 try {
1480
1481
1482 long maxWaitTime = System.currentTimeMillis() +
1483 this.master.getConfiguration().
1484 getLong("hbase.regionserver.rpc.startup.waittime", 60000);
1485 while (!this.master.isStopped()) {
1486 try {
1487 this.serverManager.sendRegionOpen(destination, regions);
1488 break;
1489 } catch (RemoteException e) {
1490 IOException decodedException = e.unwrapRemoteException();
1491 if (decodedException instanceof RegionServerStoppedException) {
1492 LOG.warn("The region server was shut down, ", decodedException);
1493
1494 return;
1495 } else if (decodedException instanceof ServerNotRunningYetException) {
1496
1497
1498 long now = System.currentTimeMillis();
1499 if (now > maxWaitTime) throw e;
1500 LOG.debug("Server is not yet up; waiting up to " +
1501 (maxWaitTime - now) + "ms", e);
1502 Thread.sleep(1000);
1503 }
1504
1505 throw decodedException;
1506 }
1507 }
1508 } catch (IOException e) {
1509
1510 LOG.info("Unable to communicate with the region server in order" +
1511 " to assign regions", e);
1512 } catch (InterruptedException e) {
1513 throw new RuntimeException(e);
1514 }
1515 LOG.debug("Bulk assigning done for " + destination.toString());
1516 }
1517
1518
1519
1520
1521 static class CreateUnassignedAsyncCallback implements AsyncCallback.StringCallback {
1522 private final Log LOG = LogFactory.getLog(CreateUnassignedAsyncCallback.class);
1523 private final ZooKeeperWatcher zkw;
1524 private final ServerName destination;
1525 private final AtomicInteger counter;
1526
1527 CreateUnassignedAsyncCallback(final ZooKeeperWatcher zkw,
1528 final ServerName destination, final AtomicInteger counter) {
1529 this.zkw = zkw;
1530 this.destination = destination;
1531 this.counter = counter;
1532 }
1533
1534 @Override
1535 public void processResult(int rc, String path, Object ctx, String name) {
1536 if (rc != 0) {
1537
1538 LOG.warn("rc != 0 for " + path + " -- retryable connectionloss -- " +
1539 "FIX see http://wiki.apache.org/hadoop/ZooKeeper/FAQ#A2");
1540 this.zkw.abort("Connectionloss writing unassigned at " + path +
1541 ", rc=" + rc, null);
1542 return;
1543 }
1544 LOG.debug("rs=" + (RegionState)ctx + ", server=" + this.destination.toString());
1545
1546
1547 this.zkw.getRecoverableZooKeeper().getZooKeeper().exists(path, this.zkw,
1548 new ExistsUnassignedAsyncCallback(this.counter, destination), ctx);
1549 }
1550 }
1551
1552
1553
1554
1555
1556 static class ExistsUnassignedAsyncCallback implements AsyncCallback.StatCallback {
1557 private final Log LOG = LogFactory.getLog(ExistsUnassignedAsyncCallback.class);
1558 private final AtomicInteger counter;
1559 private ServerName destination;
1560
1561 ExistsUnassignedAsyncCallback(final AtomicInteger counter, ServerName destination) {
1562 this.counter = counter;
1563 this.destination = destination;
1564 }
1565
1566 @Override
1567 public void processResult(int rc, String path, Object ctx, Stat stat) {
1568 if (rc != 0) {
1569
1570 LOG.warn("rc != 0 for " + path + " -- retryable connectionloss -- " +
1571 "FIX see http://wiki.apache.org/hadoop/ZooKeeper/FAQ#A2");
1572 return;
1573 }
1574 RegionState state = (RegionState)ctx;
1575 LOG.debug("rs=" + state);
1576
1577
1578
1579
1580
1581 state.update(RegionState.State.PENDING_OPEN, System.currentTimeMillis(), destination);
1582 this.counter.addAndGet(1);
1583 }
1584 }
1585
1586
1587
1588
1589
1590 private RegionState addToRegionsInTransition(final HRegionInfo region) {
1591 return addToRegionsInTransition(region, false);
1592 }
1593
1594
1595
1596
1597
1598 private RegionState addToRegionsInTransition(final HRegionInfo region,
1599 boolean hijack) {
1600 synchronized (regionsInTransition) {
1601 return forceRegionStateToOffline(region, hijack);
1602 }
1603 }
1604
1605
1606
1607
1608
1609
1610 private RegionState forceRegionStateToOffline(final HRegionInfo region) {
1611 return forceRegionStateToOffline(region, false);
1612 }
1613
1614
1615
1616
1617
1618
1619
1620
1621 private RegionState forceRegionStateToOffline(final HRegionInfo region,
1622 boolean hijack) {
1623 String encodedName = region.getEncodedName();
1624 RegionState state = this.regionsInTransition.get(encodedName);
1625 if (state == null) {
1626 state = new RegionState(region, RegionState.State.OFFLINE);
1627 this.regionsInTransition.put(encodedName, state);
1628 } else {
1629
1630
1631
1632
1633
1634
1635 if (!hijack) {
1636 LOG.debug("Forcing OFFLINE; was=" + state);
1637 state.update(RegionState.State.OFFLINE);
1638 }
1639 }
1640 return state;
1641 }
1642
1643
1644
1645
1646
1647
1648
1649
1650 private void assign(final HRegionInfo region, final RegionState state,
1651 final boolean setOfflineInZK, final boolean forceNewPlan,
1652 boolean hijack) {
1653 boolean regionAlreadyInTransitionException = false;
1654 for (int i = 0; i < this.maximumAssignmentAttempts; i++) {
1655 int versionOfOfflineNode = -1;
1656 if (setOfflineInZK) {
1657
1658
1659 versionOfOfflineNode = setOfflineInZooKeeper(state, hijack,
1660 regionAlreadyInTransitionException);
1661 if(versionOfOfflineNode != -1){
1662 if (isDisabledorDisablingRegionInRIT(region)) {
1663 return;
1664 }
1665
1666
1667
1668
1669
1670
1671 String tableName = region.getTableNameAsString();
1672 if (!zkTable.isEnablingTable(tableName) && !zkTable.isEnabledTable(tableName)) {
1673 LOG.debug("Setting table " + tableName + " to ENABLED state.");
1674 setEnabledTable(region);
1675 }
1676 }
1677 }
1678
1679 if (setOfflineInZK && versionOfOfflineNode == -1) {
1680 return;
1681 }
1682
1683 if (this.master.isStopped()) {
1684 LOG.debug("Server stopped; skipping assign of " + state);
1685 return;
1686 }
1687 RegionPlan plan = getRegionPlan(state, !regionAlreadyInTransitionException && forceNewPlan);
1688 if (plan == null) {
1689 LOG.debug("Unable to determine a plan to assign " + state);
1690 this.timeoutMonitor.setAllRegionServersOffline(true);
1691 return;
1692 }
1693 try {
1694 LOG.debug("Assigning region " + state.getRegion().getRegionNameAsString() +
1695 " to " + plan.getDestination().toString());
1696 long currentOfflineTimeStamp = state.getStamp();
1697 RegionOpeningState regionOpenState = serverManager.sendRegionOpen(plan.getDestination(),
1698 state.getRegion(), versionOfOfflineNode);
1699 if (regionOpenState == RegionOpeningState.OPENED) {
1700
1701
1702
1703 if (state.isOffline() && currentOfflineTimeStamp != state.getStamp()) {
1704 return;
1705 }
1706 if (state.isOffline() && !state.isOpening()) {
1707 state.update(RegionState.State.PENDING_OPEN,
1708 System.currentTimeMillis(), plan.getDestination());
1709 }
1710 if (state.isOpening()) return;
1711 if (state.isOpened()) return;
1712 } else if (regionOpenState == RegionOpeningState.ALREADY_OPENED) {
1713
1714
1715
1716 LOG.debug("ALREADY_OPENED region " + state.getRegion().getRegionNameAsString() +
1717 " to " + plan.getDestination().toString());
1718 String encodedRegionName = state.getRegion()
1719 .getEncodedName();
1720 try {
1721 ZKAssign.deleteOfflineNode(master.getZooKeeper(), encodedRegionName);
1722 } catch (KeeperException.NoNodeException e) {
1723 if(LOG.isDebugEnabled()){
1724 LOG.debug("The unassigned node "+encodedRegionName+" doesnot exist.");
1725 }
1726 } catch (KeeperException e) {
1727 master.abort(
1728 "Error deleting OFFLINED node in ZK for transition ZK node ("
1729 + encodedRegionName + ")", e);
1730 }
1731 synchronized (this.regionsInTransition) {
1732 this.regionsInTransition.remove(plan.getRegionInfo()
1733 .getEncodedName());
1734 }
1735 synchronized (this.regions) {
1736 this.regions.put(plan.getRegionInfo(), plan.getDestination());
1737 }
1738 }
1739 break;
1740 } catch (Throwable t) {
1741 if (t instanceof RemoteException) {
1742 t = ((RemoteException) t).unwrapRemoteException();
1743 if (t instanceof RegionAlreadyInTransitionException) {
1744 regionAlreadyInTransitionException = true;
1745 if (LOG.isDebugEnabled()) {
1746 LOG.debug("Failed assignment in: " + plan.getDestination() + " due to "
1747 + t.getMessage());
1748 }
1749 }
1750 }
1751 if (t instanceof java.net.SocketTimeoutException
1752 && this.serverManager.isServerOnline(plan.getDestination())) {
1753 LOG.warn("Call openRegion() to " + plan.getDestination()
1754 + " has timed out when trying to assign "
1755 + region.getRegionNameAsString()
1756 + ", but the region might already be opened on "
1757 + plan.getDestination() + ".", t);
1758 return;
1759 }
1760 LOG.warn("Failed assignment of "
1761 + state.getRegion().getRegionNameAsString()
1762 + " to "
1763 + plan.getDestination()
1764 + ", trying to assign "
1765 + (regionAlreadyInTransitionException ? "to the same region server"
1766 + " because of RegionAlreadyInTransitionException;" : "elsewhere instead; ")
1767 + "retry=" + i, t);
1768
1769
1770
1771 state.update(RegionState.State.OFFLINE);
1772
1773
1774
1775 RegionPlan newPlan = plan;
1776 if (!regionAlreadyInTransitionException) {
1777
1778 newPlan = getRegionPlan(state, plan.getDestination(), true);
1779 }
1780 if (newPlan == null) {
1781 this.timeoutMonitor.setAllRegionServersOffline(true);
1782 LOG.warn("Unable to find a viable location to assign region " +
1783 state.getRegion().getRegionNameAsString());
1784 return;
1785 }
1786 }
1787 }
1788 }
1789
1790 private boolean isDisabledorDisablingRegionInRIT(final HRegionInfo region) {
1791 String tableName = region.getTableNameAsString();
1792 boolean disabled = this.zkTable.isDisabledTable(tableName);
1793 if (disabled || this.zkTable.isDisablingTable(tableName)) {
1794 LOG.info("Table " + tableName + (disabled ? " disabled;" : " disabling;") +
1795 " skipping assign of " + region.getRegionNameAsString());
1796 offlineDisabledRegion(region);
1797 return true;
1798 }
1799 return false;
1800 }
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813 int setOfflineInZooKeeper(final RegionState state, boolean hijack,
1814 boolean regionAlreadyInTransitionException) {
1815
1816
1817 if (!hijack && !state.isClosed() && !state.isOffline()) {
1818 if (!regionAlreadyInTransitionException ) {
1819 String msg = "Unexpected state : " + state + " .. Cannot transit it to OFFLINE.";
1820 this.master.abort(msg, new IllegalStateException(msg));
1821 return -1;
1822 }
1823 LOG.debug("Unexpected state : " + state
1824 + " but retrying to assign because RegionAlreadyInTransitionException.");
1825 }
1826 boolean allowZNodeCreation = false;
1827
1828
1829
1830
1831
1832
1833
1834 if (hijack &&
1835 (state.getState().equals(RegionState.State.PENDING_OPEN) ||
1836 state.getState().equals(RegionState.State.OPENING))) {
1837 state.update(RegionState.State.PENDING_OPEN);
1838 allowZNodeCreation = false;
1839 } else {
1840 state.update(RegionState.State.OFFLINE);
1841 allowZNodeCreation = true;
1842 }
1843 int versionOfOfflineNode = -1;
1844 try {
1845
1846 versionOfOfflineNode = ZKAssign.createOrForceNodeOffline(master.getZooKeeper(),
1847 state.getRegion(), this.master.getServerName(),
1848 hijack, allowZNodeCreation);
1849 if (versionOfOfflineNode == -1) {
1850 LOG.warn("Attempted to create/force node into OFFLINE state before "
1851 + "completing assignment but failed to do so for " + state);
1852 return -1;
1853 }
1854 } catch (KeeperException e) {
1855 master.abort("Unexpected ZK exception creating/setting node OFFLINE", e);
1856 return -1;
1857 }
1858 return versionOfOfflineNode;
1859 }
1860
1861
1862
1863
1864
1865
1866
1867 boolean asyncSetOfflineInZooKeeper(final RegionState state,
1868 final AsyncCallback.StringCallback cb, final Object ctx) {
1869 if (!state.isClosed() && !state.isOffline()) {
1870 new RuntimeException("Unexpected state trying to OFFLINE; " + state);
1871 this.master.abort("Unexpected state trying to OFFLINE; " + state,
1872 new IllegalStateException());
1873 return false;
1874 }
1875 state.update(RegionState.State.OFFLINE);
1876 try {
1877 ZKAssign.asyncCreateNodeOffline(master.getZooKeeper(), state.getRegion(),
1878 this.master.getServerName(), cb, ctx);
1879 } catch (KeeperException e) {
1880 if (e instanceof NodeExistsException) {
1881 LOG.warn("Node for " + state.getRegion() + " already exists");
1882 } else {
1883 master.abort("Unexpected ZK exception creating/setting node OFFLINE", e);
1884 }
1885 return false;
1886 }
1887 return true;
1888 }
1889
1890
1891
1892
1893
1894
1895 RegionPlan getRegionPlan(final RegionState state,
1896 final boolean forceNewPlan) {
1897 return getRegionPlan(state, null, forceNewPlan);
1898 }
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909 RegionPlan getRegionPlan(final RegionState state,
1910 final ServerName serverToExclude, final boolean forceNewPlan) {
1911
1912 final String encodedName = state.getRegion().getEncodedName();
1913 final List<ServerName> servers = this.serverManager.getOnlineServersList();
1914 final List<ServerName> drainingServers = this.serverManager.getDrainingServersList();
1915
1916
1917 if (serverToExclude != null) servers.remove(serverToExclude);
1918
1919
1920
1921 if (!drainingServers.isEmpty()) {
1922 for (final ServerName server: drainingServers) {
1923 LOG.debug("Removing draining server: " + server +
1924 " from eligible server pool.");
1925 servers.remove(server);
1926 }
1927 }
1928
1929
1930 removeDeadNotExpiredServers(servers);
1931
1932
1933
1934 if (servers.isEmpty()) return null;
1935
1936 RegionPlan randomPlan = null;
1937 boolean newPlan = false;
1938 RegionPlan existingPlan = null;
1939
1940 synchronized (this.regionPlans) {
1941 existingPlan = this.regionPlans.get(encodedName);
1942
1943 if (existingPlan != null && existingPlan.getDestination() != null) {
1944 LOG.debug("Found an existing plan for " +
1945 state.getRegion().getRegionNameAsString() +
1946 " destination server is " + existingPlan.getDestination().toString());
1947 }
1948
1949 if (forceNewPlan
1950 || existingPlan == null
1951 || existingPlan.getDestination() == null
1952 || drainingServers.contains(existingPlan.getDestination())) {
1953 newPlan = true;
1954 randomPlan = new RegionPlan(state.getRegion(), null, balancer
1955 .randomAssignment(servers));
1956 this.regionPlans.put(encodedName, randomPlan);
1957 }
1958 }
1959
1960 if (newPlan) {
1961 LOG.debug("No previous transition plan was found (or we are ignoring " +
1962 "an existing plan) for " + state.getRegion().getRegionNameAsString() +
1963 " so generated a random one; " + randomPlan + "; " +
1964 serverManager.countOfRegionServers() +
1965 " (online=" + serverManager.getOnlineServers().size() +
1966 ", available=" + servers.size() + ") available servers");
1967 return randomPlan;
1968 }
1969 LOG.debug("Using pre-existing plan for region " +
1970 state.getRegion().getRegionNameAsString() + "; plan=" + existingPlan);
1971 return existingPlan;
1972 }
1973
1974
1975
1976
1977
1978
1979 public void removeDeadNotExpiredServers(List<ServerName> servers) {
1980 Set<ServerName> deadNotExpiredServers = this.serverManager
1981 .getDeadNotExpiredServers();
1982 if (!deadNotExpiredServers.isEmpty()) {
1983 for (ServerName server : deadNotExpiredServers) {
1984 LOG.debug("Removing dead but not expired server: " + server
1985 + " from eligible server pool.");
1986 servers.remove(server);
1987 }
1988 }
1989 }
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999 public void unassign(List<HRegionInfo> regions) {
2000 int waitTime = this.master.getConfiguration().getInt(
2001 "hbase.bulk.waitbetween.reopen", 0);
2002 for (HRegionInfo region : regions) {
2003 if (isRegionInTransition(region) != null)
2004 continue;
2005 unassign(region, false);
2006 while (isRegionInTransition(region) != null) {
2007 try {
2008 Thread.sleep(10);
2009 } catch (InterruptedException e) {
2010
2011 }
2012 }
2013 if (waitTime > 0)
2014 try {
2015 Thread.sleep(waitTime);
2016 } catch (InterruptedException e) {
2017
2018 }
2019 }
2020 }
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035 public void unassign(HRegionInfo region) {
2036 unassign(region, false);
2037 }
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053 public void unassign(HRegionInfo region, boolean force) {
2054
2055 LOG.debug("Starting unassignment of region " +
2056 region.getRegionNameAsString() + " (offlining)");
2057
2058 synchronized (this.regions) {
2059
2060 if (!regions.containsKey(region)) {
2061 LOG.debug("Attempted to unassign region " +
2062 region.getRegionNameAsString() + " but it is not " +
2063 "currently assigned anywhere");
2064 return;
2065 }
2066 }
2067 String encodedName = region.getEncodedName();
2068
2069 RegionState state;
2070 int versionOfClosingNode = -1;
2071 synchronized (regionsInTransition) {
2072 state = regionsInTransition.get(encodedName);
2073 if (state == null) {
2074
2075 try {
2076 versionOfClosingNode = ZKAssign.createNodeClosing(
2077 master.getZooKeeper(), region, master.getServerName());
2078 if (versionOfClosingNode == -1) {
2079 LOG.debug("Attempting to unassign region " +
2080 region.getRegionNameAsString() + " but ZK closing node "
2081 + "can't be created.");
2082 return;
2083 }
2084 } catch (KeeperException e) {
2085 if (e instanceof NodeExistsException) {
2086
2087
2088
2089
2090 NodeExistsException nee = (NodeExistsException)e;
2091 String path = nee.getPath();
2092 try {
2093 if (isSplitOrSplitting(path)) {
2094 LOG.debug(path + " is SPLIT or SPLITTING; " +
2095 "skipping unassign because region no longer exists -- its split");
2096 return;
2097 }
2098 } catch (KeeperException.NoNodeException ke) {
2099 LOG.warn("Failed getData on SPLITTING/SPLIT at " + path +
2100 "; presuming split and that the region to unassign, " +
2101 encodedName + ", no longer exists -- confirm", ke);
2102 return;
2103 } catch (KeeperException ke) {
2104 LOG.error("Unexpected zk state", ke);
2105 ke = e;
2106 }
2107 }
2108
2109 master.abort("Unexpected ZK exception creating node CLOSING", e);
2110 return;
2111 }
2112 state = new RegionState(region, RegionState.State.PENDING_CLOSE);
2113 regionsInTransition.put(encodedName, state);
2114 } else if (force && (state.isPendingClose() || state.isClosing())) {
2115 LOG.debug("Attempting to unassign region " + region.getRegionNameAsString() +
2116 " which is already " + state.getState() +
2117 " but forcing to send a CLOSE RPC again ");
2118 state.update(state.getState());
2119 } else {
2120 LOG.debug("Attempting to unassign region " +
2121 region.getRegionNameAsString() + " but it is " +
2122 "already in transition (" + state.getState() + ", force=" + force + ")");
2123 return;
2124 }
2125 }
2126
2127 ServerName server = null;
2128 synchronized (this.regions) {
2129 server = regions.get(region);
2130 }
2131
2132 if (server == null) {
2133
2134 synchronized (regionsInTransition) {
2135 state = regionsInTransition.get(encodedName);
2136 if (state != null) {
2137
2138 State presentState = state.getState();
2139 if (presentState == State.PENDING_CLOSE
2140 || presentState == State.CLOSING) {
2141 this.regionsInTransition.remove(encodedName);
2142 }
2143 }
2144 }
2145
2146 deleteClosingOrClosedNode(region);
2147 return;
2148 }
2149 try {
2150
2151
2152 if (serverManager.sendRegionClose(server, state.getRegion(),
2153 versionOfClosingNode)) {
2154 LOG.debug("Sent CLOSE to " + server + " for region " +
2155 region.getRegionNameAsString());
2156 return;
2157 }
2158
2159 LOG.warn("Server " + server + " region CLOSE RPC returned false for " +
2160 region.getRegionNameAsString());
2161 } catch (NotServingRegionException nsre) {
2162 LOG.info("Server " + server + " returned " + nsre + " for " +
2163 region.getRegionNameAsString());
2164
2165
2166
2167 } catch (Throwable t) {
2168 if (t instanceof RemoteException) {
2169 t = ((RemoteException)t).unwrapRemoteException();
2170 if (t instanceof NotServingRegionException) {
2171 if (checkIfRegionBelongsToDisabling(region)
2172 || checkIfRegionBelongsToDisabled(region)) {
2173
2174 LOG.info("While trying to recover the table "
2175 + region.getTableNameAsString()
2176 + " to DISABLED state the region " + region
2177 + " was offlined but the table was in DISABLING state");
2178 synchronized (this.regionsInTransition) {
2179 this.regionsInTransition.remove(region.getEncodedName());
2180 }
2181
2182 synchronized (this.regions) {
2183 this.regions.remove(region);
2184 }
2185 deleteClosingOrClosedNode(region);
2186 }
2187 }
2188
2189 if (t instanceof RegionAlreadyInTransitionException) {
2190 LOG.debug("update " + state + " the timestamp.");
2191 state.update(state.getState());
2192 }
2193 }
2194 LOG.info("Server " + server + " returned " + t + " for " +
2195 region.getEncodedName());
2196
2197 }
2198 }
2199
2200
2201
2202
2203
2204 public void deleteClosingOrClosedNode(HRegionInfo region) {
2205 try {
2206 if (!ZKAssign.deleteNode(master.getZooKeeper(), region.getEncodedName(),
2207 EventHandler.EventType.M_ZK_REGION_CLOSING)) {
2208 boolean deleteNode = ZKAssign.deleteNode(master.getZooKeeper(), region
2209 .getEncodedName(), EventHandler.EventType.RS_ZK_REGION_CLOSED);
2210
2211
2212 if (!deleteNode) {
2213 LOG.error("The deletion of the CLOSED node for the region "
2214 + region.getEncodedName() + " returned " + deleteNode);
2215 }
2216 }
2217 } catch (NoNodeException e) {
2218 LOG.debug("CLOSING/CLOSED node for the region " + region.getEncodedName()
2219 + " already deleted");
2220 } catch (KeeperException ke) {
2221 master.abort(
2222 "Unexpected ZK exception deleting node CLOSING/CLOSED for the region "
2223 + region.getEncodedName(), ke);
2224 return;
2225 }
2226 }
2227
2228
2229
2230
2231
2232
2233 private boolean isSplitOrSplitting(final String path) throws KeeperException {
2234 boolean result = false;
2235
2236
2237 RegionTransitionData data = ZKAssign.getData(master.getZooKeeper(), path);
2238 EventType evt = data.getEventType();
2239 switch (evt) {
2240 case RS_ZK_REGION_SPLIT:
2241 case RS_ZK_REGION_SPLITTING:
2242 result = true;
2243 break;
2244 default:
2245 break;
2246 }
2247 return result;
2248 }
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258 public void waitForAssignment(HRegionInfo regionInfo)
2259 throws InterruptedException {
2260 synchronized(regions) {
2261 while(!regions.containsKey(regionInfo)) {
2262
2263
2264
2265 regions.wait(100);
2266 }
2267 }
2268 }
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280 public void assignRoot() throws KeeperException {
2281 RootLocationEditor.deleteRootLocation(this.master.getZooKeeper());
2282 assign(HRegionInfo.ROOT_REGIONINFO, true);
2283 }
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293 public void assignMeta() {
2294
2295 assign(HRegionInfo.FIRST_META_REGIONINFO, true);
2296 }
2297
2298
2299
2300
2301
2302
2303
2304
2305 public void assignUserRegionsToOnlineServers(List<HRegionInfo> regions)
2306 throws IOException,
2307 InterruptedException {
2308 List<ServerName> servers = this.serverManager.getOnlineServersList();
2309 removeDeadNotExpiredServers(servers);
2310 assignUserRegions(regions, servers);
2311 }
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321 public void assignUserRegions(List<HRegionInfo> regions, List<ServerName> servers)
2322 throws IOException, InterruptedException {
2323 if (regions == null)
2324 return;
2325 Map<ServerName, List<HRegionInfo>> bulkPlan = null;
2326
2327 bulkPlan = balancer.roundRobinAssignment(regions, servers);
2328 LOG.info("Bulk assigning " + regions.size() + " region(s) round-robin across " +
2329 servers.size() + " server(s)");
2330
2331 BulkAssigner ba = new StartupBulkAssigner(this.master, bulkPlan, this);
2332 ba.bulkAssign();
2333 LOG.info("Bulk assigning done");
2334 }
2335
2336 private void setEnabledTable(HRegionInfo hri) {
2337 String tableName = hri.getTableNameAsString();
2338 boolean isTableEnabled = this.zkTable.isEnabledTable(tableName);
2339 if (!isTableEnabled) {
2340 setEnabledTable(tableName);
2341 }
2342 }
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353 public void assignAllUserRegions() throws IOException, InterruptedException {
2354
2355
2356
2357 Set<String> disablingDisabledAndEnablingTables = new HashSet<String>(this.disablingTables);
2358 disablingDisabledAndEnablingTables.addAll(this.zkTable.getDisabledTables());
2359 disablingDisabledAndEnablingTables.addAll(this.enablingTables.keySet());
2360
2361 Map<HRegionInfo, ServerName> allRegions = MetaReader.fullScan(catalogTracker,
2362 disablingDisabledAndEnablingTables, true);
2363 if (allRegions == null || allRegions.isEmpty()) return;
2364
2365
2366 List<ServerName> servers = serverManager.getOnlineServersList();
2367
2368
2369 removeDeadNotExpiredServers(servers);
2370
2371
2372 if(servers.isEmpty()) return;
2373
2374
2375 boolean retainAssignment = master.getConfiguration().
2376 getBoolean("hbase.master.startup.retainassign", true);
2377
2378 Map<ServerName, List<HRegionInfo>> bulkPlan = null;
2379 if (retainAssignment) {
2380
2381 bulkPlan = balancer.retainAssignment(allRegions, servers);
2382 } else {
2383
2384 assignUserRegions(new ArrayList<HRegionInfo>(allRegions.keySet()), servers);
2385 for (HRegionInfo hri : allRegions.keySet()) {
2386 setEnabledTable(hri);
2387 }
2388 return;
2389 }
2390 LOG.info("Bulk assigning " + allRegions.size() + " region(s) across " +
2391 servers.size() + " server(s), retainAssignment=" + retainAssignment);
2392
2393
2394 BulkAssigner ba = new StartupBulkAssigner(this.master, bulkPlan, this);
2395 ba.bulkAssign();
2396 for (HRegionInfo hri : allRegions.keySet()) {
2397 setEnabledTable(hri);
2398 }
2399 LOG.info("Bulk assigning done");
2400 }
2401
2402
2403
2404
2405
2406
2407
2408 static class StartupBulkAssigner extends BulkAssigner {
2409 final Map<ServerName, List<HRegionInfo>> bulkPlan;
2410 final AssignmentManager assignmentManager;
2411
2412 StartupBulkAssigner(final Server server,
2413 final Map<ServerName, List<HRegionInfo>> bulkPlan,
2414 final AssignmentManager am) {
2415 super(server);
2416 this.bulkPlan = bulkPlan;
2417 this.assignmentManager = am;
2418 }
2419
2420 @Override
2421 public boolean bulkAssign(boolean sync) throws InterruptedException,
2422 IOException {
2423
2424 this.assignmentManager.timeoutMonitor.bulkAssign(true);
2425 try {
2426 return super.bulkAssign(sync);
2427 } finally {
2428
2429 this.assignmentManager.timeoutMonitor.bulkAssign(false);
2430 }
2431 }
2432
2433 @Override
2434 protected String getThreadNamePrefix() {
2435 return this.server.getServerName() + "-StartupBulkAssigner";
2436 }
2437
2438 @Override
2439 protected void populatePool(java.util.concurrent.ExecutorService pool) {
2440 for (Map.Entry<ServerName, List<HRegionInfo>> e: this.bulkPlan.entrySet()) {
2441 pool.execute(new SingleServerBulkAssigner(e.getKey(), e.getValue(),
2442 this.assignmentManager));
2443 }
2444 }
2445
2446 protected boolean waitUntilDone(final long timeout)
2447 throws InterruptedException {
2448 Set<HRegionInfo> regionSet = new HashSet<HRegionInfo>();
2449 for (List<HRegionInfo> regionList : bulkPlan.values()) {
2450 regionSet.addAll(regionList);
2451 }
2452 return this.assignmentManager.waitUntilNoRegionsInTransition(timeout, regionSet);
2453 }
2454
2455 @Override
2456 protected long getTimeoutOnRIT() {
2457
2458
2459 long perRegionOpenTimeGuesstimate =
2460 this.server.getConfiguration().getLong("hbase.bulk.assignment.perregion.open.time", 1000);
2461 int regionsPerServer =
2462 this.bulkPlan.entrySet().iterator().next().getValue().size();
2463 long timeout = perRegionOpenTimeGuesstimate * regionsPerServer;
2464 LOG.debug("Timeout-on-RIT=" + timeout);
2465 return timeout;
2466 }
2467 }
2468
2469
2470
2471
2472
2473 static class GeneralBulkAssigner extends StartupBulkAssigner {
2474 GeneralBulkAssigner(final Server server,
2475 final Map<ServerName, List<HRegionInfo>> bulkPlan,
2476 final AssignmentManager am) {
2477 super(server, bulkPlan, am);
2478 }
2479
2480 @Override
2481 protected UncaughtExceptionHandler getUncaughtExceptionHandler() {
2482 return new UncaughtExceptionHandler() {
2483 @Override
2484 public void uncaughtException(Thread t, Throwable e) {
2485 LOG.warn("Assigning regions in " + t.getName(), e);
2486 }
2487 };
2488 }
2489 }
2490
2491
2492
2493
2494 static class SingleServerBulkAssigner implements Runnable {
2495 private final ServerName regionserver;
2496 private final List<HRegionInfo> regions;
2497 private final AssignmentManager assignmentManager;
2498
2499 SingleServerBulkAssigner(final ServerName regionserver,
2500 final List<HRegionInfo> regions, final AssignmentManager am) {
2501 this.regionserver = regionserver;
2502 this.regions = regions;
2503 this.assignmentManager = am;
2504 }
2505 @Override
2506 public void run() {
2507 this.assignmentManager.assign(this.regionserver, this.regions);
2508 }
2509 }
2510
2511
2512
2513
2514
2515
2516
2517 boolean waitUntilNoRegionsInTransition(final long timeout)
2518 throws InterruptedException {
2519
2520
2521
2522
2523
2524
2525 long startTime = System.currentTimeMillis();
2526 long remaining = timeout;
2527 synchronized (regionsInTransition) {
2528 while (regionsInTransition.size() > 0 && !this.master.isStopped()
2529 && remaining > 0) {
2530 regionsInTransition.wait(remaining);
2531 remaining = timeout - (System.currentTimeMillis() - startTime);
2532 }
2533 }
2534 return regionsInTransition.isEmpty();
2535 }
2536
2537
2538
2539
2540
2541
2542
2543
2544 boolean waitUntilNoRegionsInTransition(final long timeout, Set<HRegionInfo> regions)
2545 throws InterruptedException {
2546
2547 long startTime = System.currentTimeMillis();
2548 long remaining = timeout;
2549 boolean stillInTransition = true;
2550 synchronized (regionsInTransition) {
2551 while (regionsInTransition.size() > 0 && !this.master.isStopped() &&
2552 remaining > 0 && stillInTransition) {
2553 int count = 0;
2554 for (RegionState rs : regionsInTransition.values()) {
2555 if (regions.contains(rs.getRegion())) {
2556 count++;
2557 break;
2558 }
2559 }
2560 if (count == 0) {
2561 stillInTransition = false;
2562 break;
2563 }
2564 regionsInTransition.wait(remaining);
2565 remaining = timeout - (System.currentTimeMillis() - startTime);
2566 }
2567 }
2568 return stillInTransition;
2569 }
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580 Map<ServerName, List<Pair<HRegionInfo, Result>>> rebuildUserRegions() throws IOException,
2581 KeeperException {
2582
2583 List<Result> results = MetaReader.fullScan(this.catalogTracker);
2584
2585 Set<ServerName> onlineServers = serverManager.getOnlineServers().keySet();
2586
2587 Map<ServerName, List<Pair<HRegionInfo,Result>>> offlineServers =
2588 new TreeMap<ServerName, List<Pair<HRegionInfo, Result>>>();
2589
2590 for (Result result : results) {
2591 boolean disabled = false;
2592 boolean disablingOrEnabling = false;
2593 Pair<HRegionInfo, ServerName> region = MetaReader.parseCatalogResult(result);
2594 if (region == null) continue;
2595 HRegionInfo regionInfo = region.getFirst();
2596 ServerName regionLocation = region.getSecond();
2597 if (regionInfo == null) continue;
2598 String tableName = regionInfo.getTableNameAsString();
2599 if (regionLocation == null) {
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610 boolean enabling = checkIfRegionsBelongsToEnabling(regionInfo);
2611 addTheTablesInPartialState(regionInfo);
2612 if (enabling) {
2613 addToEnablingTableRegions(regionInfo);
2614 } else {
2615 LOG.warn("Region " + regionInfo.getEncodedName() + " has null regionLocation."
2616 + " But its table " + tableName + " isn't in ENABLING state.");
2617 }
2618 } else if (!onlineServers.contains(regionLocation)) {
2619
2620 List<Pair<HRegionInfo, Result>> offlineRegions =
2621 offlineServers.get(regionLocation);
2622 if (offlineRegions == null) {
2623 offlineRegions = new ArrayList<Pair<HRegionInfo,Result>>(1);
2624 offlineServers.put(regionLocation, offlineRegions);
2625 }
2626 offlineRegions.add(new Pair<HRegionInfo,Result>(regionInfo, result));
2627 disabled = checkIfRegionBelongsToDisabled(regionInfo);
2628 disablingOrEnabling = addTheTablesInPartialState(regionInfo);
2629
2630
2631 enableTableIfNotDisabledOrDisablingOrEnabling(disabled,
2632 disablingOrEnabling, tableName);
2633 } else {
2634
2635 if (regionInfo.isOffline() && regionInfo.isSplit()) {
2636 String node = ZKAssign.getNodeName(this.watcher, regionInfo
2637 .getEncodedName());
2638 Stat stat = new Stat();
2639 byte[] data = ZKUtil.getDataNoWatch(this.watcher, node, stat);
2640
2641 if (data == null) {
2642 LOG.debug("Region "+ regionInfo.getRegionNameAsString() + " split is completed. "
2643 + "Hence need not add to regions list");
2644 continue;
2645 }
2646 }
2647
2648
2649 boolean enabling = checkIfRegionsBelongsToEnabling(regionInfo);
2650 disabled = checkIfRegionBelongsToDisabled(regionInfo);
2651 if (!enabling && !disabled) {
2652 synchronized (this.regions) {
2653 regions.put(regionInfo, regionLocation);
2654 addToServers(regionLocation, regionInfo);
2655 }
2656 }
2657 disablingOrEnabling = addTheTablesInPartialState(regionInfo);
2658 if (enabling) {
2659 addToEnablingTableRegions(regionInfo);
2660 }
2661
2662
2663 enableTableIfNotDisabledOrDisablingOrEnabling(disabled,
2664 disablingOrEnabling, tableName);
2665 }
2666 }
2667 return offlineServers;
2668 }
2669
2670 private void addToEnablingTableRegions(HRegionInfo regionInfo) {
2671 String tableName = regionInfo.getTableNameAsString();
2672 List<HRegionInfo> hris = this.enablingTables.get(tableName);
2673 if (!hris.contains(regionInfo)) {
2674 if (LOG.isDebugEnabled()) {
2675 LOG.debug("Adding region" + regionInfo.getRegionNameAsString()
2676 + " to enabling table " + tableName + ".");
2677 }
2678 hris.add(regionInfo);
2679 }
2680 }
2681
2682 private void enableTableIfNotDisabledOrDisablingOrEnabling(boolean disabled,
2683 boolean disablingOrEnabling, String tableName) {
2684 if (!disabled && !disablingOrEnabling
2685 && !getZKTable().isEnabledTable(tableName)) {
2686 setEnabledTable(tableName);
2687 }
2688 }
2689
2690 private Boolean addTheTablesInPartialState(HRegionInfo regionInfo) {
2691 String tableName = regionInfo.getTableNameAsString();
2692 if (checkIfRegionBelongsToDisabling(regionInfo)) {
2693 this.disablingTables.add(tableName);
2694 return true;
2695 } else if (checkIfRegionsBelongsToEnabling(regionInfo)) {
2696 if (!this.enablingTables.containsKey(tableName)) {
2697 this.enablingTables.put(tableName, new ArrayList<HRegionInfo>());
2698 }
2699 return true;
2700 }
2701 return false;
2702 }
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714 private boolean recoverTableInDisablingState(Set<String> disablingTables)
2715 throws KeeperException, TableNotFoundException, IOException {
2716 boolean isWatcherCreated = false;
2717 if (disablingTables.size() != 0) {
2718
2719 ZKUtil.listChildrenAndWatchForNewChildren(watcher,
2720 watcher.assignmentZNode);
2721 isWatcherCreated = true;
2722 for (String tableName : disablingTables) {
2723
2724 LOG.info("The table " + tableName
2725 + " is in DISABLING state. Hence recovering by moving the table"
2726 + " to DISABLED state.");
2727 new DisableTableHandler(this.master, tableName.getBytes(),
2728 catalogTracker, this, true).process();
2729 }
2730 }
2731 return isWatcherCreated;
2732 }
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744 private void recoverTableInEnablingState(Set<String> enablingTables,
2745 boolean isWatcherCreated) throws KeeperException, TableNotFoundException,
2746 IOException {
2747 if (enablingTables.size() != 0) {
2748 if (false == isWatcherCreated) {
2749 ZKUtil.listChildrenAndWatchForNewChildren(watcher,
2750 watcher.assignmentZNode);
2751 }
2752 for (String tableName : enablingTables) {
2753
2754 LOG.info("The table " + tableName
2755 + " is in ENABLING state. Hence recovering by moving the table"
2756 + " to ENABLED state.");
2757
2758
2759 new EnableTableHandler(this.master, tableName.getBytes(),
2760 catalogTracker, this, true).process();
2761 }
2762 }
2763 }
2764
2765 private boolean checkIfRegionsBelongsToEnabling(HRegionInfo regionInfo) {
2766 String tableName = regionInfo.getTableNameAsString();
2767 return getZKTable().isEnablingTable(tableName);
2768 }
2769
2770 private boolean checkIfRegionBelongsToDisabled(HRegionInfo regionInfo) {
2771 String tableName = regionInfo.getTableNameAsString();
2772 return getZKTable().isDisabledTable(tableName);
2773 }
2774
2775 private boolean checkIfRegionBelongsToDisabling(HRegionInfo regionInfo) {
2776 String tableName = regionInfo.getTableNameAsString();
2777 return getZKTable().isDisablingTable(tableName);
2778 }
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796 private void processDeadServersAndRecoverLostRegions(
2797 Map<ServerName, List<Pair<HRegionInfo, Result>>> deadServers,
2798 List<String> nodes) throws IOException, KeeperException {
2799 if (null != deadServers) {
2800 Set<ServerName> actualDeadServers = this.serverManager.getDeadServers();
2801 for (Map.Entry<ServerName, List<Pair<HRegionInfo, Result>>> deadServer :
2802 deadServers.entrySet()) {
2803
2804
2805 if (actualDeadServers.contains(deadServer.getKey())) {
2806 for (Pair<HRegionInfo, Result> deadRegion : deadServer.getValue()) {
2807 nodes.remove(deadRegion.getFirst().getEncodedName());
2808 }
2809 continue;
2810 }
2811 List<Pair<HRegionInfo, Result>> regions = deadServer.getValue();
2812 for (Pair<HRegionInfo, Result> region : regions) {
2813 HRegionInfo regionInfo = region.getFirst();
2814 Result result = region.getSecond();
2815
2816
2817 try {
2818 RegionTransitionData data = ZKAssign.getData(watcher,
2819 regionInfo.getEncodedName());
2820
2821
2822
2823
2824
2825 if (data != null && data.getOrigin() != null &&
2826 serverManager.isServerOnline(data.getOrigin())) {
2827 LOG.info("The region " + regionInfo.getEncodedName()
2828 + "is being handled on " + data.getOrigin());
2829 continue;
2830 }
2831
2832 boolean assign = ServerShutdownHandler.processDeadRegion(
2833 regionInfo, result, this, this.catalogTracker);
2834 if (assign) {
2835 ZKAssign.createOrForceNodeOffline(watcher, regionInfo,
2836 master.getServerName());
2837 if (!nodes.contains(regionInfo.getEncodedName())) {
2838 nodes.add(regionInfo.getEncodedName());
2839 }
2840 }
2841 } catch (KeeperException.NoNodeException nne) {
2842
2843 }
2844 }
2845 }
2846 }
2847
2848 if (!nodes.isEmpty()) {
2849 for (String encodedRegionName : nodes) {
2850 processRegionInTransition(encodedRegionName, null, deadServers);
2851 }
2852 }
2853 }
2854
2855
2856
2857
2858
2859
2860 private void addToServers(final ServerName sn, final HRegionInfo hri) {
2861 Set<HRegionInfo> hris = servers.get(sn);
2862 if (hris == null) {
2863 hris = new ConcurrentSkipListSet<HRegionInfo>();
2864 servers.put(sn, hris);
2865 }
2866 if (!hris.contains(hri)) hris.add(hri);
2867 }
2868
2869
2870
2871
2872 public NavigableMap<String, RegionState> getRegionsInTransition() {
2873 synchronized (this.regionsInTransition) {
2874 return new TreeMap<String, RegionState>(this.regionsInTransition);
2875 }
2876 }
2877
2878
2879
2880
2881 public boolean isRegionsInTransition() {
2882 synchronized (this.regionsInTransition) {
2883 return !this.regionsInTransition.isEmpty();
2884 }
2885 }
2886
2887
2888
2889
2890
2891
2892 public RegionState isRegionInTransition(final HRegionInfo hri) {
2893 synchronized (this.regionsInTransition) {
2894 return this.regionsInTransition.get(hri.getEncodedName());
2895 }
2896 }
2897
2898
2899
2900
2901
2902
2903
2904
2905 public void clearRegionFromTransition(HRegionInfo hri) {
2906 synchronized (this.regionsInTransition) {
2907 this.regionsInTransition.remove(hri.getEncodedName());
2908 }
2909 synchronized (this.regions) {
2910 this.regions.remove(hri);
2911 for (Set<HRegionInfo> regions : this.servers.values()) {
2912 regions.remove(hri);
2913 }
2914 }
2915 clearRegionPlan(hri);
2916 }
2917
2918
2919
2920
2921 void clearRegionPlan(final HRegionInfo region) {
2922 synchronized (this.regionPlans) {
2923 this.regionPlans.remove(region.getEncodedName());
2924 }
2925 }
2926
2927
2928
2929
2930
2931
2932 public void waitOnRegionToClearRegionsInTransition(final HRegionInfo hri)
2933 throws IOException {
2934 if (isRegionInTransition(hri) == null) return;
2935 RegionState rs = null;
2936
2937
2938 while(!this.master.isStopped() && (rs = isRegionInTransition(hri)) != null) {
2939 Threads.sleep(1000);
2940 LOG.info("Waiting on " + rs + " to clear regions-in-transition");
2941 }
2942 if (this.master.isStopped()) {
2943 LOG.info("Giving up wait on regions in " +
2944 "transition because stoppable.isStopped is set");
2945 }
2946 }
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959 public List<HRegionInfo> getRegionsOfTable(byte[] tableName) {
2960 List<HRegionInfo> tableRegions = new ArrayList<HRegionInfo>();
2961
2962
2963 HRegionInfo boundary =
2964 new HRegionInfo(tableName, null, null, false, 0L);
2965 synchronized (this.regions) {
2966 for (HRegionInfo regionInfo: this.regions.tailMap(boundary).keySet()) {
2967 if(Bytes.equals(regionInfo.getTableName(), tableName)) {
2968 tableRegions.add(regionInfo);
2969 } else {
2970 break;
2971 }
2972 }
2973 }
2974 return tableRegions;
2975 }
2976
2977
2978
2979
2980
2981 public class TimerUpdater extends Chore {
2982
2983 public TimerUpdater(final int period, final Stoppable stopper) {
2984 super("AssignmentTimerUpdater", period, stopper);
2985 }
2986
2987 @Override
2988 protected void chore() {
2989 ServerName serverToUpdateTimer = null;
2990 while (!serversInUpdatingTimer.isEmpty() && !stopper.isStopped()) {
2991 if (serverToUpdateTimer == null) {
2992 serverToUpdateTimer = serversInUpdatingTimer.first();
2993 } else {
2994 serverToUpdateTimer = serversInUpdatingTimer
2995 .higher(serverToUpdateTimer);
2996 }
2997 if (serverToUpdateTimer == null) {
2998 break;
2999 }
3000 updateTimers(serverToUpdateTimer);
3001 serversInUpdatingTimer.remove(serverToUpdateTimer);
3002 }
3003 }
3004 }
3005
3006
3007
3008
3009 public class TimeoutMonitor extends Chore {
3010 private final int timeout;
3011 private boolean bulkAssign = false;
3012 private boolean allRegionServersOffline = false;
3013 private ServerManager serverManager;
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024 public TimeoutMonitor(final int period, final Stoppable stopper,
3025 ServerManager serverManager,
3026 final int timeout) {
3027 super("AssignmentTimeoutMonitor", period, stopper);
3028 this.timeout = timeout;
3029 this.serverManager = serverManager;
3030 }
3031
3032
3033
3034
3035
3036
3037 public boolean bulkAssign(final boolean bulkAssign) {
3038 boolean result = this.bulkAssign;
3039 this.bulkAssign = bulkAssign;
3040 return result;
3041 }
3042
3043 private synchronized void setAllRegionServersOffline(
3044 boolean allRegionServersOffline) {
3045 this.allRegionServersOffline = allRegionServersOffline;
3046 }
3047
3048 @Override
3049 protected void chore() {
3050
3051 if (this.bulkAssign) return;
3052 boolean allRSsOffline = this.serverManager.getOnlineServersList().
3053 isEmpty();
3054
3055 synchronized (regionsInTransition) {
3056
3057 long now = System.currentTimeMillis();
3058 for (RegionState regionState : regionsInTransition.values()) {
3059 if (regionState.getStamp() + timeout <= now) {
3060
3061 actOnTimeOut(regionState);
3062 } else if (this.allRegionServersOffline && !allRSsOffline) {
3063 RegionPlan existingPlan = regionPlans.get(regionState.getRegion().getEncodedName());
3064 if (existingPlan == null
3065 || !this.serverManager.isServerOnline(existingPlan.getDestination())) {
3066
3067
3068 actOnTimeOut(regionState);
3069 }
3070 }
3071 }
3072 }
3073 setAllRegionServersOffline(allRSsOffline);
3074 }
3075
3076 private void actOnTimeOut(RegionState regionState) {
3077 HRegionInfo regionInfo = regionState.getRegion();
3078 LOG.info("Regions in transition timed out: " + regionState);
3079
3080 switch (regionState.getState()) {
3081 case CLOSED:
3082 LOG.info("Region " + regionInfo.getEncodedName()
3083 + " has been CLOSED for too long, waiting on queued "
3084 + "ClosedRegionHandler to run or server shutdown");
3085
3086 regionState.updateTimestampToNow();
3087 break;
3088 case OFFLINE:
3089 LOG.info("Region has been OFFLINE for too long, " + "reassigning "
3090 + regionInfo.getRegionNameAsString() + " to a random server");
3091 invokeAssign(regionInfo);
3092 break;
3093 case PENDING_OPEN:
3094 LOG.info("Region has been PENDING_OPEN for too "
3095 + "long, reassigning region=" + regionInfo.getRegionNameAsString());
3096 invokeAssign(regionInfo);
3097 break;
3098 case OPENING:
3099 processOpeningState(regionInfo);
3100 break;
3101 case OPEN:
3102 LOG.error("Region has been OPEN for too long, " +
3103 "we don't know where region was opened so can't do anything");
3104 synchronized (regionState) {
3105 regionState.updateTimestampToNow();
3106 }
3107 break;
3108
3109 case PENDING_CLOSE:
3110 LOG.info("Region has been PENDING_CLOSE for too "
3111 + "long, running forced unassign again on region="
3112 + regionInfo.getRegionNameAsString());
3113 invokeUnassign(regionInfo);
3114 break;
3115 case CLOSING:
3116 LOG.info("Region has been CLOSING for too " +
3117 "long, this should eventually complete or the server will " +
3118 "expire, send RPC again");
3119 invokeUnassign(regionInfo);
3120 break;
3121 }
3122 }
3123 }
3124
3125 private void processOpeningState(HRegionInfo regionInfo) {
3126 LOG.info("Region has been OPENING for too " + "long, reassigning region="
3127 + regionInfo.getRegionNameAsString());
3128
3129 try {
3130 String node = ZKAssign.getNodeName(watcher, regionInfo.getEncodedName());
3131 Stat stat = new Stat();
3132 RegionTransitionData dataInZNode = ZKAssign.getDataNoWatch(watcher, node,
3133 stat);
3134 if (dataInZNode == null) {
3135 LOG.warn("Data is null, node " + node + " no longer exists");
3136 return;
3137 }
3138 if (dataInZNode.getEventType() == EventType.RS_ZK_REGION_OPENED) {
3139 LOG.debug("Region has transitioned to OPENED, allowing "
3140 + "watched event handlers to process");
3141 return;
3142 } else if (dataInZNode.getEventType() != EventType.RS_ZK_REGION_OPENING &&
3143 dataInZNode.getEventType() != EventType.RS_ZK_REGION_FAILED_OPEN ) {
3144 LOG.warn("While timing out a region in state OPENING, "
3145 + "found ZK node in unexpected state: "
3146 + dataInZNode.getEventType());
3147 return;
3148 }
3149 invokeAssign(regionInfo);
3150 } catch (KeeperException ke) {
3151 LOG.error("Unexpected ZK exception timing out CLOSING region", ke);
3152 return;
3153 }
3154 return;
3155 }
3156
3157 private void invokeAssign(HRegionInfo regionInfo) {
3158 threadPoolExecutorService.submit(new AssignCallable(this, regionInfo));
3159 }
3160
3161 private void invokeUnassign(HRegionInfo regionInfo) {
3162 threadPoolExecutorService.submit(new UnAssignCallable(this, regionInfo));
3163 }
3164
3165 public boolean isCarryingRoot(ServerName serverName) {
3166 return isCarryingRegion(serverName, HRegionInfo.ROOT_REGIONINFO);
3167 }
3168
3169 public boolean isCarryingMeta(ServerName serverName) {
3170 return isCarryingRegion(serverName, HRegionInfo.FIRST_META_REGIONINFO);
3171 }
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182 public boolean isCarryingRegion(ServerName serverName, HRegionInfo hri) {
3183 RegionTransitionData data = null;
3184 try {
3185 data = ZKAssign.getData(master.getZooKeeper(), hri.getEncodedName());
3186 } catch (KeeperException e) {
3187 master.abort("Unexpected ZK exception reading unassigned node for region="
3188 + hri.getEncodedName(), e);
3189 }
3190
3191 ServerName addressFromZK = (data != null && data.getOrigin() != null) ?
3192 data.getOrigin() : null;
3193 if (addressFromZK != null) {
3194
3195 boolean matchZK = (addressFromZK != null &&
3196 addressFromZK.equals(serverName));
3197 LOG.debug("based on ZK, current region=" + hri.getRegionNameAsString() +
3198 " is on server=" + addressFromZK +
3199 " server being checked=: " + serverName);
3200 return matchZK;
3201 }
3202
3203 ServerName addressFromAM = getRegionServerOfRegion(hri);
3204 boolean matchAM = (addressFromAM != null &&
3205 addressFromAM.equals(serverName));
3206 LOG.debug("based on AM, current region=" + hri.getRegionNameAsString() +
3207 " is on server=" + (addressFromAM != null ? addressFromAM : "null") +
3208 " server being checked: " + serverName);
3209
3210 return matchAM;
3211 }
3212
3213
3214
3215
3216
3217
3218
3219 public Pair<Set<HRegionInfo>, List<RegionState>> processServerShutdown(final ServerName sn) {
3220
3221 synchronized (this.regionPlans) {
3222 for (Iterator <Map.Entry<String, RegionPlan>> i =
3223 this.regionPlans.entrySet().iterator(); i.hasNext();) {
3224 Map.Entry<String, RegionPlan> e = i.next();
3225 ServerName otherSn = e.getValue().getDestination();
3226
3227 if (otherSn != null && otherSn.equals(sn)) {
3228
3229 i.remove();
3230 }
3231 }
3232 }
3233
3234
3235
3236 Set<HRegionInfo> deadRegions = new TreeSet<HRegionInfo>();
3237 synchronized (this.regions) {
3238 Set<HRegionInfo> assignedRegions = this.servers.remove(sn);
3239 if (assignedRegions != null && !assignedRegions.isEmpty()) {
3240 deadRegions.addAll(assignedRegions);
3241 for (HRegionInfo region : deadRegions) {
3242 this.regions.remove(region);
3243 }
3244 }
3245 }
3246
3247
3248
3249 Set<HRegionInfo> ritsGoingToServer = new ConcurrentSkipListSet<HRegionInfo>();
3250 List<RegionState> ritsOnServer = new ArrayList<RegionState>();
3251 synchronized (regionsInTransition) {
3252 for (RegionState state : this.regionsInTransition.values()) {
3253
3254
3255
3256
3257 if ((state.getServerName() != null) && state.getServerName().equals(sn)) {
3258 ritsGoingToServer.add(state.getRegion());
3259 }
3260 if (deadRegions.contains(state.getRegion())) {
3261 ritsOnServer.add(state);
3262 }
3263 }
3264 }
3265 return new Pair<Set<HRegionInfo>, List<RegionState>>(ritsGoingToServer, ritsOnServer);
3266 }
3267
3268
3269
3270
3271
3272
3273
3274
3275 public void handleSplitReport(final ServerName sn, final HRegionInfo parent,
3276 final HRegionInfo a, final HRegionInfo b) {
3277 regionOffline(parent);
3278 regionOnline(a, sn);
3279 regionOnline(b, sn);
3280
3281
3282
3283
3284
3285 if (this.zkTable.isDisablingOrDisabledTable(
3286 parent.getTableNameAsString())) {
3287 unassign(a);
3288 unassign(b);
3289 }
3290 }
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300 Map<String, Map<ServerName, List<HRegionInfo>>> getAssignmentsByTable() {
3301 Map<String, Map<ServerName, List<HRegionInfo>>> result = null;
3302 synchronized (this.regions) {
3303 result = new HashMap<String, Map<ServerName,List<HRegionInfo>>>();
3304 if (!this.master.getConfiguration().
3305 getBoolean("hbase.master.loadbalance.bytable", true)) {
3306 result.put("ensemble", getAssignments());
3307 } else {
3308 for (Map.Entry<ServerName, Set<HRegionInfo>> e: this.servers.entrySet()) {
3309 for (HRegionInfo hri : e.getValue()) {
3310 if (hri.isMetaRegion() || hri.isRootRegion()) continue;
3311 String tablename = hri.getTableNameAsString();
3312 Map<ServerName, List<HRegionInfo>> svrToRegions = result.get(tablename);
3313 if (svrToRegions == null) {
3314 svrToRegions = new HashMap<ServerName, List<HRegionInfo>>(this.servers.size());
3315 result.put(tablename, svrToRegions);
3316 }
3317 List<HRegionInfo> regions = null;
3318 if (!svrToRegions.containsKey(e.getKey())) {
3319 regions = new ArrayList<HRegionInfo>();
3320 svrToRegions.put(e.getKey(), regions);
3321 } else {
3322 regions = svrToRegions.get(e.getKey());
3323 }
3324 regions.add(hri);
3325 }
3326 }
3327 }
3328 }
3329 Map<ServerName, HServerLoad> onlineSvrs = this.serverManager.getOnlineServers();
3330
3331 for (Map<ServerName,List<HRegionInfo>> map : result.values()) {
3332 for (Map.Entry<ServerName, HServerLoad> svrEntry: onlineSvrs.entrySet()) {
3333 if (!map.containsKey(svrEntry.getKey())) {
3334 map.put(svrEntry.getKey(), new ArrayList<HRegionInfo>());
3335 }
3336 }
3337 }
3338 return result;
3339 }
3340
3341
3342
3343
3344
3345
3346 Map<ServerName, List<HRegionInfo>> getAssignments() {
3347
3348
3349
3350
3351 Map<ServerName, List<HRegionInfo>> result = null;
3352 synchronized (this.regions) {
3353 result = new HashMap<ServerName, List<HRegionInfo>>(this.servers.size());
3354 for (Map.Entry<ServerName, Set<HRegionInfo>> e: this.servers.entrySet()) {
3355 result.put(e.getKey(), new ArrayList<HRegionInfo>(e.getValue()));
3356 }
3357 }
3358 return result;
3359 }
3360
3361
3362
3363
3364
3365
3366 Pair<HRegionInfo, ServerName> getAssignment(final byte [] encodedRegionName) {
3367 String name = Bytes.toString(encodedRegionName);
3368 synchronized(this.regions) {
3369 for (Map.Entry<HRegionInfo, ServerName> e: this.regions.entrySet()) {
3370 if (e.getKey().getEncodedName().equals(name)) {
3371 return new Pair<HRegionInfo, ServerName>(e.getKey(), e.getValue());
3372 }
3373 }
3374 }
3375 return null;
3376 }
3377
3378
3379
3380
3381 void balance(final RegionPlan plan) {
3382 synchronized (this.regionPlans) {
3383 this.regionPlans.put(plan.getRegionName(), plan);
3384 }
3385 unassign(plan.getRegionInfo());
3386 }
3387
3388
3389
3390
3391 void unassignCatalogRegions() {
3392 synchronized (this.regions) {
3393 for (Map.Entry<ServerName, Set<HRegionInfo>> e: this.servers.entrySet()) {
3394 Set<HRegionInfo> regions = e.getValue();
3395 if (regions == null || regions.isEmpty()) continue;
3396 for (HRegionInfo hri: regions) {
3397 if (hri.isMetaRegion()) {
3398 unassign(hri);
3399 }
3400 }
3401 }
3402 }
3403 }
3404
3405
3406
3407
3408 public static class RegionState implements org.apache.hadoop.io.Writable {
3409 private HRegionInfo region;
3410
3411 public enum State {
3412 OFFLINE,
3413 PENDING_OPEN,
3414 OPENING,
3415 OPEN,
3416 PENDING_CLOSE,
3417 CLOSING,
3418 CLOSED,
3419 SPLITTING,
3420 SPLIT
3421 }
3422
3423 private State state;
3424
3425 private final AtomicLong stamp;
3426 private ServerName serverName;
3427
3428 public RegionState() {
3429 this.stamp = new AtomicLong(System.currentTimeMillis());
3430 }
3431
3432 RegionState(HRegionInfo region, State state) {
3433 this(region, state, System.currentTimeMillis(), null);
3434 }
3435
3436 RegionState(HRegionInfo region, State state, long stamp, ServerName serverName) {
3437 this.region = region;
3438 this.state = state;
3439 this.stamp = new AtomicLong(stamp);
3440 this.serverName = serverName;
3441 }
3442
3443 public void update(State state, long stamp, ServerName serverName) {
3444 this.state = state;
3445 updateTimestamp(stamp);
3446 this.serverName = serverName;
3447 }
3448
3449 public void update(State state) {
3450 this.state = state;
3451 updateTimestampToNow();
3452 this.serverName = null;
3453 }
3454
3455 public void updateTimestamp(long stamp) {
3456 this.stamp.set(stamp);
3457 }
3458
3459 public void updateTimestampToNow() {
3460 this.stamp.set(System.currentTimeMillis());
3461 }
3462
3463 public State getState() {
3464 return state;
3465 }
3466
3467 public long getStamp() {
3468 return stamp.get();
3469 }
3470
3471 public HRegionInfo getRegion() {
3472 return region;
3473 }
3474
3475 public ServerName getServerName() {
3476 return serverName;
3477 }
3478
3479 public boolean isClosing() {
3480 return state == State.CLOSING;
3481 }
3482
3483 public boolean isClosed() {
3484 return state == State.CLOSED;
3485 }
3486
3487 public boolean isPendingClose() {
3488 return state == State.PENDING_CLOSE;
3489 }
3490
3491 public boolean isOpening() {
3492 return state == State.OPENING;
3493 }
3494
3495 public boolean isOpened() {
3496 return state == State.OPEN;
3497 }
3498
3499 public boolean isPendingOpen() {
3500 return state == State.PENDING_OPEN;
3501 }
3502
3503 public boolean isOffline() {
3504 return state == State.OFFLINE;
3505 }
3506
3507 public boolean isSplitting() {
3508 return state == State.SPLITTING;
3509 }
3510
3511 public boolean isSplit() {
3512 return state == State.SPLIT;
3513 }
3514
3515 @Override
3516 public String toString() {
3517 return region.getRegionNameAsString()
3518 + " state=" + state
3519 + ", ts=" + stamp
3520 + ", server=" + serverName;
3521 }
3522
3523
3524
3525
3526 public String toDescriptiveString() {
3527 long lstamp = stamp.get();
3528 long relTime = System.currentTimeMillis() - lstamp;
3529
3530 return region.getRegionNameAsString()
3531 + " state=" + state
3532 + ", ts=" + new Date(lstamp) + " (" + (relTime/1000) + "s ago)"
3533 + ", server=" + serverName;
3534 }
3535
3536 @Override
3537 public void readFields(DataInput in) throws IOException {
3538 region = new HRegionInfo();
3539 region.readFields(in);
3540 state = State.valueOf(in.readUTF());
3541 stamp.set(in.readLong());
3542 }
3543
3544 @Override
3545 public void write(DataOutput out) throws IOException {
3546 region.write(out);
3547 out.writeUTF(state.name());
3548 out.writeLong(stamp.get());
3549 }
3550 }
3551
3552 public void stop() {
3553 this.timeoutMonitor.interrupt();
3554 this.timerUpdater.interrupt();
3555 }
3556
3557
3558
3559
3560
3561
3562 public boolean isServerOnline(ServerName serverName) {
3563 return this.serverManager.isServerOnline(serverName);
3564 }
3565
3566
3567
3568 public void shutdown() {
3569 if (null != threadPoolExecutorService) {
3570 this.threadPoolExecutorService.shutdown();
3571 }
3572 }
3573
3574 protected void setEnabledTable(String tableName) {
3575 try {
3576 this.zkTable.setEnabledTable(tableName);
3577 } catch (KeeperException e) {
3578
3579 String errorMsg = "Unable to ensure that the table " + tableName
3580 + " will be" + " enabled because of a ZooKeeper issue";
3581 LOG.error(errorMsg);
3582 this.master.abort(errorMsg, e);
3583 }
3584 }
3585
3586 }