1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.master;
21
22 import java.io.IOException;
23 import java.util.ArrayList;
24 import java.util.Collections;
25 import java.util.HashMap;
26 import java.util.HashSet;
27 import java.util.Iterator;
28 import java.util.List;
29 import java.util.Map;
30 import java.util.NavigableMap;
31 import java.util.Set;
32 import java.util.SortedMap;
33 import java.util.TreeMap;
34 import java.util.concurrent.ConcurrentSkipListMap;
35 import java.util.concurrent.atomic.AtomicInteger;
36 import java.util.concurrent.atomic.AtomicReference;
37
38 import org.apache.commons.logging.Log;
39 import org.apache.commons.logging.LogFactory;
40 import org.apache.hadoop.conf.Configuration;
41 import org.apache.hadoop.fs.FileStatus;
42 import org.apache.hadoop.fs.Path;
43 import org.apache.hadoop.fs.PathFilter;
44 import org.apache.hadoop.hbase.HConstants;
45 import org.apache.hadoop.hbase.HMsg;
46 import org.apache.hadoop.hbase.HRegionInfo;
47 import org.apache.hadoop.hbase.HServerAddress;
48 import org.apache.hadoop.hbase.HServerInfo;
49 import org.apache.hadoop.hbase.HServerLoad;
50 import org.apache.hadoop.hbase.client.Put;
51 import org.apache.hadoop.hbase.ipc.HRegionInterface;
52 import org.apache.hadoop.hbase.regionserver.HRegion;
53 import org.apache.hadoop.hbase.util.Bytes;
54 import org.apache.hadoop.hbase.util.Pair;
55 import org.apache.hadoop.hbase.util.Threads;
56 import org.apache.hadoop.hbase.util.Writables;
57 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
58
59
60
61
62 public class RegionManager {
63 protected static final Log LOG = LogFactory.getLog(RegionManager.class);
64
65 private AtomicReference<HServerAddress> rootRegionLocation =
66 new AtomicReference<HServerAddress>(null);
67
68 private final RootScanner rootScannerThread;
69 final MetaScanner metaScannerThread;
70
71
72 private final AtomicInteger numberOfMetaRegions = new AtomicInteger();
73
74
75 private final NavigableMap<byte [], MetaRegion> onlineMetaRegions =
76 new ConcurrentSkipListMap<byte [], MetaRegion>(Bytes.BYTES_COMPARATOR);
77
78 private static final byte[] OVERLOADED = Bytes.toBytes("Overloaded");
79
80 private static final byte [] META_REGION_PREFIX = Bytes.toBytes(".META.,");
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95 final SortedMap<String, RegionState> regionsInTransition =
96 Collections.synchronizedSortedMap(new TreeMap<String, RegionState>());
97
98
99 private final int maxAssignInOneGo;
100
101 final HMaster master;
102 private final LoadBalancer loadBalancer;
103
104
105 private final SortedMap<byte[], Pair<HRegionInfo,HServerAddress>>
106 regionsToSplit = Collections.synchronizedSortedMap(
107 new TreeMap<byte[],Pair<HRegionInfo,HServerAddress>>
108 (Bytes.BYTES_COMPARATOR));
109
110 private final SortedMap<byte[], Pair<HRegionInfo,HServerAddress>>
111 regionsToCompact = Collections.synchronizedSortedMap(
112 new TreeMap<byte[],Pair<HRegionInfo,HServerAddress>>
113 (Bytes.BYTES_COMPARATOR));
114
115 private final SortedMap<byte[], Pair<HRegionInfo,HServerAddress>>
116 regionsToMajorCompact = Collections.synchronizedSortedMap(
117 new TreeMap<byte[],Pair<HRegionInfo,HServerAddress>>
118 (Bytes.BYTES_COMPARATOR));
119
120 private final SortedMap<byte[], Pair<HRegionInfo,HServerAddress>>
121 regionsToFlush = Collections.synchronizedSortedMap(
122 new TreeMap<byte[],Pair<HRegionInfo,HServerAddress>>
123 (Bytes.BYTES_COMPARATOR));
124 private final int zooKeeperNumRetries;
125 private final int zooKeeperPause;
126
127 RegionManager(HMaster master) {
128 Configuration conf = master.getConfiguration();
129
130 this.master = master;
131 this.maxAssignInOneGo = conf.getInt("hbase.regions.percheckin", 10);
132 this.loadBalancer = new LoadBalancer(conf);
133
134
135 rootScannerThread = new RootScanner(master);
136
137
138 metaScannerThread = new MetaScanner(master);
139
140 zooKeeperNumRetries = conf.getInt(HConstants.ZOOKEEPER_RETRIES,
141 HConstants.DEFAULT_ZOOKEEPER_RETRIES);
142 zooKeeperPause = conf.getInt(HConstants.ZOOKEEPER_PAUSE,
143 HConstants.DEFAULT_ZOOKEEPER_PAUSE);
144
145 reassignRootRegion();
146 }
147
148 void start() {
149 Threads.setDaemonThreadRunning(rootScannerThread,
150 "RegionManager.rootScanner");
151 Threads.setDaemonThreadRunning(metaScannerThread,
152 "RegionManager.metaScanner");
153 }
154
155 void unsetRootRegion() {
156 synchronized (regionsInTransition) {
157 rootRegionLocation.set(null);
158 regionsInTransition.remove(
159 HRegionInfo.ROOT_REGIONINFO.getRegionNameAsString());
160 LOG.info("-ROOT- region unset (but not set to be reassigned)");
161 }
162 }
163
164 void reassignRootRegion() {
165 unsetRootRegion();
166 if (!master.getShutdownRequested().get()) {
167 synchronized (regionsInTransition) {
168 RegionState s = new RegionState(HRegionInfo.ROOT_REGIONINFO,
169 RegionState.State.UNASSIGNED);
170 regionsInTransition.put(
171 HRegionInfo.ROOT_REGIONINFO.getRegionNameAsString(), s);
172 LOG.info("ROOT inserted into regionsInTransition");
173 }
174 }
175 }
176
177
178
179
180
181
182
183
184
185
186 void assignRegions(HServerInfo info, HRegionInfo[] mostLoadedRegions,
187 ArrayList<HMsg> returnMsgs) {
188 HServerLoad thisServersLoad = info.getLoad();
189 boolean isSingleServer = this.master.numServers() == 1;
190
191
192
193 Set<RegionState> regionsToAssign =
194 regionsAwaitingAssignment(info.getServerAddress(), isSingleServer);
195 if (regionsToAssign.size() == 0) {
196
197 this.loadBalancer.loadBalancing(info, mostLoadedRegions, returnMsgs);
198 } else {
199
200 if (isSingleServer) {
201 assignRegionsToOneServer(regionsToAssign, info, returnMsgs);
202 } else {
203
204
205 assignRegionsToMultipleServers(thisServersLoad, regionsToAssign,
206 info, returnMsgs);
207 }
208 }
209 }
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224 private void assignRegionsToMultipleServers(final HServerLoad thisServersLoad,
225 final Set<RegionState> regionsToAssign, final HServerInfo info,
226 final ArrayList<HMsg> returnMsgs) {
227 boolean isMetaAssign = false;
228 for (RegionState s : regionsToAssign) {
229 if (s.getRegionInfo().isMetaRegion())
230 isMetaAssign = true;
231 }
232 int nRegionsToAssign = regionsToAssign.size();
233 int otherServersRegionsCount =
234 regionsToGiveOtherServers(nRegionsToAssign, thisServersLoad);
235 nRegionsToAssign -= otherServersRegionsCount;
236 if (nRegionsToAssign > 0 || isMetaAssign) {
237 LOG.debug("Assigning for " + info + ": total nregions to assign=" +
238 nRegionsToAssign + ", regions to give other servers than this=" +
239 otherServersRegionsCount + ", isMetaAssign=" + isMetaAssign);
240
241
242
243 HServerLoad heavierLoad = new HServerLoad();
244 int nservers = computeNextHeaviestLoad(thisServersLoad, heavierLoad);
245 int nregions = 0;
246
247 for (HServerLoad load = new HServerLoad(thisServersLoad);
248 load.compareTo(heavierLoad) <= 0 && nregions < nRegionsToAssign;
249 load.setNumberOfRegions(load.getNumberOfRegions() + 1), nregions++) {
250
251 }
252 if (nregions < nRegionsToAssign) {
253
254
255 if (nservers > 0) {
256
257
258 nregions = (int) Math.ceil((1.0 * nRegionsToAssign)/(1.0 * nservers));
259 } else {
260
261
262 nregions = (int) Math.ceil((1.0 * nRegionsToAssign)/
263 (1.0 * master.getServerManager().numServers()));
264 }
265 } else {
266
267 nregions = nRegionsToAssign;
268 }
269 LOG.debug("Assigning " + info + " " + nregions + " regions");
270 assignRegions(regionsToAssign, nregions, info, returnMsgs);
271 }
272 }
273
274
275
276
277
278
279
280
281 private void assignRegions(final Set<RegionState> regionsToAssign,
282 final int nregions, final HServerInfo info,
283 final ArrayList<HMsg> returnMsgs) {
284 int count = nregions;
285 if (count > this.maxAssignInOneGo) {
286 count = this.maxAssignInOneGo;
287 }
288 for (RegionState s: regionsToAssign) {
289 doRegionAssignment(s, info, returnMsgs);
290 if (--count <= 0) {
291 break;
292 }
293 }
294 }
295
296
297
298
299
300
301
302
303
304
305
306
307 private void assignRegionsToOneServer(final Set<RegionState> regionsToAssign,
308 final HServerInfo info, final ArrayList<HMsg> returnMsgs) {
309 for (RegionState s: regionsToAssign) {
310 doRegionAssignment(s, info, returnMsgs);
311 }
312 }
313
314
315
316
317
318
319
320 private void doRegionAssignment(final RegionState rs,
321 final HServerInfo sinfo, final ArrayList<HMsg> returnMsgs) {
322 String regionName = rs.getRegionInfo().getRegionNameAsString();
323 LOG.info("Assigning region " + regionName + " to " + sinfo.getServerName());
324 rs.setPendingOpen(sinfo.getServerName());
325 synchronized (this.regionsInTransition) {
326 this.regionsInTransition.put(regionName, rs);
327 }
328
329 returnMsgs.add(new HMsg(HMsg.Type.MSG_REGION_OPEN, rs.getRegionInfo()));
330 }
331
332
333
334
335
336
337
338 private int regionsToGiveOtherServers(final int numUnassignedRegions,
339 final HServerLoad thisServersLoad) {
340 SortedMap<HServerLoad, Set<String>> lightServers =
341 new TreeMap<HServerLoad, Set<String>>();
342 this.master.getLightServers(thisServersLoad, lightServers);
343
344
345
346
347 int nRegions = 0;
348 for (Map.Entry<HServerLoad, Set<String>> e: lightServers.entrySet()) {
349 HServerLoad lightLoad = new HServerLoad(e.getKey());
350 do {
351 lightLoad.setNumberOfRegions(lightLoad.getNumberOfRegions() + 1);
352 nRegions += 1;
353 } while (lightLoad.compareTo(thisServersLoad) <= 0
354 && nRegions < numUnassignedRegions);
355 nRegions *= e.getValue().size();
356 if (nRegions >= numUnassignedRegions) {
357 break;
358 }
359 }
360 return nRegions;
361 }
362
363
364
365
366
367
368
369
370 private Set<RegionState> regionsAwaitingAssignment(HServerAddress addr,
371 boolean isSingleServer) {
372
373 Set<RegionState> regionsToAssign = new HashSet<RegionState>();
374
375 boolean isMetaServer = isMetaServer(addr);
376 RegionState rootState = null;
377
378 synchronized (this.regionsInTransition) {
379 rootState = regionsInTransition.get(HRegionInfo.ROOT_REGIONINFO.getRegionNameAsString());
380 }
381 if (rootState != null && rootState.isUnassigned()) {
382
383
384
385
386
387 if (!isMetaServer || isSingleServer) {
388 regionsToAssign.add(rootState);
389 }
390 return regionsToAssign;
391 }
392
393
394
395 boolean reassigningMetas = numberOfMetaRegions.get() != onlineMetaRegions.size();
396 boolean isMetaOrRoot = isMetaServer || isRootServer(addr);
397 if (reassigningMetas && isMetaOrRoot && !isSingleServer) {
398 return regionsToAssign;
399 }
400 synchronized (this.regionsInTransition) {
401 for (RegionState s: regionsInTransition.values()) {
402 HRegionInfo i = s.getRegionInfo();
403 if (i == null) {
404 continue;
405 }
406 if (reassigningMetas &&
407 !i.isMetaRegion()) {
408
409
410 continue;
411 }
412 if (!i.isMetaRegion() &&
413 !master.getServerManager().canAssignUserRegions()) {
414 LOG.debug("user region " + i.getRegionNameAsString() +
415 " is in transition but not enough servers yet");
416 continue;
417 }
418 if (s.isUnassigned()) {
419 regionsToAssign.add(s);
420 }
421 }
422 }
423 return regionsToAssign;
424 }
425
426
427
428
429
430 private int computeNextHeaviestLoad(HServerLoad referenceLoad,
431 HServerLoad heavierLoad) {
432
433 SortedMap<HServerLoad, Set<String>> heavyServers =
434 new TreeMap<HServerLoad, Set<String>>();
435 synchronized (master.getLoadToServers()) {
436 heavyServers.putAll(
437 master.getLoadToServers().tailMap(referenceLoad));
438 }
439 int nservers = 0;
440 for (Map.Entry<HServerLoad, Set<String>> e : heavyServers.entrySet()) {
441 Set<String> servers = e.getValue();
442 nservers += servers.size();
443 if (e.getKey().compareTo(referenceLoad) == 0) {
444
445 nservers -= 1;
446 continue;
447 }
448
449
450
451 heavierLoad.setNumberOfRequests(e.getKey().getNumberOfRequests());
452 heavierLoad.setNumberOfRegions(e.getKey().getNumberOfRegions());
453 break;
454 }
455 return nservers;
456 }
457
458
459
460
461
462
463
464
465
466 void unassignSomeRegions(final HServerInfo info,
467 int numRegionsToClose, final HRegionInfo[] mostLoadedRegions,
468 ArrayList<HMsg> returnMsgs) {
469 LOG.debug("Unassigning " + numRegionsToClose + " regions from " +
470 info.getServerName());
471 int regionIdx = 0;
472 int regionsClosed = 0;
473 int skipped = 0;
474 while (regionsClosed < numRegionsToClose &&
475 regionIdx < mostLoadedRegions.length) {
476 HRegionInfo currentRegion = mostLoadedRegions[regionIdx];
477 regionIdx++;
478
479 if (currentRegion.isRootRegion() || currentRegion.isMetaTable()) {
480 continue;
481 }
482 final String regionName = currentRegion.getRegionNameAsString();
483 if (regionIsInTransition(regionName)) {
484 skipped++;
485 continue;
486 }
487 if (LOG.isDebugEnabled()) {
488 LOG.debug("Going to close region " + regionName);
489 }
490
491 returnMsgs.add(new HMsg(HMsg.Type.MSG_REGION_CLOSE, currentRegion,
492 OVERLOADED));
493
494 setClosing(info.getServerName(), currentRegion, false);
495 setPendingClose(regionName);
496
497 regionsClosed++;
498 }
499 LOG.info("Skipped assigning " + skipped + " region(s) to " +
500 info.getServerName() + "because already in transition");
501 }
502
503
504
505
506 static class TableDirFilter implements PathFilter {
507 public boolean accept(final Path path) {
508
509
510
511 final String pathname = path.getName();
512 return (!pathname.equals(HConstants.HREGION_LOGDIR_NAME)
513 && !pathname.equals(HConstants.VERSION_FILE_NAME));
514 }
515
516 }
517
518
519
520
521 static class RegionDirFilter implements PathFilter {
522 public boolean accept(Path path) {
523 return !path.getName().equals(HConstants.HREGION_COMPACTIONDIR_NAME);
524 }
525 }
526
527
528
529
530
531
532
533 public int countRegionsOnFS() throws IOException {
534 int regions = 0;
535 FileStatus [] tableDirs =
536 this.master.getFileSystem().listStatus(this.master.getRootDir(), new TableDirFilter());
537 FileStatus[] regionDirs;
538 RegionDirFilter rdf = new RegionDirFilter();
539 for(FileStatus tabledir : tableDirs) {
540 if(tabledir.isDir()) {
541 regionDirs = this.master.getFileSystem().listStatus(tabledir.getPath(), rdf);
542 regions += regionDirs.length;
543 }
544 }
545 return regions;
546 }
547
548
549
550
551 public Map<byte [], MetaRegion> getOnlineMetaRegions() {
552 synchronized (onlineMetaRegions) {
553 return Collections.unmodifiableMap(onlineMetaRegions);
554 }
555 }
556
557 public boolean metaRegionsInTransition() {
558 synchronized (onlineMetaRegions) {
559 for (MetaRegion metaRegion : onlineMetaRegions.values()) {
560 String regionName = Bytes.toString(metaRegion.getRegionName());
561 if (regionIsInTransition(regionName)) {
562 return true;
563 }
564 }
565 }
566 return false;
567 }
568
569
570
571
572
573 Map<String, RegionState> getRegionsInTransitionOnServer(String serverName) {
574 Map<String, RegionState> ret = new HashMap<String, RegionState>();
575 synchronized (regionsInTransition) {
576 for (Map.Entry<String, RegionState> entry : regionsInTransition.entrySet()) {
577 RegionState rs = entry.getValue();
578 if (serverName.equals(rs.getServerName())) {
579 ret.put(entry.getKey(), rs);
580 }
581 }
582 }
583 return ret;
584 }
585
586
587
588
589
590 public void stopScanners() {
591 this.rootScannerThread.interruptAndStop();
592 this.metaScannerThread.interruptAndStop();
593 }
594
595
596 public void stop() {
597 try {
598 if (rootScannerThread.isAlive()) {
599 rootScannerThread.join();
600 }
601 } catch (Exception iex) {
602 LOG.warn("root scanner", iex);
603 }
604 try {
605 if (metaScannerThread.isAlive()) {
606 metaScannerThread.join();
607 }
608 } catch(Exception iex) {
609 LOG.warn("meta scanner", iex);
610 }
611 master.getZooKeeperWrapper().clearRSDirectory();
612 master.getZooKeeperWrapper().close();
613 }
614
615
616
617
618
619 public boolean areAllMetaRegionsOnline() {
620 synchronized (onlineMetaRegions) {
621 return (rootRegionLocation.get() != null &&
622 numberOfMetaRegions.get() == onlineMetaRegions.size());
623 }
624 }
625
626
627
628
629
630
631
632 public MetaRegion getFirstMetaRegionForRegion(HRegionInfo newRegion) {
633 synchronized (onlineMetaRegions) {
634 if (onlineMetaRegions.size() == 0) {
635 return null;
636 } else if (onlineMetaRegions.size() == 1) {
637 return onlineMetaRegions.get(onlineMetaRegions.firstKey());
638 } else {
639 if (onlineMetaRegions.containsKey(newRegion.getRegionName())) {
640 return onlineMetaRegions.get(newRegion.getRegionName());
641 }
642 return onlineMetaRegions.get(onlineMetaRegions.headMap(
643 newRegion.getRegionName()).lastKey());
644 }
645 }
646 }
647
648
649
650
651
652
653
654 public Set<MetaRegion> getMetaRegionsForTable(byte [] tableName)
655 throws NotAllMetaRegionsOnlineException {
656 byte [] firstMetaRegion = null;
657 Set<MetaRegion> metaRegions = new HashSet<MetaRegion>();
658 if (Bytes.equals(tableName, HConstants.META_TABLE_NAME)) {
659 if (rootRegionLocation.get() == null) {
660 throw new NotAllMetaRegionsOnlineException(
661 Bytes.toString(HConstants.ROOT_TABLE_NAME));
662 }
663 metaRegions.add(new MetaRegion(rootRegionLocation.get(),
664 HRegionInfo.ROOT_REGIONINFO));
665 } else {
666 if (!areAllMetaRegionsOnline()) {
667 throw new NotAllMetaRegionsOnlineException();
668 }
669 synchronized (onlineMetaRegions) {
670 if (onlineMetaRegions.size() == 1) {
671 firstMetaRegion = onlineMetaRegions.firstKey();
672 } else if (onlineMetaRegions.containsKey(tableName)) {
673 firstMetaRegion = tableName;
674 } else {
675 firstMetaRegion = onlineMetaRegions.headMap(tableName).lastKey();
676 }
677 metaRegions.addAll(onlineMetaRegions.tailMap(firstMetaRegion).values());
678 }
679 }
680 return metaRegions;
681 }
682
683
684
685
686
687
688
689 public MetaRegion getMetaRegionForRow(final byte [] row)
690 throws NotAllMetaRegionsOnlineException {
691 if (!areAllMetaRegionsOnline()) {
692 throw new NotAllMetaRegionsOnlineException();
693 }
694
695 int prefixlen = META_REGION_PREFIX.length;
696 if (row.length > prefixlen &&
697 Bytes.compareTo(META_REGION_PREFIX, 0, prefixlen, row, 0, prefixlen) == 0) {
698 return new MetaRegion(this.master.getRegionManager().getRootRegionLocation(),
699 HRegionInfo.ROOT_REGIONINFO);
700 }
701 return this.onlineMetaRegions.floorEntry(row).getValue();
702 }
703
704
705
706
707
708
709
710
711
712
713
714 public void createRegion(HRegionInfo newRegion, HRegionInterface server,
715 byte [] metaRegionName)
716 throws IOException {
717
718 HRegion region = HRegion.createHRegion(newRegion, this.master.getRootDir(),
719 master.getConfiguration());
720
721
722 HRegionInfo info = region.getRegionInfo();
723 byte [] regionName = region.getRegionName();
724
725 Put put = new Put(regionName);
726 put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
727 Writables.getBytes(info));
728 server.put(metaRegionName, put);
729
730
731 region.close();
732 region.getLog().closeAndDelete();
733
734
735 setUnassigned(info, true);
736 }
737
738
739
740
741
742 public void putMetaRegionOnline(MetaRegion metaRegion) {
743 onlineMetaRegions.put(metaRegion.getStartKey(), metaRegion);
744 }
745
746
747
748
749
750 public List<MetaRegion> getListOfOnlineMetaRegions() {
751 List<MetaRegion> regions;
752 synchronized(onlineMetaRegions) {
753 regions = new ArrayList<MetaRegion>(onlineMetaRegions.values());
754 }
755 return regions;
756 }
757
758
759
760
761
762 public int numOnlineMetaRegions() {
763 return onlineMetaRegions.size();
764 }
765
766
767
768
769
770
771 public boolean isMetaRegionOnline(byte [] startKey) {
772 return onlineMetaRegions.containsKey(startKey);
773 }
774
775
776
777
778
779
780 public MetaRegion offlineMetaRegionWithStartKey(byte [] startKey) {
781 LOG.info("META region whose startkey is " + Bytes.toString(startKey) +
782 " removed from onlineMetaRegions");
783 return onlineMetaRegions.remove(startKey);
784 }
785
786 public boolean isRootServer(HServerAddress server) {
787 return this.master.getRegionManager().getRootRegionLocation() != null &&
788 server.equals(master.getRegionManager().getRootRegionLocation());
789 }
790
791
792
793
794
795
796
797
798 public List<byte[]> listMetaRegionsForServer(HServerAddress server) {
799 List<byte[]> metas = new ArrayList<byte[]>();
800 for ( MetaRegion region : onlineMetaRegions.values() ) {
801 if (server.equals(region.getServer())) {
802 metas.add(region.getStartKey());
803 }
804 }
805 return metas;
806 }
807
808
809
810
811
812
813
814
815 public boolean isMetaServer(HServerAddress server) {
816 for ( MetaRegion region : onlineMetaRegions.values() ) {
817 if (server.equals(region.getServer())) {
818 return true;
819 }
820 }
821
822
823
824 synchronized(regionsInTransition) {
825 for (RegionState s : regionsInTransition.values()) {
826 if (s.getRegionInfo().isMetaRegion()
827 && !s.isUnassigned()
828 && s.getServerName() != null
829 && s.getServerName().equals(server.toString())) {
830
831
832
833 LOG.fatal("I DONT BELIEVE YOU WILL EVER SEE THIS!");
834
835 return true;
836 }
837 }
838 }
839 return false;
840 }
841
842
843
844
845
846
847
848 public boolean isRootInTransitionOnThisServer(final String server) {
849 synchronized (this.regionsInTransition) {
850 for (RegionState s : regionsInTransition.values()) {
851 if (s.getRegionInfo().isRootRegion()
852 && !s.isUnassigned()
853 && s.getServerName() != null
854 && s.getServerName().equals(server)) {
855
856 return true;
857 }
858 }
859 }
860 return false;
861 }
862
863
864
865
866
867
868
869 public HRegionInfo getMetaServerRegionInfo(final String server) {
870 synchronized (this.regionsInTransition) {
871 for (RegionState s : regionsInTransition.values()) {
872 if (s.getRegionInfo().isMetaRegion()
873 && !s.isUnassigned()
874 && s.getServerName() != null
875 && s.getServerName().equals(server)) {
876
877 return s.getRegionInfo();
878 }
879 }
880 }
881 return null;
882 }
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897 public synchronized boolean offlineMetaServer(HServerAddress server) {
898 boolean hasMeta = false;
899
900
901
902 if (master.getRegionManager().getRootRegionLocation() != null &&
903 server.equals(master.getRegionManager().getRootRegionLocation())) {
904 LOG.info("Offlined ROOT server: " + server);
905 reassignRootRegion();
906 hasMeta = true;
907 }
908
909 for ( MetaRegion region : onlineMetaRegions.values() ) {
910 if (server.equals(region.getServer())) {
911 LOG.info("Offlining META region: " + region);
912 offlineMetaRegionWithStartKey(region.getStartKey());
913
914 setUnassigned(region.getRegionInfo(), true);
915 hasMeta = true;
916 }
917 }
918 return hasMeta;
919 }
920
921
922
923
924
925
926 public void removeRegion(HRegionInfo info) {
927 synchronized (this.regionsInTransition) {
928 this.regionsInTransition.remove(info.getRegionNameAsString());
929 }
930 }
931
932
933
934
935
936 public boolean regionIsInTransition(String regionName) {
937 synchronized (this.regionsInTransition) {
938 return regionsInTransition.containsKey(regionName);
939 }
940 }
941
942
943
944
945
946 public boolean regionIsOpening(String regionName) {
947 synchronized (this.regionsInTransition) {
948 RegionState state = regionsInTransition.get(regionName);
949 if (state != null) {
950 return state.isOpening();
951 }
952 }
953 return false;
954 }
955
956
957
958
959
960
961 public void setUnassigned(HRegionInfo info, boolean force) {
962 RegionState s = null;
963 synchronized(this.regionsInTransition) {
964 s = regionsInTransition.get(info.getRegionNameAsString());
965 if (s == null) {
966 s = new RegionState(info, RegionState.State.UNASSIGNED);
967 regionsInTransition.put(info.getRegionNameAsString(), s);
968 }
969 }
970 if (force || (!s.isPendingOpen() && !s.isOpen())) {
971 s.setUnassigned();
972 }
973 }
974
975
976
977
978
979
980
981
982 public boolean isUnassigned(HRegionInfo info) {
983 synchronized (regionsInTransition) {
984 RegionState s = regionsInTransition.get(info.getRegionNameAsString());
985 if (s != null) {
986 return s.isUnassigned();
987 }
988 }
989 return false;
990 }
991
992
993
994
995
996
997
998
999 public boolean isPendingOpen(String regionName) {
1000 synchronized (regionsInTransition) {
1001 RegionState s = regionsInTransition.get(regionName);
1002 if (s != null) {
1003 return s.isPendingOpen();
1004 }
1005 }
1006 return false;
1007 }
1008
1009
1010
1011
1012
1013 public void setOpen(String regionName) {
1014 synchronized (regionsInTransition) {
1015 RegionState s = regionsInTransition.get(regionName);
1016 if (s != null) {
1017 s.setOpen();
1018 }
1019 }
1020 }
1021
1022
1023
1024
1025
1026 public boolean isOfflined(String regionName) {
1027 synchronized (regionsInTransition) {
1028 RegionState s = regionsInTransition.get(regionName);
1029 if (s != null) {
1030 return s.isOfflined();
1031 }
1032 }
1033 return false;
1034 }
1035
1036
1037
1038
1039
1040
1041
1042 public void setClosing(String serverName, final HRegionInfo regionInfo,
1043 final boolean setOffline) {
1044 synchronized (this.regionsInTransition) {
1045 RegionState s =
1046 this.regionsInTransition.get(regionInfo.getRegionNameAsString());
1047 if (s == null) {
1048 s = new RegionState(regionInfo, RegionState.State.CLOSING);
1049 }
1050
1051
1052 if(s.isPendingOpen()) {
1053 serverName = s.getServerName();
1054 }
1055 s.setClosing(serverName, setOffline);
1056 this.regionsInTransition.put(regionInfo.getRegionNameAsString(), s);
1057 }
1058 }
1059
1060
1061
1062
1063
1064
1065
1066
1067 public Set<HRegionInfo> getMarkedToClose(String serverName) {
1068 Set<HRegionInfo> result = new HashSet<HRegionInfo>();
1069 synchronized (regionsInTransition) {
1070 for (RegionState s: regionsInTransition.values()) {
1071 if (s.isClosing() && !s.isPendingClose() && !s.isClosed() &&
1072 s.getServerName().compareTo(serverName) == 0) {
1073 result.add(s.getRegionInfo());
1074 }
1075 }
1076 }
1077 return result;
1078 }
1079
1080
1081
1082
1083
1084
1085 public void setPendingClose(String regionName) {
1086 synchronized (regionsInTransition) {
1087 RegionState s = regionsInTransition.get(regionName);
1088 if (s != null) {
1089 s.setPendingClose();
1090 }
1091 }
1092 }
1093
1094
1095
1096
1097 public void setClosed(String regionName) {
1098 synchronized (regionsInTransition) {
1099 RegionState s = regionsInTransition.get(regionName);
1100 if (s != null) {
1101 s.setClosed();
1102 }
1103 }
1104 }
1105
1106
1107
1108
1109 public void addMetaRegionToScan(MetaRegion m) {
1110 metaScannerThread.addMetaRegionToScan(m);
1111 }
1112
1113
1114
1115
1116
1117 public boolean isInitialRootScanComplete() {
1118 return rootScannerThread.isInitialScanComplete();
1119 }
1120
1121
1122
1123
1124
1125 public boolean isInitialMetaScanComplete() {
1126 return metaScannerThread.isInitialScanComplete();
1127 }
1128
1129
1130
1131
1132
1133 public HServerAddress getRootRegionLocation() {
1134 return rootRegionLocation.get();
1135 }
1136
1137
1138
1139
1140
1141 public void waitForRootRegionLocation() {
1142 synchronized (rootRegionLocation) {
1143 while (!master.getShutdownRequested().get() &&
1144 !master.isClosed() && rootRegionLocation.get() == null) {
1145
1146
1147
1148 try {
1149
1150 rootRegionLocation.wait(this.master.getThreadWakeFrequency());
1151 } catch (InterruptedException e) {
1152
1153 }
1154 }
1155 }
1156 }
1157
1158
1159
1160
1161
1162 public int numMetaRegions() {
1163 return numberOfMetaRegions.get();
1164 }
1165
1166
1167
1168
1169 public void incrementNumMetaRegions() {
1170 numberOfMetaRegions.incrementAndGet();
1171 }
1172
1173 private long getPauseTime(int tries) {
1174 int attempt = tries;
1175 if (attempt >= HConstants.RETRY_BACKOFF.length) {
1176 attempt = HConstants.RETRY_BACKOFF.length - 1;
1177 }
1178 return this.zooKeeperPause * HConstants.RETRY_BACKOFF[attempt];
1179 }
1180
1181 private void sleep(int attempt) {
1182 try {
1183 Thread.sleep(getPauseTime(attempt));
1184 } catch (InterruptedException e) {
1185
1186 }
1187 }
1188
1189 private void writeRootRegionLocationToZooKeeper(HServerAddress address) {
1190 for (int attempt = 0; attempt < zooKeeperNumRetries; ++attempt) {
1191 if (master.getZooKeeperWrapper().writeRootRegionLocation(address)) {
1192 return;
1193 }
1194
1195 sleep(attempt);
1196 }
1197
1198 LOG.error("Failed to write root region location to ZooKeeper after " +
1199 zooKeeperNumRetries + " retries, shutting down");
1200
1201 this.master.shutdown();
1202 }
1203
1204
1205
1206
1207
1208 public void setRootRegionLocation(HServerAddress address) {
1209 writeRootRegionLocationToZooKeeper(address);
1210 synchronized (rootRegionLocation) {
1211 rootRegionLocation.set(new HServerAddress(address));
1212 rootRegionLocation.notifyAll();
1213 }
1214 }
1215
1216
1217
1218
1219
1220 public void setNumMetaRegions(int num) {
1221 numberOfMetaRegions.set(num);
1222 }
1223
1224
1225
1226
1227
1228
1229
1230 public void startAction(byte[] regionName, HRegionInfo info,
1231 HServerAddress server, HConstants.Modify op) {
1232 if (LOG.isDebugEnabled()) {
1233 LOG.debug("Adding operation " + op + " from tasklist");
1234 }
1235 switch (op) {
1236 case TABLE_SPLIT:
1237 startAction(regionName, info, server, this.regionsToSplit);
1238 break;
1239 case TABLE_COMPACT:
1240 startAction(regionName, info, server, this.regionsToCompact);
1241 break;
1242 case TABLE_MAJOR_COMPACT:
1243 startAction(regionName, info, server, this.regionsToMajorCompact);
1244 break;
1245 case TABLE_FLUSH:
1246 startAction(regionName, info, server, this.regionsToFlush);
1247 break;
1248 default:
1249 throw new IllegalArgumentException("illegal table action " + op);
1250 }
1251 }
1252
1253 private void startAction(final byte[] regionName, final HRegionInfo info,
1254 final HServerAddress server,
1255 final SortedMap<byte[], Pair<HRegionInfo,HServerAddress>> map) {
1256 map.put(regionName, new Pair<HRegionInfo,HServerAddress>(info, server));
1257 }
1258
1259
1260
1261
1262
1263 public void endAction(byte[] regionName, HConstants.Modify op) {
1264 if (LOG.isDebugEnabled()) {
1265 LOG.debug("Removing operation " + op + " from tasklist");
1266 }
1267 switch (op) {
1268 case TABLE_SPLIT:
1269 this.regionsToSplit.remove(regionName);
1270 break;
1271 case TABLE_COMPACT:
1272 this.regionsToCompact.remove(regionName);
1273 break;
1274 case TABLE_MAJOR_COMPACT:
1275 this.regionsToMajorCompact.remove(regionName);
1276 break;
1277 case TABLE_FLUSH:
1278 this.regionsToFlush.remove(regionName);
1279 break;
1280 default:
1281 throw new IllegalArgumentException("illegal table action " + op);
1282 }
1283 }
1284
1285
1286
1287
1288 public void endActions(byte[] regionName) {
1289 regionsToSplit.remove(regionName);
1290 regionsToCompact.remove(regionName);
1291 }
1292
1293
1294
1295
1296
1297
1298
1299 public void applyActions(HServerInfo serverInfo, ArrayList<HMsg> returnMsgs) {
1300 applyActions(serverInfo, returnMsgs, this.regionsToCompact,
1301 HMsg.Type.MSG_REGION_COMPACT);
1302 applyActions(serverInfo, returnMsgs, this.regionsToSplit,
1303 HMsg.Type.MSG_REGION_SPLIT);
1304 applyActions(serverInfo, returnMsgs, this.regionsToFlush,
1305 HMsg.Type.MSG_REGION_FLUSH);
1306 applyActions(serverInfo, returnMsgs, this.regionsToMajorCompact,
1307 HMsg.Type.MSG_REGION_MAJOR_COMPACT);
1308 }
1309
1310 private void applyActions(final HServerInfo serverInfo,
1311 final ArrayList<HMsg> returnMsgs,
1312 final SortedMap<byte[], Pair<HRegionInfo,HServerAddress>> map,
1313 final HMsg.Type msg) {
1314 HServerAddress addr = serverInfo.getServerAddress();
1315 synchronized (map) {
1316 Iterator<Pair<HRegionInfo, HServerAddress>> i = map.values().iterator();
1317 while (i.hasNext()) {
1318 Pair<HRegionInfo,HServerAddress> pair = i.next();
1319 if (addr.equals(pair.getSecond())) {
1320 if (LOG.isDebugEnabled()) {
1321 LOG.debug("Sending " + msg + " " + pair.getFirst() + " to " + addr);
1322 }
1323 returnMsgs.add(new HMsg(msg, pair.getFirst()));
1324 i.remove();
1325 }
1326 }
1327 }
1328 }
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340 private class LoadBalancer {
1341 private float slop;
1342 private final int maxRegToClose;
1343
1344 LoadBalancer(Configuration conf) {
1345 this.slop = conf.getFloat("hbase.regions.slop", (float)0.3);
1346 if (this.slop <= 0) this.slop = 1;
1347
1348
1349
1350 this.maxRegToClose = conf.getInt("hbase.regions.close.max", -1);
1351 }
1352
1353
1354
1355
1356
1357
1358
1359
1360 void loadBalancing(HServerInfo info, HRegionInfo[] mostLoadedRegions,
1361 ArrayList<HMsg> returnMsgs) {
1362 HServerLoad servLoad = info.getLoad();
1363 double avg = master.getAverageLoad();
1364
1365
1366 if(servLoad.getLoad() <= Math.ceil(avg) || avg <= 2.0) {
1367 return;
1368 }
1369
1370
1371 int numRegionsToClose = balanceFromOverloaded(info.getServerName(),
1372 servLoad, avg);
1373
1374
1375 if(numRegionsToClose <= 0) {
1376 numRegionsToClose = balanceToLowloaded(info.getServerName(), servLoad,
1377 avg);
1378 }
1379
1380 if(maxRegToClose > 0) {
1381 numRegionsToClose = Math.min(numRegionsToClose, maxRegToClose);
1382 }
1383
1384 if(numRegionsToClose > 0) {
1385 unassignSomeRegions(info, numRegionsToClose, mostLoadedRegions,
1386 returnMsgs);
1387 }
1388 }
1389
1390
1391
1392
1393
1394 private int balanceFromOverloaded(final String serverName,
1395 HServerLoad srvLoad, double avgLoad) {
1396 int avgLoadPlusSlop = (int)Math.ceil(avgLoad * (1 + this.slop));
1397 int numSrvRegs = srvLoad.getNumberOfRegions();
1398 if (numSrvRegs > avgLoadPlusSlop) {
1399 if (LOG.isDebugEnabled()) {
1400 LOG.debug("Server " + serverName + " is carrying more than its fair " +
1401 "share of regions: " +
1402 "load=" + numSrvRegs + ", avg=" + avgLoad + ", slop=" + this.slop);
1403 }
1404 return numSrvRegs - (int)Math.ceil(avgLoad);
1405 }
1406 return 0;
1407 }
1408
1409
1410
1411
1412
1413
1414 private int balanceToLowloaded(String srvName, HServerLoad srvLoad,
1415 double avgLoad) {
1416
1417 SortedMap<HServerLoad, Set<String>> loadToServers =
1418 master.getLoadToServers();
1419
1420 if (!loadToServers.get(loadToServers.lastKey()).contains(srvName))
1421 return 0;
1422
1423
1424
1425 int avgLoadMinusSlop = (int)Math.floor(avgLoad * (1 - this.slop)) - 1;
1426 int lowestLoad = loadToServers.firstKey().getNumberOfRegions();
1427
1428 if(lowestLoad >= avgLoadMinusSlop)
1429 return 0;
1430
1431 int lowSrvCount = loadToServers.get(loadToServers.firstKey()).size();
1432 int numRegionsToClose = 0;
1433
1434 int numSrvRegs = srvLoad.getNumberOfRegions();
1435 int numMoveToLowLoaded = (avgLoadMinusSlop - lowestLoad) * lowSrvCount;
1436 numRegionsToClose = numSrvRegs - (int)Math.ceil(avgLoad);
1437 numRegionsToClose = Math.min(numRegionsToClose, numMoveToLowLoaded);
1438 if (LOG.isDebugEnabled()) {
1439 LOG.debug("Server(s) are carrying only " + lowestLoad + " regions. " +
1440 "Server " + srvName + " is most loaded (" + numSrvRegs +
1441 "). Shedding " + numRegionsToClose + " regions to pass to " +
1442 " least loaded (numMoveToLowLoaded=" + numMoveToLowLoaded +")");
1443 }
1444 return numRegionsToClose;
1445 }
1446 }
1447
1448
1449
1450
1451 NavigableMap<String, String> getRegionsInTransition() {
1452 NavigableMap<String, String> result = new TreeMap<String, String>();
1453 synchronized (this.regionsInTransition) {
1454 if (this.regionsInTransition.isEmpty()) return result;
1455 for (Map.Entry<String, RegionState> e: this.regionsInTransition.entrySet()) {
1456 result.put(e.getKey(), e.getValue().toString());
1457 }
1458 }
1459 return result;
1460 }
1461
1462
1463
1464
1465
1466 boolean clearFromInTransition(final byte [] regionname) {
1467 boolean result = false;
1468 synchronized (this.regionsInTransition) {
1469 if (this.regionsInTransition.isEmpty()) return result;
1470 for (Map.Entry<String, RegionState> e: this.regionsInTransition.entrySet()) {
1471 if (Bytes.equals(regionname, e.getValue().getRegionName())) {
1472 this.regionsInTransition.remove(e.getKey());
1473 LOG.debug("Removed " + e.getKey() + ", " + e.getValue());
1474 result = true;
1475 break;
1476 }
1477 }
1478 }
1479 return result;
1480 }
1481
1482
1483
1484
1485
1486
1487 static class RegionState implements Comparable<RegionState> {
1488 private final HRegionInfo regionInfo;
1489
1490 enum State {
1491 UNASSIGNED,
1492 PENDING_OPEN,
1493 OPEN,
1494 CLOSING,
1495 PENDING_CLOSE,
1496 CLOSED
1497
1498 }
1499
1500 private State state;
1501
1502 private boolean isOfflined;
1503
1504
1505 private String serverName = null;
1506
1507
1508 RegionState(HRegionInfo info, State state) {
1509 this.regionInfo = info;
1510 this.state = state;
1511 }
1512
1513 synchronized HRegionInfo getRegionInfo() {
1514 return this.regionInfo;
1515 }
1516
1517 synchronized byte [] getRegionName() {
1518 return this.regionInfo.getRegionName();
1519 }
1520
1521
1522
1523
1524 synchronized String getServerName() {
1525 return this.serverName;
1526 }
1527
1528
1529
1530
1531 synchronized boolean isOpening() {
1532 return state == State.UNASSIGNED ||
1533 state == State.PENDING_OPEN ||
1534 state == State.OPEN;
1535 }
1536
1537
1538
1539
1540 synchronized boolean isUnassigned() {
1541 return state == State.UNASSIGNED;
1542 }
1543
1544
1545
1546
1547
1548
1549 synchronized void setUnassigned() {
1550 state = State.UNASSIGNED;
1551 this.serverName = null;
1552 }
1553
1554 synchronized boolean isPendingOpen() {
1555 return state == State.PENDING_OPEN;
1556 }
1557
1558
1559
1560
1561 synchronized void setPendingOpen(final String serverName) {
1562 if (state != State.UNASSIGNED) {
1563 LOG.warn("Cannot assign a region that is not currently unassigned. " +
1564 "FIX!! State: " + toString());
1565 }
1566 state = State.PENDING_OPEN;
1567 this.serverName = serverName;
1568 }
1569
1570 synchronized boolean isOpen() {
1571 return state == State.OPEN;
1572 }
1573
1574 synchronized void setOpen() {
1575 if (state != State.PENDING_OPEN) {
1576 LOG.warn("Cannot set a region as open if it has not been pending. " +
1577 "FIX!! State: " + toString());
1578 }
1579 state = State.OPEN;
1580 }
1581
1582 synchronized boolean isClosing() {
1583 return state == State.CLOSING;
1584 }
1585
1586 synchronized void setClosing(String serverName, boolean setOffline) {
1587 state = State.CLOSING;
1588 this.serverName = serverName;
1589 this.isOfflined = setOffline;
1590 }
1591
1592 synchronized boolean isPendingClose() {
1593 return state == State.PENDING_CLOSE;
1594 }
1595
1596 synchronized void setPendingClose() {
1597 if (state != State.CLOSING) {
1598 LOG.warn("Cannot set a region as pending close if it has not been " +
1599 "closing. FIX!! State: " + toString());
1600 }
1601 state = State.PENDING_CLOSE;
1602 }
1603
1604 synchronized boolean isClosed() {
1605 return state == State.CLOSED;
1606 }
1607
1608 synchronized void setClosed() {
1609 if (state != State.PENDING_CLOSE &&
1610 state != State.PENDING_OPEN &&
1611 state != State.CLOSING) {
1612 throw new IllegalStateException(
1613 "Cannot set a region to be closed if it was not already marked as" +
1614 " pending close, pending open or closing. State: " + this);
1615 }
1616 state = State.CLOSED;
1617 }
1618
1619 synchronized boolean isOfflined() {
1620 return (state == State.CLOSING ||
1621 state == State.PENDING_CLOSE) && isOfflined;
1622 }
1623
1624 @Override
1625 public synchronized String toString() {
1626 return ("name=" + Bytes.toString(getRegionName()) +
1627 ", state=" + this.state);
1628 }
1629
1630 @Override
1631 public boolean equals(Object o) {
1632 if (this == o) {
1633 return true;
1634 }
1635 if (o == null || getClass() != o.getClass()) {
1636 return false;
1637 }
1638 return this.compareTo((RegionState) o) == 0;
1639 }
1640
1641 @Override
1642 public int hashCode() {
1643 return Bytes.toString(getRegionName()).hashCode();
1644 }
1645
1646 public int compareTo(RegionState o) {
1647 if (o == null) {
1648 return 1;
1649 }
1650 return Bytes.compareTo(getRegionName(), o.getRegionName());
1651 }
1652 }
1653 }