1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import static org.apache.hadoop.hbase.executor.EventType.RS_ZK_REGION_MERGED;
22 import static org.apache.hadoop.hbase.executor.EventType.RS_ZK_REGION_MERGING;
23 import static org.apache.hadoop.hbase.executor.EventType.RS_ZK_REQUEST_REGION_MERGE;
24
25 import java.io.IOException;
26 import java.io.InterruptedIOException;
27 import java.security.PrivilegedExceptionAction;
28 import java.util.ArrayList;
29 import java.util.List;
30 import java.util.ListIterator;
31 import java.util.Map;
32
33 import org.apache.commons.logging.Log;
34 import org.apache.commons.logging.LogFactory;
35 import org.apache.hadoop.hbase.classification.InterfaceAudience;
36 import org.apache.hadoop.fs.Path;
37 import org.apache.hadoop.hbase.HConstants;
38 import org.apache.hadoop.hbase.HRegionInfo;
39 import org.apache.hadoop.hbase.MetaMutationAnnotation;
40 import org.apache.hadoop.hbase.RegionTransition;
41 import org.apache.hadoop.hbase.Server;
42 import org.apache.hadoop.hbase.ServerName;
43 import org.apache.hadoop.hbase.catalog.CatalogTracker;
44 import org.apache.hadoop.hbase.catalog.MetaEditor;
45 import org.apache.hadoop.hbase.catalog.MetaReader;
46 import org.apache.hadoop.hbase.client.Delete;
47 import org.apache.hadoop.hbase.client.Mutation;
48 import org.apache.hadoop.hbase.client.Put;
49 import org.apache.hadoop.hbase.executor.EventType;
50 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
51 import org.apache.hadoop.hbase.regionserver.SplitTransaction.LoggingProgressable;
52 import org.apache.hadoop.hbase.security.User;
53 import org.apache.hadoop.hbase.util.Bytes;
54 import org.apache.hadoop.hbase.util.ConfigUtil;
55 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
56 import org.apache.hadoop.hbase.util.Pair;
57 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
58 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
59 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
60 import org.apache.zookeeper.KeeperException;
61 import org.apache.zookeeper.KeeperException.NodeExistsException;
62 import org.apache.zookeeper.data.Stat;
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92 @InterfaceAudience.Private
93 public class RegionMergeTransaction {
94 private static final Log LOG = LogFactory.getLog(RegionMergeTransaction.class);
95
96
97 private HRegionInfo mergedRegionInfo;
98
99 private final HRegion region_a;
100 private final HRegion region_b;
101
102 private final Path mergesdir;
103 private int znodeVersion = -1;
104
105 private final boolean forcible;
106 private boolean useZKForAssignment;
107 private final long masterSystemTime;
108
109
110
111
112
113 enum JournalEntry {
114
115
116
117 SET_MERGING_IN_ZK,
118
119
120
121 CREATED_MERGE_DIR,
122
123
124
125 CLOSED_REGION_A,
126
127
128
129 OFFLINED_REGION_A,
130
131
132
133 CLOSED_REGION_B,
134
135
136
137 OFFLINED_REGION_B,
138
139
140
141 STARTED_MERGED_REGION_CREATION,
142
143
144
145
146 PONR
147 }
148
149
150
151
152 private final List<JournalEntry> journal = new ArrayList<JournalEntry>();
153
154 private static IOException closedByOtherException = new IOException(
155 "Failed to close region: already closed by another thread");
156
157 private RegionServerCoprocessorHost rsCoprocessorHost = null;
158
159
160
161
162
163
164
165 public RegionMergeTransaction(final HRegion a, final HRegion b,
166 final boolean forcible) {
167 this(a, b, forcible, EnvironmentEdgeManager.currentTimeMillis());
168 }
169
170
171
172
173
174
175
176
177 public RegionMergeTransaction(final HRegion a, final HRegion b,
178 final boolean forcible, long masterSystemTime) {
179 if (a.getRegionInfo().compareTo(b.getRegionInfo()) <= 0) {
180 this.region_a = a;
181 this.region_b = b;
182 } else {
183 this.region_a = b;
184 this.region_b = a;
185 }
186 this.forcible = forcible;
187 this.masterSystemTime = masterSystemTime;
188 this.mergesdir = region_a.getRegionFileSystem().getMergesDir();
189 }
190
191
192
193
194
195
196
197 public boolean prepare(final RegionServerServices services) {
198 if (!region_a.getTableDesc().getTableName()
199 .equals(region_b.getTableDesc().getTableName())) {
200 LOG.info("Can't merge regions " + region_a + "," + region_b
201 + " because they do not belong to the same table");
202 return false;
203 }
204 if (region_a.getRegionInfo().equals(region_b.getRegionInfo())) {
205 LOG.info("Can't merge the same region " + region_a);
206 return false;
207 }
208 if (!forcible && !HRegionInfo.areAdjacent(region_a.getRegionInfo(),
209 region_b.getRegionInfo())) {
210 String msg = "Skip merging " + this.region_a.getRegionNameAsString()
211 + " and " + this.region_b.getRegionNameAsString()
212 + ", because they are not adjacent.";
213 LOG.info(msg);
214 return false;
215 }
216 if (!this.region_a.isMergeable() || !this.region_b.isMergeable()) {
217 return false;
218 }
219 try {
220 boolean regionAHasMergeQualifier = hasMergeQualifierInMeta(services,
221 region_a.getRegionName());
222 if (regionAHasMergeQualifier ||
223 hasMergeQualifierInMeta(services, region_b.getRegionName())) {
224 LOG.debug("Region " + (regionAHasMergeQualifier ? region_a.getRegionNameAsString()
225 : region_b.getRegionNameAsString())
226 + " is not mergeable because it has merge qualifier in META");
227 return false;
228 }
229 } catch (IOException e) {
230 LOG.warn("Failed judging whether merge transaction is available for "
231 + region_a.getRegionNameAsString() + " and "
232 + region_b.getRegionNameAsString(), e);
233 return false;
234 }
235
236
237
238
239
240
241
242
243 this.mergedRegionInfo = getMergedRegionInfo(region_a.getRegionInfo(),
244 region_b.getRegionInfo());
245 return true;
246 }
247
248
249
250
251
252
253
254
255
256
257
258
259 public HRegion execute(final Server server,
260 final RegionServerServices services) throws IOException {
261 if (User.isHBaseSecurityEnabled(region_a.getBaseConf())) {
262 LOG.warn("Should use execute(Server, RegionServerServices, User)");
263 }
264 return execute(server, services, null);
265 }
266
267 public HRegion execute(final Server server, final RegionServerServices services, User user)
268 throws IOException {
269 useZKForAssignment = server == null ? true :
270 ConfigUtil.useZKForAssignment(server.getConfiguration());
271 if (rsCoprocessorHost == null) {
272 rsCoprocessorHost = server != null ? ((HRegionServer) server).getCoprocessorHost() : null;
273 }
274 final HRegion mergedRegion = createMergedRegion(server, services, user);
275 if (rsCoprocessorHost != null) {
276 if (user == null) {
277 rsCoprocessorHost.postMergeCommit(this.region_a, this.region_b, mergedRegion);
278 } else {
279 try {
280 user.getUGI().doAs(new PrivilegedExceptionAction<Void>() {
281 @Override
282 public Void run() throws Exception {
283 rsCoprocessorHost.postMergeCommit(region_a, region_b, mergedRegion);
284 return null;
285 }
286 });
287 } catch (InterruptedException ie) {
288 InterruptedIOException iioe = new InterruptedIOException();
289 iioe.initCause(ie);
290 throw iioe;
291 }
292 }
293 }
294 stepsAfterPONR(server, services, mergedRegion, user);
295 return mergedRegion;
296 }
297
298 @Deprecated
299 public void stepsAfterPONR(final Server server, final RegionServerServices services,
300 final HRegion mergedRegion) throws IOException {
301 stepsAfterPONR(server, services, mergedRegion, null);
302 }
303
304 public void stepsAfterPONR(final Server server, final RegionServerServices services,
305 final HRegion mergedRegion, User user) throws IOException {
306 openMergedRegion(server, services, mergedRegion);
307 transitionZKNode(server, services, mergedRegion, user);
308 }
309
310
311
312
313
314
315
316
317
318
319 HRegion createMergedRegion(final Server server,
320 final RegionServerServices services, User user) throws IOException {
321 LOG.info("Starting merge of " + region_a + " and "
322 + region_b.getRegionNameAsString() + ", forcible=" + forcible);
323 if ((server != null && server.isStopped())
324 || (services != null && services.isStopping())) {
325 throw new IOException("Server is stopped or stopping");
326 }
327
328 if (rsCoprocessorHost != null) {
329 boolean ret = false;
330 if (user == null) {
331 ret = rsCoprocessorHost.preMerge(region_a, region_b);
332 } else {
333 try {
334 ret = user.getUGI().doAs(new PrivilegedExceptionAction<Boolean>() {
335 @Override
336 public Boolean run() throws Exception {
337 return rsCoprocessorHost.preMerge(region_a, region_b);
338 }
339 });
340 } catch (InterruptedException ie) {
341 InterruptedIOException iioe = new InterruptedIOException();
342 iioe.initCause(ie);
343 throw iioe;
344 }
345 }
346 if (ret) {
347 throw new IOException("Coprocessor bypassing regions " + this.region_a + " "
348 + this.region_b + " merge.");
349 }
350 }
351
352
353 boolean testing = server == null ? true : server.getConfiguration()
354 .getBoolean("hbase.testing.nocluster", false);
355
356 HRegion mergedRegion = stepsBeforePONR(server, services, testing);
357
358 @MetaMutationAnnotation
359 final List<Mutation> metaEntries = new ArrayList<Mutation>();
360 if (rsCoprocessorHost != null) {
361 boolean ret = false;
362 if (user == null) {
363 ret = rsCoprocessorHost.preMergeCommit(region_a, region_b, metaEntries);
364 } else {
365 try {
366 ret = user.getUGI().doAs(new PrivilegedExceptionAction<Boolean>() {
367 @Override
368 public Boolean run() throws Exception {
369 return rsCoprocessorHost.preMergeCommit(region_a, region_b, metaEntries);
370 }
371 });
372 } catch (InterruptedException ie) {
373 InterruptedIOException iioe = new InterruptedIOException();
374 iioe.initCause(ie);
375 throw iioe;
376 }
377 }
378
379 if (ret) {
380 throw new IOException("Coprocessor bypassing regions " + this.region_a + " "
381 + this.region_b + " merge.");
382 }
383 try {
384 for (Mutation p : metaEntries) {
385 HRegionInfo.parseRegionName(p.getRow());
386 }
387 } catch (IOException e) {
388 LOG.error("Row key of mutation from coprocessor is not parsable as region name."
389 + "Mutations from coprocessor should only be for hbase:meta table.", e);
390 throw e;
391 }
392 }
393
394
395
396
397 this.journal.add(JournalEntry.PONR);
398
399
400
401
402
403
404 if (!testing && useZKForAssignment) {
405 if (metaEntries.isEmpty()) {
406 MetaEditor.mergeRegions(server.getCatalogTracker(), mergedRegion.getRegionInfo(), region_a
407 .getRegionInfo(), region_b.getRegionInfo(), server.getServerName(), masterSystemTime);
408 } else {
409 mergeRegionsAndPutMetaEntries(server.getCatalogTracker(), mergedRegion.getRegionInfo(),
410 region_a.getRegionInfo(), region_b.getRegionInfo(), server.getServerName(), metaEntries);
411 }
412 } else if (services != null && !useZKForAssignment) {
413 if (!services.reportRegionStateTransition(TransitionCode.MERGE_PONR,
414 mergedRegionInfo, region_a.getRegionInfo(), region_b.getRegionInfo())) {
415
416 throw new IOException("Failed to notify master that merge passed PONR: "
417 + region_a.getRegionInfo().getRegionNameAsString() + " and "
418 + region_b.getRegionInfo().getRegionNameAsString());
419 }
420 }
421 return mergedRegion;
422 }
423
424 private void mergeRegionsAndPutMetaEntries(CatalogTracker catalogTracker,
425 HRegionInfo mergedRegion, HRegionInfo regionA, HRegionInfo regionB, ServerName serverName,
426 List<Mutation> metaEntries) throws IOException {
427 prepareMutationsForMerge(mergedRegion, regionA, regionB, serverName, metaEntries);
428 MetaEditor.mutateMetaTable(catalogTracker, metaEntries);
429 }
430
431 public void prepareMutationsForMerge(HRegionInfo mergedRegion, HRegionInfo regionA,
432 HRegionInfo regionB, ServerName serverName, List<Mutation> mutations) throws IOException {
433 HRegionInfo copyOfMerged = new HRegionInfo(mergedRegion);
434
435
436 long time = Math.max(EnvironmentEdgeManager.currentTimeMillis(), masterSystemTime);
437
438
439 Put putOfMerged = MetaEditor.makePutFromRegionInfo(copyOfMerged, time);
440 putOfMerged.add(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER, regionA.toByteArray());
441 putOfMerged.add(HConstants.CATALOG_FAMILY, HConstants.MERGEB_QUALIFIER, regionB.toByteArray());
442 mutations.add(putOfMerged);
443
444 Delete deleteA = MetaEditor.makeDeleteFromRegionInfo(regionA, time);
445 Delete deleteB = MetaEditor.makeDeleteFromRegionInfo(regionB, time);
446 mutations.add(deleteA);
447 mutations.add(deleteB);
448
449 addLocation(putOfMerged, serverName, 1);
450 }
451
452 @SuppressWarnings("deprecation")
453 public Put addLocation(final Put p, final ServerName sn, long openSeqNum) {
454 p.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, Bytes
455 .toBytes(sn.getHostAndPort()));
456 p.add(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER, Bytes.toBytes(sn
457 .getStartcode()));
458 p.add(HConstants.CATALOG_FAMILY, HConstants.SEQNUM_QUALIFIER, Bytes.toBytes(openSeqNum));
459 return p;
460 }
461
462 public HRegion stepsBeforePONR(final Server server, final RegionServerServices services,
463 boolean testing) throws IOException {
464
465
466 if (useZKAndZKIsSet(server)) {
467 try {
468 createNodeMerging(server.getZooKeeper(), this.mergedRegionInfo,
469 server.getServerName(), region_a.getRegionInfo(), region_b.getRegionInfo());
470 } catch (KeeperException e) {
471 throw new IOException("Failed creating PENDING_MERGE znode on "
472 + this.mergedRegionInfo.getRegionNameAsString(), e);
473 }
474 } else if (services != null && !useZKForAssignment) {
475 if (!services.reportRegionStateTransition(TransitionCode.READY_TO_MERGE,
476 mergedRegionInfo, region_a.getRegionInfo(), region_b.getRegionInfo())) {
477 throw new IOException("Failed to get ok from master to merge "
478 + region_a.getRegionInfo().getRegionNameAsString() + " and "
479 + region_b.getRegionInfo().getRegionNameAsString());
480 }
481 }
482 this.journal.add(JournalEntry.SET_MERGING_IN_ZK);
483 if (useZKAndZKIsSet(server)) {
484
485
486
487 znodeVersion = getZKNode(server, services);
488 }
489
490 this.region_a.getRegionFileSystem().createMergesDir();
491 this.journal.add(JournalEntry.CREATED_MERGE_DIR);
492
493 Map<byte[], List<StoreFile>> hstoreFilesOfRegionA = closeAndOfflineRegion(
494 services, this.region_a, true, testing);
495 Map<byte[], List<StoreFile>> hstoreFilesOfRegionB = closeAndOfflineRegion(
496 services, this.region_b, false, testing);
497
498 assert hstoreFilesOfRegionA != null && hstoreFilesOfRegionB != null;
499
500
501
502
503
504
505 mergeStoreFiles(hstoreFilesOfRegionA, hstoreFilesOfRegionB);
506
507 if (server != null && useZKAndZKIsSet(server)) {
508 try {
509
510
511 this.znodeVersion = transitionMergingNode(server.getZooKeeper(),
512 this.mergedRegionInfo, region_a.getRegionInfo(), region_b.getRegionInfo(),
513 server.getServerName(), this.znodeVersion,
514 RS_ZK_REGION_MERGING, RS_ZK_REGION_MERGING);
515 } catch (KeeperException e) {
516 throw new IOException("Failed setting MERGING znode on "
517 + this.mergedRegionInfo.getRegionNameAsString(), e);
518 }
519 }
520
521
522
523
524
525 this.journal.add(JournalEntry.STARTED_MERGED_REGION_CREATION);
526 HRegion mergedRegion = createMergedRegionFromMerges(this.region_a,
527 this.region_b, this.mergedRegionInfo);
528 return mergedRegion;
529 }
530
531
532
533
534
535
536
537
538
539
540 HRegion createMergedRegionFromMerges(final HRegion a, final HRegion b,
541 final HRegionInfo mergedRegion) throws IOException {
542 return a.createMergedRegionFromMerges(mergedRegion, b);
543 }
544
545
546
547
548
549
550
551
552
553
554 private Map<byte[], List<StoreFile>> closeAndOfflineRegion(
555 final RegionServerServices services, final HRegion region,
556 final boolean isRegionA, final boolean testing) throws IOException {
557 Map<byte[], List<StoreFile>> hstoreFilesToMerge = null;
558 Exception exceptionToThrow = null;
559 try {
560 hstoreFilesToMerge = region.close(false);
561 } catch (Exception e) {
562 exceptionToThrow = e;
563 }
564 if (exceptionToThrow == null && hstoreFilesToMerge == null) {
565
566
567
568
569
570 exceptionToThrow = closedByOtherException;
571 }
572 if (exceptionToThrow != closedByOtherException) {
573 this.journal.add(isRegionA ? JournalEntry.CLOSED_REGION_A
574 : JournalEntry.CLOSED_REGION_B);
575 }
576 if (exceptionToThrow != null) {
577 if (exceptionToThrow instanceof IOException)
578 throw (IOException) exceptionToThrow;
579 throw new IOException(exceptionToThrow);
580 }
581
582 if (!testing) {
583 services.removeFromOnlineRegions(region, null);
584 }
585 this.journal.add(isRegionA ? JournalEntry.OFFLINED_REGION_A
586 : JournalEntry.OFFLINED_REGION_B);
587 return hstoreFilesToMerge;
588 }
589
590
591
592
593
594
595
596 public static HRegionInfo getMergedRegionInfo(final HRegionInfo a,
597 final HRegionInfo b) {
598 long rid = EnvironmentEdgeManager.currentTimeMillis();
599
600
601 if (rid < a.getRegionId() || rid < b.getRegionId()) {
602 LOG.warn("Clock skew; merging regions id are " + a.getRegionId()
603 + " and " + b.getRegionId() + ", but current time here is " + rid);
604 rid = Math.max(a.getRegionId(), b.getRegionId()) + 1;
605 }
606
607 byte[] startKey = null;
608 byte[] endKey = null;
609
610 if (a.compareTo(b) <= 0) {
611 startKey = a.getStartKey();
612 } else {
613 startKey = b.getStartKey();
614 }
615
616 if (Bytes.equals(a.getEndKey(), HConstants.EMPTY_BYTE_ARRAY)
617 || (!Bytes.equals(b.getEndKey(), HConstants.EMPTY_BYTE_ARRAY)
618 && Bytes.compareTo(a.getEndKey(), b.getEndKey()) > 0)) {
619 endKey = a.getEndKey();
620 } else {
621 endKey = b.getEndKey();
622 }
623
624
625 HRegionInfo mergedRegionInfo = new HRegionInfo(a.getTable(), startKey,
626 endKey, false, rid);
627 return mergedRegionInfo;
628 }
629
630
631
632
633
634
635
636
637
638
639 void openMergedRegion(final Server server,
640 final RegionServerServices services, HRegion merged) throws IOException {
641 boolean stopped = server != null && server.isStopped();
642 boolean stopping = services != null && services.isStopping();
643 if (stopped || stopping) {
644 LOG.info("Not opening merged region " + merged.getRegionNameAsString()
645 + " because stopping=" + stopping + ", stopped=" + stopped);
646 return;
647 }
648 HRegionInfo hri = merged.getRegionInfo();
649 LoggingProgressable reporter = server == null ? null
650 : new LoggingProgressable(hri, server.getConfiguration().getLong(
651 "hbase.regionserver.regionmerge.open.log.interval", 10000));
652 merged.openHRegion(reporter);
653
654 if (services != null) {
655 try {
656 if (useZKForAssignment) {
657 services.postOpenDeployTasks(merged, server.getCatalogTracker());
658 } else if (!services.reportRegionStateTransition(TransitionCode.MERGED,
659 mergedRegionInfo, region_a.getRegionInfo(), region_b.getRegionInfo())) {
660 throw new IOException("Failed to report merged region to master: "
661 + mergedRegionInfo.getShortNameToLog());
662 }
663 services.addToOnlineRegions(merged);
664 } catch (KeeperException ke) {
665 throw new IOException(ke);
666 }
667 }
668
669 }
670
671
672
673
674
675
676
677
678
679 void transitionZKNode(final Server server, final RegionServerServices services,
680 final HRegion mergedRegion, User user) throws IOException {
681 if (useZKAndZKIsSet(server)) {
682
683 try {
684 this.znodeVersion = transitionMergingNode(server.getZooKeeper(),
685 this.mergedRegionInfo, region_a.getRegionInfo(),
686 region_b.getRegionInfo(), server.getServerName(), this.znodeVersion,
687 RS_ZK_REGION_MERGING, RS_ZK_REGION_MERGED);
688
689 long startTime = EnvironmentEdgeManager.currentTimeMillis();
690 int spins = 0;
691
692
693
694 do {
695 if (spins % 10 == 0) {
696 LOG.debug("Still waiting on the master to process the merge for "
697 + this.mergedRegionInfo.getEncodedName() + ", waited "
698 + (EnvironmentEdgeManager.currentTimeMillis() - startTime) + "ms");
699 }
700 Thread.sleep(100);
701
702 this.znodeVersion = transitionMergingNode(server.getZooKeeper(),
703 this.mergedRegionInfo, region_a.getRegionInfo(),
704 region_b.getRegionInfo(), server.getServerName(), this.znodeVersion,
705 RS_ZK_REGION_MERGED, RS_ZK_REGION_MERGED);
706 spins++;
707 } while (this.znodeVersion != -1 && !server.isStopped()
708 && !services.isStopping());
709 } catch (Exception e) {
710 if (e instanceof InterruptedException) {
711 Thread.currentThread().interrupt();
712 }
713 throw new IOException("Failed telling master about merge "
714 + mergedRegionInfo.getEncodedName(), e);
715 }
716 }
717
718 if (rsCoprocessorHost != null) {
719 if (user == null) {
720 rsCoprocessorHost.postMerge(region_a, region_b, mergedRegion);
721 } else {
722 try {
723 user.getUGI().doAs(new PrivilegedExceptionAction<Void>() {
724 @Override
725 public Void run() throws Exception {
726 rsCoprocessorHost.postMerge(region_a, region_b, mergedRegion);
727 return null;
728 }
729 });
730 } catch (InterruptedException ie) {
731 InterruptedIOException iioe = new InterruptedIOException();
732 iioe.initCause(ie);
733 throw iioe;
734 }
735 }
736 }
737
738
739
740
741 }
742
743
744
745
746
747
748
749
750
751 private int getZKNode(final Server server,
752 final RegionServerServices services) throws IOException {
753
754 try {
755 int spins = 0;
756 Stat stat = new Stat();
757 ZooKeeperWatcher zkw = server.getZooKeeper();
758 ServerName expectedServer = server.getServerName();
759 String node = mergedRegionInfo.getEncodedName();
760 while (!(server.isStopped() || services.isStopping())) {
761 if (spins % 5 == 0) {
762 LOG.debug("Still waiting for master to process "
763 + "the pending_merge for " + node);
764 transitionMergingNode(zkw, mergedRegionInfo, region_a.getRegionInfo(),
765 region_b.getRegionInfo(), expectedServer, -1, RS_ZK_REQUEST_REGION_MERGE,
766 RS_ZK_REQUEST_REGION_MERGE);
767 }
768 Thread.sleep(100);
769 spins++;
770 byte [] data = ZKAssign.getDataNoWatch(zkw, node, stat);
771 if (data == null) {
772 throw new IOException("Data is null, merging node "
773 + node + " no longer exists");
774 }
775 RegionTransition rt = RegionTransition.parseFrom(data);
776 EventType et = rt.getEventType();
777 if (et == RS_ZK_REGION_MERGING) {
778 ServerName serverName = rt.getServerName();
779 if (!serverName.equals(expectedServer)) {
780 throw new IOException("Merging node " + node + " is for "
781 + serverName + ", not us " + expectedServer);
782 }
783 byte [] payloadOfMerging = rt.getPayload();
784 List<HRegionInfo> mergingRegions = HRegionInfo.parseDelimitedFrom(
785 payloadOfMerging, 0, payloadOfMerging.length);
786 assert mergingRegions.size() == 3;
787 HRegionInfo a = mergingRegions.get(1);
788 HRegionInfo b = mergingRegions.get(2);
789 HRegionInfo hri_a = region_a.getRegionInfo();
790 HRegionInfo hri_b = region_b.getRegionInfo();
791 if (!(hri_a.equals(a) && hri_b.equals(b))) {
792 throw new IOException("Merging node " + node + " is for " + a + ", "
793 + b + ", not expected regions: " + hri_a + ", " + hri_b);
794 }
795
796 return stat.getVersion();
797 }
798 if (et != RS_ZK_REQUEST_REGION_MERGE) {
799 throw new IOException("Merging node " + node
800 + " moved out of merging to " + et);
801 }
802 }
803
804 throw new IOException("Server is "
805 + (services.isStopping() ? "stopping" : "stopped"));
806 } catch (Exception e) {
807 if (e instanceof InterruptedException) {
808 Thread.currentThread().interrupt();
809 }
810 throw new IOException("Failed getting MERGING znode on "
811 + mergedRegionInfo.getRegionNameAsString(), e);
812 }
813 }
814
815
816
817
818
819
820
821 private void mergeStoreFiles(
822 Map<byte[], List<StoreFile>> hstoreFilesOfRegionA,
823 Map<byte[], List<StoreFile>> hstoreFilesOfRegionB)
824 throws IOException {
825
826 HRegionFileSystem fs_a = this.region_a.getRegionFileSystem();
827 for (Map.Entry<byte[], List<StoreFile>> entry : hstoreFilesOfRegionA
828 .entrySet()) {
829 String familyName = Bytes.toString(entry.getKey());
830 for (StoreFile storeFile : entry.getValue()) {
831 fs_a.mergeStoreFile(this.mergedRegionInfo, familyName, storeFile,
832 this.mergesdir);
833 }
834 }
835
836 HRegionFileSystem fs_b = this.region_b.getRegionFileSystem();
837 for (Map.Entry<byte[], List<StoreFile>> entry : hstoreFilesOfRegionB
838 .entrySet()) {
839 String familyName = Bytes.toString(entry.getKey());
840 for (StoreFile storeFile : entry.getValue()) {
841 fs_b.mergeStoreFile(this.mergedRegionInfo, familyName, storeFile,
842 this.mergesdir);
843 }
844 }
845 }
846
847
848
849
850
851
852
853
854
855 @SuppressWarnings("deprecation")
856 public boolean rollback(final Server server,
857 final RegionServerServices services) throws IOException {
858 if (User.isHBaseSecurityEnabled(region_a.getBaseConf())) {
859 LOG.warn("Should use execute(Server, RegionServerServices, User)");
860 }
861 return rollback(server, services, null);
862 }
863
864 public boolean rollback(final Server server,
865 final RegionServerServices services, User user) throws IOException {
866 assert this.mergedRegionInfo != null;
867
868 if (rsCoprocessorHost != null) {
869 if (user == null) {
870 rsCoprocessorHost.preRollBackMerge(region_a, region_b);
871 } else {
872 try {
873 user.getUGI().doAs(new PrivilegedExceptionAction<Void>() {
874 @Override
875 public Void run() throws Exception {
876 rsCoprocessorHost.preRollBackMerge(region_a, region_b);
877 return null;
878 }
879 });
880 } catch (InterruptedException ie) {
881 InterruptedIOException iioe = new InterruptedIOException();
882 iioe.initCause(ie);
883 throw iioe;
884 }
885 }
886 }
887
888 boolean result = true;
889 ListIterator<JournalEntry> iterator = this.journal
890 .listIterator(this.journal.size());
891
892 while (iterator.hasPrevious()) {
893 JournalEntry je = iterator.previous();
894 switch (je) {
895
896 case SET_MERGING_IN_ZK:
897 if (useZKAndZKIsSet(server)) {
898 cleanZK(server, this.mergedRegionInfo);
899 } else if (services != null && !useZKForAssignment
900 && !services.reportRegionStateTransition(TransitionCode.MERGE_REVERTED,
901 mergedRegionInfo, region_a.getRegionInfo(), region_b.getRegionInfo())) {
902 return false;
903 }
904 break;
905
906 case CREATED_MERGE_DIR:
907 this.region_a.writestate.writesEnabled = true;
908 this.region_b.writestate.writesEnabled = true;
909 this.region_a.getRegionFileSystem().cleanupMergesDir();
910 break;
911
912 case CLOSED_REGION_A:
913 try {
914
915
916
917
918
919 this.region_a.initialize();
920 } catch (IOException e) {
921 LOG.error("Failed rollbacking CLOSED_REGION_A of region "
922 + this.region_a.getRegionNameAsString(), e);
923 throw new RuntimeException(e);
924 }
925 break;
926
927 case OFFLINED_REGION_A:
928 if (services != null)
929 services.addToOnlineRegions(this.region_a);
930 break;
931
932 case CLOSED_REGION_B:
933 try {
934 this.region_b.initialize();
935 } catch (IOException e) {
936 LOG.error("Failed rollbacking CLOSED_REGION_A of region "
937 + this.region_b.getRegionNameAsString(), e);
938 throw new RuntimeException(e);
939 }
940 break;
941
942 case OFFLINED_REGION_B:
943 if (services != null)
944 services.addToOnlineRegions(this.region_b);
945 break;
946
947 case STARTED_MERGED_REGION_CREATION:
948 this.region_a.getRegionFileSystem().cleanupMergedRegion(
949 this.mergedRegionInfo);
950 break;
951
952 case PONR:
953
954
955 return false;
956
957 default:
958 throw new RuntimeException("Unhandled journal entry: " + je);
959 }
960 }
961
962 if (rsCoprocessorHost != null) {
963 if (user == null) {
964 rsCoprocessorHost.postRollBackMerge(region_a, region_b);
965 } else {
966 try {
967 user.getUGI().doAs(new PrivilegedExceptionAction<Void>() {
968 @Override
969 public Void run() throws Exception {
970 rsCoprocessorHost.postRollBackMerge(region_a, region_b);
971 return null;
972 }
973 });
974 } catch (InterruptedException ie) {
975 InterruptedIOException iioe = new InterruptedIOException();
976 iioe.initCause(ie);
977 throw iioe;
978 }
979 }
980 }
981
982 return result;
983 }
984
985 HRegionInfo getMergedRegionInfo() {
986 return this.mergedRegionInfo;
987 }
988
989
990 Path getMergesDir() {
991 return this.mergesdir;
992 }
993
994 private boolean useZKAndZKIsSet(final Server server) {
995 return server != null && useZKForAssignment && server.getZooKeeper() != null;
996 }
997
998 private static void cleanZK(final Server server, final HRegionInfo hri) {
999 try {
1000
1001 if (!ZKAssign.deleteNode(server.getZooKeeper(), hri.getEncodedName(),
1002 RS_ZK_REQUEST_REGION_MERGE, server.getServerName())) {
1003 ZKAssign.deleteNode(server.getZooKeeper(), hri.getEncodedName(),
1004 RS_ZK_REGION_MERGING, server.getServerName());
1005 }
1006 } catch (KeeperException.NoNodeException e) {
1007 LOG.info("Failed cleanup zk node of " + hri.getRegionNameAsString(), e);
1008 } catch (KeeperException e) {
1009 server.abort("Failed cleanup zk node of " + hri.getRegionNameAsString(),e);
1010 }
1011 }
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027 public static void createNodeMerging(final ZooKeeperWatcher zkw, final HRegionInfo region,
1028 final ServerName serverName, final HRegionInfo a,
1029 final HRegionInfo b) throws KeeperException, IOException {
1030 LOG.debug(zkw.prefix("Creating ephemeral node for "
1031 + region.getEncodedName() + " in PENDING_MERGE state"));
1032 byte [] payload = HRegionInfo.toDelimitedByteArray(region, a, b);
1033 RegionTransition rt = RegionTransition.createRegionTransition(
1034 RS_ZK_REQUEST_REGION_MERGE, region.getRegionName(), serverName, payload);
1035 String node = ZKAssign.getNodeName(zkw, region.getEncodedName());
1036 if (!ZKUtil.createEphemeralNodeAndWatch(zkw, node, rt.toByteArray())) {
1037 throw new IOException("Failed create of ephemeral " + node);
1038 }
1039 }
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080 public static int transitionMergingNode(ZooKeeperWatcher zkw,
1081 HRegionInfo merged, HRegionInfo a, HRegionInfo b, ServerName serverName,
1082 final int znodeVersion, final EventType beginState,
1083 final EventType endState) throws KeeperException, IOException {
1084 byte[] payload = HRegionInfo.toDelimitedByteArray(merged, a, b);
1085 return ZKAssign.transitionNode(zkw, merged, serverName,
1086 beginState, endState, znodeVersion, payload);
1087 }
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097 boolean hasMergeQualifierInMeta(final RegionServerServices services,
1098 final byte[] regionName) throws IOException {
1099 if (services == null) return false;
1100
1101
1102 Pair<HRegionInfo, HRegionInfo> mergeRegions = MetaReader
1103 .getRegionsFromMergeQualifier(services.getCatalogTracker(), regionName);
1104 if (mergeRegions != null &&
1105 (mergeRegions.getFirst() != null || mergeRegions.getSecond() != null)) {
1106
1107 return true;
1108 }
1109 return false;
1110 }
1111 }