1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import java.io.IOException;
22 import java.util.ArrayList;
23 import java.util.List;
24 import java.util.ListIterator;
25 import java.util.Map;
26
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.hbase.classification.InterfaceAudience;
30 import org.apache.hadoop.fs.Path;
31 import org.apache.hadoop.hbase.HConstants;
32 import org.apache.hadoop.hbase.HRegionInfo;
33 import org.apache.hadoop.hbase.MetaMutationAnnotation;
34 import org.apache.hadoop.hbase.Server;
35 import org.apache.hadoop.hbase.ServerName;
36 import org.apache.hadoop.hbase.MetaTableAccessor;
37 import org.apache.hadoop.hbase.client.Delete;
38 import org.apache.hadoop.hbase.client.HConnection;
39 import org.apache.hadoop.hbase.client.Mutation;
40 import org.apache.hadoop.hbase.client.Put;
41 import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager;
42 import org.apache.hadoop.hbase.coordination.RegionMergeCoordination.RegionMergeDetails;
43 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
44 import org.apache.hadoop.hbase.regionserver.SplitTransaction.LoggingProgressable;
45 import org.apache.hadoop.hbase.util.Bytes;
46 import org.apache.hadoop.hbase.util.ConfigUtil;
47 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
48 import org.apache.hadoop.hbase.util.Pair;
49 import org.apache.zookeeper.KeeperException;
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79 @InterfaceAudience.Private
80 public class RegionMergeTransaction {
81 private static final Log LOG = LogFactory.getLog(RegionMergeTransaction.class);
82
83
84 private HRegionInfo mergedRegionInfo;
85
86 private final HRegion region_a;
87 private final HRegion region_b;
88
89 private final Path mergesdir;
90
91 private final boolean forcible;
92 private boolean useCoordinationForAssignment;
93
94
95
96
97
98 enum JournalEntry {
99
100
101
102 SET_MERGING,
103
104
105
106 CREATED_MERGE_DIR,
107
108
109
110 CLOSED_REGION_A,
111
112
113
114 OFFLINED_REGION_A,
115
116
117
118 CLOSED_REGION_B,
119
120
121
122 OFFLINED_REGION_B,
123
124
125
126 STARTED_MERGED_REGION_CREATION,
127
128
129
130
131 PONR
132 }
133
134
135
136
137 private final List<JournalEntry> journal = new ArrayList<JournalEntry>();
138
139 private static IOException closedByOtherException = new IOException(
140 "Failed to close region: already closed by another thread");
141
142 private RegionServerCoprocessorHost rsCoprocessorHost = null;
143
144 private RegionMergeDetails rmd;
145
146
147
148
149
150
151
152 public RegionMergeTransaction(final HRegion a, final HRegion b,
153 final boolean forcible) {
154 if (a.getRegionInfo().compareTo(b.getRegionInfo()) <= 0) {
155 this.region_a = a;
156 this.region_b = b;
157 } else {
158 this.region_a = b;
159 this.region_b = a;
160 }
161 this.forcible = forcible;
162 this.mergesdir = region_a.getRegionFileSystem().getMergesDir();
163 }
164
165
166
167
168
169
170
171 public boolean prepare(final RegionServerServices services) {
172 if (!region_a.getTableDesc().getTableName()
173 .equals(region_b.getTableDesc().getTableName())) {
174 LOG.info("Can't merge regions " + region_a + "," + region_b
175 + " because they do not belong to the same table");
176 return false;
177 }
178 if (region_a.getRegionInfo().equals(region_b.getRegionInfo())) {
179 LOG.info("Can't merge the same region " + region_a);
180 return false;
181 }
182 if (!forcible && !HRegionInfo.areAdjacent(region_a.getRegionInfo(),
183 region_b.getRegionInfo())) {
184 String msg = "Skip merging " + this.region_a.getRegionNameAsString()
185 + " and " + this.region_b.getRegionNameAsString()
186 + ", because they are not adjacent.";
187 LOG.info(msg);
188 return false;
189 }
190 if (!this.region_a.isMergeable() || !this.region_b.isMergeable()) {
191 return false;
192 }
193 try {
194 boolean regionAHasMergeQualifier = hasMergeQualifierInMeta(services,
195 region_a.getRegionName());
196 if (regionAHasMergeQualifier ||
197 hasMergeQualifierInMeta(services, region_b.getRegionName())) {
198 LOG.debug("Region " + (regionAHasMergeQualifier ? region_a.getRegionNameAsString()
199 : region_b.getRegionNameAsString())
200 + " is not mergeable because it has merge qualifier in META");
201 return false;
202 }
203 } catch (IOException e) {
204 LOG.warn("Failed judging whether merge transaction is available for "
205 + region_a.getRegionNameAsString() + " and "
206 + region_b.getRegionNameAsString(), e);
207 return false;
208 }
209
210
211
212
213
214
215
216
217 this.mergedRegionInfo = getMergedRegionInfo(region_a.getRegionInfo(),
218 region_b.getRegionInfo());
219 return true;
220 }
221
222
223
224
225
226
227
228
229
230
231
232 public HRegion execute(final Server server,
233 final RegionServerServices services) throws IOException {
234 useCoordinationForAssignment =
235 server == null ? true : ConfigUtil.useZKForAssignment(server.getConfiguration());
236 if (rmd == null) {
237 rmd =
238 server != null && server.getCoordinatedStateManager() != null ? ((BaseCoordinatedStateManager) server
239 .getCoordinatedStateManager()).getRegionMergeCoordination().getDefaultDetails()
240 : null;
241 }
242 if (rsCoprocessorHost == null) {
243 rsCoprocessorHost = server != null ?
244 ((HRegionServer) server).getRegionServerCoprocessorHost() : null;
245 }
246 HRegion mergedRegion = createMergedRegion(server, services);
247 if (rsCoprocessorHost != null) {
248 rsCoprocessorHost.postMergeCommit(this.region_a, this.region_b, mergedRegion);
249 }
250 return stepsAfterPONR(server, services, mergedRegion);
251 }
252
253 public HRegion stepsAfterPONR(final Server server, final RegionServerServices services,
254 HRegion mergedRegion) throws IOException {
255 openMergedRegion(server, services, mergedRegion);
256 if (useCoordination(server)) {
257 ((BaseCoordinatedStateManager) server.getCoordinatedStateManager())
258 .getRegionMergeCoordination().completeRegionMergeTransaction(services, mergedRegionInfo,
259 region_a, region_b, rmd, mergedRegion);
260 }
261 if (rsCoprocessorHost != null) {
262 rsCoprocessorHost.postMerge(this.region_a, this.region_b, mergedRegion);
263 }
264 return mergedRegion;
265 }
266
267
268
269
270
271
272
273
274
275 HRegion createMergedRegion(final Server server,
276 final RegionServerServices services) throws IOException {
277 LOG.info("Starting merge of " + region_a + " and "
278 + region_b.getRegionNameAsString() + ", forcible=" + forcible);
279 if ((server != null && server.isStopped())
280 || (services != null && services.isStopping())) {
281 throw new IOException("Server is stopped or stopping");
282 }
283
284 if (rsCoprocessorHost != null) {
285 if (rsCoprocessorHost.preMerge(this.region_a, this.region_b)) {
286 throw new IOException("Coprocessor bypassing regions " + this.region_a + " "
287 + this.region_b + " merge.");
288 }
289 }
290
291
292 boolean testing = server == null ? true : server.getConfiguration()
293 .getBoolean("hbase.testing.nocluster", false);
294
295 HRegion mergedRegion = stepsBeforePONR(server, services, testing);
296
297 @MetaMutationAnnotation
298 List<Mutation> metaEntries = new ArrayList<Mutation>();
299 if (rsCoprocessorHost != null) {
300 if (rsCoprocessorHost.preMergeCommit(this.region_a, this.region_b, metaEntries)) {
301 throw new IOException("Coprocessor bypassing regions " + this.region_a + " "
302 + this.region_b + " merge.");
303 }
304 try {
305 for (Mutation p : metaEntries) {
306 HRegionInfo.parseRegionName(p.getRow());
307 }
308 } catch (IOException e) {
309 LOG.error("Row key of mutation from coprocessor is not parsable as region name."
310 + "Mutations from coprocessor should only be for hbase:meta table.", e);
311 throw e;
312 }
313 }
314
315
316
317
318 this.journal.add(JournalEntry.PONR);
319
320
321
322
323
324
325 if (!testing && useCoordinationForAssignment) {
326 if (metaEntries.isEmpty()) {
327 MetaTableAccessor.mergeRegions(server.getConnection(),
328 mergedRegion.getRegionInfo(), region_a.getRegionInfo(), region_b.getRegionInfo(),
329 server.getServerName());
330 } else {
331 mergeRegionsAndPutMetaEntries(server.getConnection(),
332 mergedRegion.getRegionInfo(), region_a.getRegionInfo(), region_b.getRegionInfo(),
333 server.getServerName(), metaEntries);
334 }
335 } else if (services != null && !useCoordinationForAssignment) {
336 if (!services.reportRegionStateTransition(TransitionCode.MERGE_PONR,
337 mergedRegionInfo, region_a.getRegionInfo(), region_b.getRegionInfo())) {
338
339 throw new IOException("Failed to notify master that merge passed PONR: "
340 + region_a.getRegionInfo().getRegionNameAsString() + " and "
341 + region_b.getRegionInfo().getRegionNameAsString());
342 }
343 }
344 return mergedRegion;
345 }
346
347 private void mergeRegionsAndPutMetaEntries(HConnection hConnection,
348 HRegionInfo mergedRegion, HRegionInfo regionA, HRegionInfo regionB,
349 ServerName serverName, List<Mutation> metaEntries) throws IOException {
350 prepareMutationsForMerge(mergedRegion, regionA, regionB, serverName, metaEntries);
351 MetaTableAccessor.mutateMetaTable(hConnection, metaEntries);
352 }
353
354 public void prepareMutationsForMerge(HRegionInfo mergedRegion, HRegionInfo regionA,
355 HRegionInfo regionB, ServerName serverName, List<Mutation> mutations) throws IOException {
356 HRegionInfo copyOfMerged = new HRegionInfo(mergedRegion);
357
358
359 Put putOfMerged = MetaTableAccessor.makePutFromRegionInfo(copyOfMerged);
360 putOfMerged.add(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER, regionA.toByteArray());
361 putOfMerged.add(HConstants.CATALOG_FAMILY, HConstants.MERGEB_QUALIFIER, regionB.toByteArray());
362 mutations.add(putOfMerged);
363
364 Delete deleteA = MetaTableAccessor.makeDeleteFromRegionInfo(regionA);
365 Delete deleteB = MetaTableAccessor.makeDeleteFromRegionInfo(regionB);
366 mutations.add(deleteA);
367 mutations.add(deleteB);
368
369 addLocation(putOfMerged, serverName, 1);
370 }
371
372 public Put addLocation(final Put p, final ServerName sn, long openSeqNum) {
373 p.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, Bytes
374 .toBytes(sn.getHostAndPort()));
375 p.add(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER, Bytes.toBytes(sn
376 .getStartcode()));
377 p.add(HConstants.CATALOG_FAMILY, HConstants.SEQNUM_QUALIFIER, Bytes.toBytes(openSeqNum));
378 return p;
379 }
380
381 public HRegion stepsBeforePONR(final Server server, final RegionServerServices services,
382 boolean testing) throws IOException {
383 if (rmd == null) {
384 rmd =
385 server != null && server.getCoordinatedStateManager() != null ? ((BaseCoordinatedStateManager) server
386 .getCoordinatedStateManager()).getRegionMergeCoordination().getDefaultDetails()
387 : null;
388 }
389
390
391 if (useCoordination(server)) {
392 try {
393 ((BaseCoordinatedStateManager) server.getCoordinatedStateManager())
394 .getRegionMergeCoordination().startRegionMergeTransaction(mergedRegionInfo,
395 server.getServerName(), region_a.getRegionInfo(), region_b.getRegionInfo());
396 } catch (IOException e) {
397 throw new IOException("Failed to start region merge transaction for "
398 + this.mergedRegionInfo.getRegionNameAsString(), e);
399 }
400 } else if (services != null && !useCoordinationForAssignment) {
401 if (!services.reportRegionStateTransition(TransitionCode.READY_TO_MERGE,
402 mergedRegionInfo, region_a.getRegionInfo(), region_b.getRegionInfo())) {
403 throw new IOException("Failed to get ok from master to merge "
404 + region_a.getRegionInfo().getRegionNameAsString() + " and "
405 + region_b.getRegionInfo().getRegionNameAsString());
406 }
407 }
408 this.journal.add(JournalEntry.SET_MERGING);
409 if (useCoordination(server)) {
410
411
412
413 ((BaseCoordinatedStateManager) server.getCoordinatedStateManager())
414 .getRegionMergeCoordination().waitForRegionMergeTransaction(services, mergedRegionInfo,
415 region_a, region_b, rmd);
416 }
417
418 this.region_a.getRegionFileSystem().createMergesDir();
419 this.journal.add(JournalEntry.CREATED_MERGE_DIR);
420
421 Map<byte[], List<StoreFile>> hstoreFilesOfRegionA = closeAndOfflineRegion(
422 services, this.region_a, true, testing);
423 Map<byte[], List<StoreFile>> hstoreFilesOfRegionB = closeAndOfflineRegion(
424 services, this.region_b, false, testing);
425
426 assert hstoreFilesOfRegionA != null && hstoreFilesOfRegionB != null;
427
428
429
430
431
432
433 mergeStoreFiles(hstoreFilesOfRegionA, hstoreFilesOfRegionB);
434
435 if (useCoordination(server)) {
436 try {
437
438
439 ((BaseCoordinatedStateManager) server.getCoordinatedStateManager())
440 .getRegionMergeCoordination().confirmRegionMergeTransaction(this.mergedRegionInfo,
441 region_a.getRegionInfo(), region_b.getRegionInfo(), server.getServerName(), rmd);
442 } catch (IOException e) {
443 throw new IOException("Failed setting MERGING on "
444 + this.mergedRegionInfo.getRegionNameAsString(), e);
445 }
446 }
447
448
449
450
451
452 this.journal.add(JournalEntry.STARTED_MERGED_REGION_CREATION);
453 HRegion mergedRegion = createMergedRegionFromMerges(this.region_a,
454 this.region_b, this.mergedRegionInfo);
455 return mergedRegion;
456 }
457
458
459
460
461
462
463
464
465
466
467 HRegion createMergedRegionFromMerges(final HRegion a, final HRegion b,
468 final HRegionInfo mergedRegion) throws IOException {
469 return a.createMergedRegionFromMerges(mergedRegion, b);
470 }
471
472
473
474
475
476
477
478
479
480
481 private Map<byte[], List<StoreFile>> closeAndOfflineRegion(
482 final RegionServerServices services, final HRegion region,
483 final boolean isRegionA, final boolean testing) throws IOException {
484 Map<byte[], List<StoreFile>> hstoreFilesToMerge = null;
485 Exception exceptionToThrow = null;
486 try {
487 hstoreFilesToMerge = region.close(false);
488 } catch (Exception e) {
489 exceptionToThrow = e;
490 }
491 if (exceptionToThrow == null && hstoreFilesToMerge == null) {
492
493
494
495
496
497 exceptionToThrow = closedByOtherException;
498 }
499 if (exceptionToThrow != closedByOtherException) {
500 this.journal.add(isRegionA ? JournalEntry.CLOSED_REGION_A
501 : JournalEntry.CLOSED_REGION_B);
502 }
503 if (exceptionToThrow != null) {
504 if (exceptionToThrow instanceof IOException)
505 throw (IOException) exceptionToThrow;
506 throw new IOException(exceptionToThrow);
507 }
508
509 if (!testing) {
510 services.removeFromOnlineRegions(region, null);
511 }
512 this.journal.add(isRegionA ? JournalEntry.OFFLINED_REGION_A
513 : JournalEntry.OFFLINED_REGION_B);
514 return hstoreFilesToMerge;
515 }
516
517
518
519
520
521
522
523 public static HRegionInfo getMergedRegionInfo(final HRegionInfo a,
524 final HRegionInfo b) {
525 long rid = EnvironmentEdgeManager.currentTime();
526
527
528 if (rid < a.getRegionId() || rid < b.getRegionId()) {
529 LOG.warn("Clock skew; merging regions id are " + a.getRegionId()
530 + " and " + b.getRegionId() + ", but current time here is " + rid);
531 rid = Math.max(a.getRegionId(), b.getRegionId()) + 1;
532 }
533
534 byte[] startKey = null;
535 byte[] endKey = null;
536
537 if (a.compareTo(b) <= 0) {
538 startKey = a.getStartKey();
539 } else {
540 startKey = b.getStartKey();
541 }
542
543 if (Bytes.equals(a.getEndKey(), HConstants.EMPTY_BYTE_ARRAY)
544 || (!Bytes.equals(b.getEndKey(), HConstants.EMPTY_BYTE_ARRAY)
545 && Bytes.compareTo(a.getEndKey(), b.getEndKey()) > 0)) {
546 endKey = a.getEndKey();
547 } else {
548 endKey = b.getEndKey();
549 }
550
551
552 HRegionInfo mergedRegionInfo = new HRegionInfo(a.getTable(), startKey,
553 endKey, false, rid);
554 return mergedRegionInfo;
555 }
556
557
558
559
560
561
562
563
564
565 void openMergedRegion(final Server server,
566 final RegionServerServices services, HRegion merged) throws IOException {
567 boolean stopped = server != null && server.isStopped();
568 boolean stopping = services != null && services.isStopping();
569 if (stopped || stopping) {
570 LOG.info("Not opening merged region " + merged.getRegionNameAsString()
571 + " because stopping=" + stopping + ", stopped=" + stopped);
572 return;
573 }
574 HRegionInfo hri = merged.getRegionInfo();
575 LoggingProgressable reporter = server == null ? null
576 : new LoggingProgressable(hri, server.getConfiguration().getLong(
577 "hbase.regionserver.regionmerge.open.log.interval", 10000));
578 merged.openHRegion(reporter);
579
580 if (services != null) {
581 try {
582 if (useCoordinationForAssignment) {
583 services.postOpenDeployTasks(merged);
584 } else if (!services.reportRegionStateTransition(TransitionCode.MERGED,
585 mergedRegionInfo, region_a.getRegionInfo(), region_b.getRegionInfo())) {
586 throw new IOException("Failed to report merged region to master: "
587 + mergedRegionInfo.getShortNameToLog());
588 }
589 services.addToOnlineRegions(merged);
590 } catch (KeeperException ke) {
591 throw new IOException(ke);
592 }
593 }
594
595 }
596
597
598
599
600
601
602
603 private void mergeStoreFiles(
604 Map<byte[], List<StoreFile>> hstoreFilesOfRegionA,
605 Map<byte[], List<StoreFile>> hstoreFilesOfRegionB)
606 throws IOException {
607
608 HRegionFileSystem fs_a = this.region_a.getRegionFileSystem();
609 for (Map.Entry<byte[], List<StoreFile>> entry : hstoreFilesOfRegionA
610 .entrySet()) {
611 String familyName = Bytes.toString(entry.getKey());
612 for (StoreFile storeFile : entry.getValue()) {
613 fs_a.mergeStoreFile(this.mergedRegionInfo, familyName, storeFile,
614 this.mergesdir);
615 }
616 }
617
618 HRegionFileSystem fs_b = this.region_b.getRegionFileSystem();
619 for (Map.Entry<byte[], List<StoreFile>> entry : hstoreFilesOfRegionB
620 .entrySet()) {
621 String familyName = Bytes.toString(entry.getKey());
622 for (StoreFile storeFile : entry.getValue()) {
623 fs_b.mergeStoreFile(this.mergedRegionInfo, familyName, storeFile,
624 this.mergesdir);
625 }
626 }
627 }
628
629
630
631
632
633
634
635
636
637 @SuppressWarnings("deprecation")
638 public boolean rollback(final Server server,
639 final RegionServerServices services) throws IOException {
640 assert this.mergedRegionInfo != null;
641
642 if (rsCoprocessorHost != null) {
643 rsCoprocessorHost.preRollBackMerge(this.region_a, this.region_b);
644 }
645
646 boolean result = true;
647 ListIterator<JournalEntry> iterator = this.journal
648 .listIterator(this.journal.size());
649
650 while (iterator.hasPrevious()) {
651 JournalEntry je = iterator.previous();
652 switch (je) {
653
654 case SET_MERGING:
655 if (useCoordination(server)) {
656 ((BaseCoordinatedStateManager) server.getCoordinatedStateManager())
657 .getRegionMergeCoordination().clean(this.mergedRegionInfo);
658 } else if (services != null && !useCoordinationForAssignment
659 && !services.reportRegionStateTransition(TransitionCode.MERGE_REVERTED,
660 mergedRegionInfo, region_a.getRegionInfo(), region_b.getRegionInfo())) {
661 return false;
662 }
663 break;
664
665 case CREATED_MERGE_DIR:
666 this.region_a.writestate.writesEnabled = true;
667 this.region_b.writestate.writesEnabled = true;
668 this.region_a.getRegionFileSystem().cleanupMergesDir();
669 break;
670
671 case CLOSED_REGION_A:
672 try {
673
674
675
676
677
678 this.region_a.initialize();
679 } catch (IOException e) {
680 LOG.error("Failed rollbacking CLOSED_REGION_A of region "
681 + this.region_a.getRegionNameAsString(), e);
682 throw new RuntimeException(e);
683 }
684 break;
685
686 case OFFLINED_REGION_A:
687 if (services != null)
688 services.addToOnlineRegions(this.region_a);
689 break;
690
691 case CLOSED_REGION_B:
692 try {
693 this.region_b.initialize();
694 } catch (IOException e) {
695 LOG.error("Failed rollbacking CLOSED_REGION_A of region "
696 + this.region_b.getRegionNameAsString(), e);
697 throw new RuntimeException(e);
698 }
699 break;
700
701 case OFFLINED_REGION_B:
702 if (services != null)
703 services.addToOnlineRegions(this.region_b);
704 break;
705
706 case STARTED_MERGED_REGION_CREATION:
707 this.region_a.getRegionFileSystem().cleanupMergedRegion(
708 this.mergedRegionInfo);
709 break;
710
711 case PONR:
712
713
714 return false;
715
716 default:
717 throw new RuntimeException("Unhandled journal entry: " + je);
718 }
719 }
720
721 if (rsCoprocessorHost != null) {
722 rsCoprocessorHost.postRollBackMerge(this.region_a, this.region_b);
723 }
724
725 return result;
726 }
727
728 HRegionInfo getMergedRegionInfo() {
729 return this.mergedRegionInfo;
730 }
731
732
733 Path getMergesDir() {
734 return this.mergesdir;
735 }
736
737 private boolean useCoordination(final Server server) {
738 return server != null && useCoordinationForAssignment
739 && server.getCoordinatedStateManager() != null;
740 }
741
742
743
744
745
746
747
748
749
750
751
752 boolean hasMergeQualifierInMeta(final RegionServerServices services,
753 final byte[] regionName) throws IOException {
754 if (services == null) return false;
755
756
757 Pair<HRegionInfo, HRegionInfo> mergeRegions = MetaTableAccessor
758 .getRegionsFromMergeQualifier(services.getConnection(), regionName);
759 if (mergeRegions != null &&
760 (mergeRegions.getFirst() != null || mergeRegions.getSecond() != null)) {
761
762 return true;
763 }
764 return false;
765 }
766 }