1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import static org.apache.hadoop.hbase.executor.EventType.RS_ZK_REQUEST_REGION_SPLIT;
22 import static org.apache.hadoop.hbase.executor.EventType.RS_ZK_REGION_SPLIT;
23 import static org.apache.hadoop.hbase.executor.EventType.RS_ZK_REGION_SPLITTING;
24
25 import java.io.IOException;
26 import java.io.InterruptedIOException;
27 import java.util.ArrayList;
28 import java.util.List;
29 import java.util.ListIterator;
30 import java.util.Map;
31 import java.util.concurrent.Callable;
32 import java.util.concurrent.ExecutionException;
33 import java.util.concurrent.Executors;
34 import java.util.concurrent.Future;
35 import java.util.concurrent.ThreadFactory;
36 import java.util.concurrent.ThreadPoolExecutor;
37 import java.util.concurrent.TimeUnit;
38
39 import org.apache.commons.logging.Log;
40 import org.apache.commons.logging.LogFactory;
41 import org.apache.hadoop.fs.Path;
42 import org.apache.hadoop.hbase.classification.InterfaceAudience;
43 import org.apache.hadoop.hbase.HConstants;
44 import org.apache.hadoop.hbase.HRegionInfo;
45 import org.apache.hadoop.hbase.RegionTransition;
46 import org.apache.hadoop.hbase.Server;
47 import org.apache.hadoop.hbase.ServerName;
48 import org.apache.hadoop.hbase.catalog.CatalogTracker;
49 import org.apache.hadoop.hbase.catalog.MetaEditor;
50 import org.apache.hadoop.hbase.client.Mutation;
51 import org.apache.hadoop.hbase.client.Put;
52 import org.apache.hadoop.hbase.executor.EventType;
53 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
54 import org.apache.hadoop.hbase.util.Bytes;
55 import org.apache.hadoop.hbase.util.CancelableProgressable;
56 import org.apache.hadoop.hbase.util.ConfigUtil;
57 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
58 import org.apache.hadoop.hbase.util.FSUtils;
59 import org.apache.hadoop.hbase.util.HasThread;
60 import org.apache.hadoop.hbase.util.Pair;
61 import org.apache.hadoop.hbase.util.PairOfSameType;
62 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
63 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
64 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
65 import org.apache.zookeeper.KeeperException;
66 import org.apache.zookeeper.KeeperException.NodeExistsException;
67 import org.apache.zookeeper.data.Stat;
68
69 import com.google.common.util.concurrent.ThreadFactoryBuilder;
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94 @InterfaceAudience.Private
95 public class SplitTransaction {
96 private static final Log LOG = LogFactory.getLog(SplitTransaction.class);
97
98
99
100
101 private final HRegion parent;
102 private HRegionInfo hri_a;
103 private HRegionInfo hri_b;
104 private long fileSplitTimeout = 30000;
105 private int znodeVersion = -1;
106 boolean useZKForAssignment;
107
108
109
110
111 private final byte [] splitrow;
112
113
114
115
116
117
118 static enum JournalEntryType {
119
120
121
122 STARTED,
123
124
125
126 PREPARED,
127
128
129
130 BEFORE_PRE_SPLIT_HOOK,
131
132
133
134 AFTER_PRE_SPLIT_HOOK,
135
136
137
138 SET_SPLITTING_IN_ZK,
139
140
141
142 CREATE_SPLIT_DIR,
143
144
145
146 CLOSED_PARENT_REGION,
147
148
149
150 OFFLINED_PARENT,
151
152
153
154 STARTED_REGION_A_CREATION,
155
156
157
158 STARTED_REGION_B_CREATION,
159
160
161
162 OPENED_REGION_A,
163
164
165
166 OPENED_REGION_B,
167
168
169
170 BEFORE_POST_SPLIT_HOOK,
171
172
173
174 AFTER_POST_SPLIT_HOOK,
175
176
177
178
179
180 PONR
181 }
182
183 static class JournalEntry {
184 private JournalEntryType type;
185 private long timestamp;
186
187 public JournalEntry(JournalEntryType type) {
188 this(type, EnvironmentEdgeManager.currentTimeMillis());
189 }
190
191 public JournalEntry(JournalEntryType type, long timestamp) {
192 this.type = type;
193 this.timestamp = timestamp;
194 }
195
196 @Override
197 public String toString() {
198 StringBuilder sb = new StringBuilder();
199 sb.append(type);
200 sb.append(" at ");
201 sb.append(timestamp);
202 return sb.toString();
203 }
204 }
205
206
207
208
209 private final List<JournalEntry> journal = new ArrayList<JournalEntry>();
210
211
212
213
214
215
216 public SplitTransaction(final HRegion r, final byte [] splitrow) {
217 this.parent = r;
218 this.splitrow = splitrow;
219 this.journal.add(new JournalEntry(JournalEntryType.STARTED));
220 this.useZKForAssignment = ConfigUtil.useZKForAssignment(r.getBaseConf());
221 }
222
223
224
225
226
227
228 public boolean prepare() {
229 if (!this.parent.isSplittable()) return false;
230
231 if (this.splitrow == null) return false;
232 HRegionInfo hri = this.parent.getRegionInfo();
233 parent.prepareToSplit();
234
235 byte [] startKey = hri.getStartKey();
236 byte [] endKey = hri.getEndKey();
237 if (Bytes.equals(startKey, splitrow) ||
238 !this.parent.getRegionInfo().containsRow(splitrow)) {
239 LOG.info("Split row is not inside region key range or is equal to " +
240 "startkey: " + Bytes.toStringBinary(this.splitrow));
241 return false;
242 }
243 long rid = getDaughterRegionIdTimestamp(hri);
244 this.hri_a = new HRegionInfo(hri.getTable(), startKey, this.splitrow, false, rid);
245 this.hri_b = new HRegionInfo(hri.getTable(), this.splitrow, endKey, false, rid);
246 this.journal.add(new JournalEntry(JournalEntryType.PREPARED));
247 return true;
248 }
249
250
251
252
253
254
255 private static long getDaughterRegionIdTimestamp(final HRegionInfo hri) {
256 long rid = EnvironmentEdgeManager.currentTimeMillis();
257
258
259 if (rid < hri.getRegionId()) {
260 LOG.warn("Clock skew; parent regions id is " + hri.getRegionId() +
261 " but current time here is " + rid);
262 rid = hri.getRegionId() + 1;
263 }
264 return rid;
265 }
266
267 private static IOException closedByOtherException = new IOException(
268 "Failed to close region: already closed by another thread");
269
270
271
272
273
274
275
276
277
278
279
280 final RegionServerServices services) throws IOException {
281 LOG.info("Starting split of region " + this.parent);
282 if ((server != null && server.isStopped()) ||
283 (services != null && services.isStopping())) {
284 throw new IOException("Server is stopped or stopping");
285 }
286 assert !this.parent.lock.writeLock().isHeldByCurrentThread():
287 "Unsafe to hold write lock while performing RPCs";
288
289 journal.add(new JournalEntry(JournalEntryType.BEFORE_PRE_SPLIT_HOOK));
290
291
292 if (this.parent.getCoprocessorHost() != null) {
293
294 this.parent.getCoprocessorHost().preSplit();
295 this.parent.getCoprocessorHost().preSplit(this.splitrow);
296 }
297
298 journal.add(new JournalEntry(JournalEntryType.AFTER_PRE_SPLIT_HOOK));
299
300
301 boolean testing = server == null? true:
302 server.getConfiguration().getBoolean("hbase.testing.nocluster", false);
303 this.fileSplitTimeout = testing ? this.fileSplitTimeout :
304 server.getConfiguration().getLong("hbase.regionserver.fileSplitTimeout",
305 this.fileSplitTimeout);
306
307 PairOfSameType<HRegion> daughterRegions = stepsBeforePONR(server, services, testing);
308
309 List<Mutation> metaEntries = new ArrayList<Mutation>();
310 if (this.parent.getCoprocessorHost() != null) {
311 if (this.parent.getCoprocessorHost().
312 preSplitBeforePONR(this.splitrow, metaEntries)) {
313 throw new IOException("Coprocessor bypassing region "
314 + this.parent.getRegionNameAsString() + " split.");
315 }
316 try {
317 for (Mutation p : metaEntries) {
318 HRegionInfo.parseRegionName(p.getRow());
319 }
320 } catch (IOException e) {
321 LOG.error("Row key of mutation from coprossor is not parsable as region name."
322 + "Mutations from coprocessor should only for hbase:meta table.");
323 throw e;
324 }
325 }
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342 this.journal.add(new JournalEntry(JournalEntryType.PONR));
343
344
345
346
347
348
349 if (!testing && useZKForAssignment) {
350 if (metaEntries == null || metaEntries.isEmpty()) {
351 MetaEditor.splitRegion(server.getCatalogTracker(), parent.getRegionInfo(), daughterRegions
352 .getFirst().getRegionInfo(), daughterRegions.getSecond().getRegionInfo(), server
353 .getServerName());
354 } else {
355 offlineParentInMetaAndputMetaEntries(server.getCatalogTracker(), parent.getRegionInfo(),
356 daughterRegions.getFirst().getRegionInfo(), daughterRegions.getSecond().getRegionInfo(),
357 server.getServerName(), metaEntries);
358 }
359 } else if (services != null && !useZKForAssignment) {
360 if (!services.reportRegionStateTransition(TransitionCode.SPLIT_PONR, parent.getRegionInfo(),
361 hri_a, hri_b)) {
362
363 throw new IOException("Failed to notify master that split passed PONR: "
364 + parent.getRegionInfo().getRegionNameAsString());
365 }
366 }
367 return daughterRegions;
368 }
369
370 public PairOfSameType<HRegion> stepsBeforePONR(final Server server,
371 final RegionServerServices services, boolean testing) throws IOException {
372
373
374 if (server != null && server.getZooKeeper() != null && useZKForAssignment) {
375 try {
376 createNodeSplitting(server.getZooKeeper(),
377 parent.getRegionInfo(), server.getServerName(), hri_a, hri_b);
378 } catch (KeeperException e) {
379 throw new IOException("Failed creating PENDING_SPLIT znode on " +
380 this.parent.getRegionNameAsString(), e);
381 }
382 } else if (services != null && !useZKForAssignment) {
383 if (!services.reportRegionStateTransition(TransitionCode.READY_TO_SPLIT,
384 parent.getRegionInfo(), hri_a, hri_b)) {
385 throw new IOException("Failed to get ok from master to split "
386 + parent.getRegionNameAsString());
387 }
388 }
389 this.journal.add(new JournalEntry(JournalEntryType.SET_SPLITTING_IN_ZK));
390 if (server != null && server.getZooKeeper() != null && useZKForAssignment) {
391
392
393
394 znodeVersion = getZKNode(server, services);
395 }
396
397 this.parent.getRegionFileSystem().createSplitsDir();
398 this.journal.add(new JournalEntry(JournalEntryType.CREATE_SPLIT_DIR));
399
400 Map<byte[], List<StoreFile>> hstoreFilesToSplit = null;
401 Exception exceptionToThrow = null;
402 try{
403 hstoreFilesToSplit = this.parent.close(false);
404 } catch (Exception e) {
405 exceptionToThrow = e;
406 }
407 if (exceptionToThrow == null && hstoreFilesToSplit == null) {
408
409
410
411
412
413 exceptionToThrow = closedByOtherException;
414 }
415 if (exceptionToThrow != closedByOtherException) {
416 this.journal.add(new JournalEntry(JournalEntryType.CLOSED_PARENT_REGION));
417 }
418 if (exceptionToThrow != null) {
419 if (exceptionToThrow instanceof IOException) throw (IOException)exceptionToThrow;
420 throw new IOException(exceptionToThrow);
421 }
422 if (!testing) {
423 services.removeFromOnlineRegions(this.parent, null);
424 }
425 this.journal.add(new JournalEntry(JournalEntryType.OFFLINED_PARENT));
426
427
428
429
430
431
432
433 Pair<Integer, Integer> expectedReferences = splitStoreFiles(hstoreFilesToSplit);
434
435
436
437
438
439 this.journal.add(new JournalEntry(JournalEntryType.STARTED_REGION_A_CREATION));
440 assertReferenceFileCount(expectedReferences.getFirst(),
441 this.parent.getRegionFileSystem().getSplitsDir(this.hri_a));
442 HRegion a = this.parent.createDaughterRegionFromSplits(this.hri_a);
443 assertReferenceFileCount(expectedReferences.getFirst(),
444 new Path(this.parent.getRegionFileSystem().getTableDir(), this.hri_a.getEncodedName()));
445
446
447 this.journal.add(new JournalEntry(JournalEntryType.STARTED_REGION_B_CREATION));
448 assertReferenceFileCount(expectedReferences.getSecond(),
449 this.parent.getRegionFileSystem().getSplitsDir(this.hri_b));
450 HRegion b = this.parent.createDaughterRegionFromSplits(this.hri_b);
451 assertReferenceFileCount(expectedReferences.getSecond(),
452 new Path(this.parent.getRegionFileSystem().getTableDir(), this.hri_b.getEncodedName()));
453
454 return new PairOfSameType<HRegion>(a, b);
455 }
456
457 void assertReferenceFileCount(int expectedReferenceFileCount, Path dir)
458 throws IOException {
459 if (expectedReferenceFileCount != 0 &&
460 expectedReferenceFileCount != FSUtils.getRegionReferenceFileCount(this.parent.getFilesystem(), dir)) {
461 throw new IOException("Failing split. Expected reference file count isn't equal.");
462 }
463 }
464
465
466
467
468
469
470
471
472
473
474
475
476 final RegionServerServices services, HRegion a, HRegion b)
477 throws IOException {
478 boolean stopped = server != null && server.isStopped();
479 boolean stopping = services != null && services.isStopping();
480
481 if (stopped || stopping) {
482 LOG.info("Not opening daughters " +
483 b.getRegionInfo().getRegionNameAsString() +
484 " and " +
485 a.getRegionInfo().getRegionNameAsString() +
486 " because stopping=" + stopping + ", stopped=" + stopped);
487 } else {
488
489 DaughterOpener aOpener = new DaughterOpener(server, a);
490 DaughterOpener bOpener = new DaughterOpener(server, b);
491 aOpener.start();
492 bOpener.start();
493 try {
494 aOpener.join();
495 if (aOpener.getException() == null) {
496 journal.add(new JournalEntry(JournalEntryType.OPENED_REGION_A));
497 }
498 bOpener.join();
499 if (bOpener.getException() == null) {
500 journal.add(new JournalEntry(JournalEntryType.OPENED_REGION_B));
501 }
502 } catch (InterruptedException e) {
503 throw (InterruptedIOException)new InterruptedIOException().initCause(e);
504 }
505 if (aOpener.getException() != null) {
506 throw new IOException("Failed " +
507 aOpener.getName(), aOpener.getException());
508 }
509 if (bOpener.getException() != null) {
510 throw new IOException("Failed " +
511 bOpener.getName(), bOpener.getException());
512 }
513 if (services != null) {
514 try {
515 if (useZKForAssignment) {
516
517 services.postOpenDeployTasks(b, server.getCatalogTracker());
518 } else if (!services.reportRegionStateTransition(TransitionCode.SPLIT,
519 parent.getRegionInfo(), hri_a, hri_b)) {
520 throw new IOException("Failed to report split region to master: "
521 + parent.getRegionInfo().getShortNameToLog());
522 }
523
524 services.addToOnlineRegions(b);
525 if (useZKForAssignment) {
526 services.postOpenDeployTasks(a, server.getCatalogTracker());
527 }
528 services.addToOnlineRegions(a);
529 } catch (KeeperException ke) {
530 throw new IOException(ke);
531 }
532 }
533 }
534 }
535
536
537
538
539
540
541
542
543
544
545
546
547 final RegionServerServices services, HRegion a, HRegion b)
548 throws IOException {
549
550 if (server != null && server.getZooKeeper() != null) {
551 try {
552 this.znodeVersion = transitionSplittingNode(server.getZooKeeper(),
553 parent.getRegionInfo(), a.getRegionInfo(), b.getRegionInfo(),
554 server.getServerName(), this.znodeVersion,
555 RS_ZK_REGION_SPLITTING, RS_ZK_REGION_SPLIT);
556
557 int spins = 0;
558
559
560
561 do {
562 if (spins % 10 == 0) {
563 LOG.debug("Still waiting on the master to process the split for " +
564 this.parent.getRegionInfo().getEncodedName());
565 }
566 Thread.sleep(100);
567
568 this.znodeVersion = transitionSplittingNode(server.getZooKeeper(),
569 parent.getRegionInfo(), a.getRegionInfo(), b.getRegionInfo(),
570 server.getServerName(), this.znodeVersion,
571 RS_ZK_REGION_SPLIT, RS_ZK_REGION_SPLIT);
572 spins++;
573 } while (this.znodeVersion != -1 && !server.isStopped()
574 && !services.isStopping());
575 } catch (Exception e) {
576 if (e instanceof InterruptedException) {
577 Thread.currentThread().interrupt();
578 }
579 throw new IOException("Failed telling master about split", e);
580 }
581 }
582
583
584
585
586
587
588 }
589
590
591
592
593
594
595
596
597
598 private int getZKNode(final Server server,
599 final RegionServerServices services) throws IOException {
600
601 try {
602 int spins = 0;
603 Stat stat = new Stat();
604 ZooKeeperWatcher zkw = server.getZooKeeper();
605 ServerName expectedServer = server.getServerName();
606 String node = parent.getRegionInfo().getEncodedName();
607 while (!(server.isStopped() || services.isStopping())) {
608 if (spins % 5 == 0) {
609 LOG.debug("Still waiting for master to process "
610 + "the pending_split for " + node);
611 transitionSplittingNode(zkw, parent.getRegionInfo(),
612 hri_a, hri_b, expectedServer, -1, RS_ZK_REQUEST_REGION_SPLIT,
613 RS_ZK_REQUEST_REGION_SPLIT);
614 }
615 Thread.sleep(100);
616 spins++;
617 byte [] data = ZKAssign.getDataNoWatch(zkw, node, stat);
618 if (data == null) {
619 throw new IOException("Data is null, splitting node "
620 + node + " no longer exists");
621 }
622 RegionTransition rt = RegionTransition.parseFrom(data);
623 EventType et = rt.getEventType();
624 if (et == RS_ZK_REGION_SPLITTING) {
625 ServerName serverName = rt.getServerName();
626 if (!serverName.equals(expectedServer)) {
627 throw new IOException("Splitting node " + node + " is for "
628 + serverName + ", not us " + expectedServer);
629 }
630 byte [] payloadOfSplitting = rt.getPayload();
631 List<HRegionInfo> splittingRegions = HRegionInfo.parseDelimitedFrom(
632 payloadOfSplitting, 0, payloadOfSplitting.length);
633 assert splittingRegions.size() == 2;
634 HRegionInfo a = splittingRegions.get(0);
635 HRegionInfo b = splittingRegions.get(1);
636 if (!(hri_a.equals(a) && hri_b.equals(b))) {
637 throw new IOException("Splitting node " + node + " is for " + a + ", "
638 + b + ", not expected daughters: " + hri_a + ", " + hri_b);
639 }
640
641 return stat.getVersion();
642 }
643 if (et != RS_ZK_REQUEST_REGION_SPLIT) {
644 throw new IOException("Splitting node " + node
645 + " moved out of splitting to " + et);
646 }
647 }
648
649 throw new IOException("Server is "
650 + (services.isStopping() ? "stopping" : "stopped"));
651 } catch (Exception e) {
652 if (e instanceof InterruptedException) {
653 Thread.currentThread().interrupt();
654 }
655 throw new IOException("Failed getting SPLITTING znode on "
656 + parent.getRegionNameAsString(), e);
657 }
658 }
659
660
661
662
663
664
665
666
667
668
669
670
671 public PairOfSameType<HRegion> execute(final Server server,
672 final RegionServerServices services)
673 throws IOException {
674 useZKForAssignment =
675 server == null ? true : ConfigUtil.useZKForAssignment(server.getConfiguration());
676 PairOfSameType<HRegion> regions = createDaughters(server, services);
677 if (this.parent.getCoprocessorHost() != null) {
678 this.parent.getCoprocessorHost().preSplitAfterPONR();
679 }
680 return stepsAfterPONR(server, services, regions);
681 }
682
683 public PairOfSameType<HRegion> stepsAfterPONR(final Server server,
684 final RegionServerServices services, PairOfSameType<HRegion> regions)
685 throws IOException {
686 openDaughters(server, services, regions.getFirst(), regions.getSecond());
687 if (server != null && server.getZooKeeper() != null && useZKForAssignment) {
688 transitionZKNode(server, services, regions.getFirst(), regions.getSecond());
689 }
690 journal.add(new JournalEntry(JournalEntryType.BEFORE_POST_SPLIT_HOOK));
691
692 if (this.parent.getCoprocessorHost() != null) {
693 this.parent.getCoprocessorHost().postSplit(regions.getFirst(), regions.getSecond());
694 }
695 journal.add(new JournalEntry(JournalEntryType.AFTER_POST_SPLIT_HOOK));
696 return regions;
697 }
698
699 private void offlineParentInMetaAndputMetaEntries(CatalogTracker catalogTracker,
700 HRegionInfo parent, HRegionInfo splitA, HRegionInfo splitB,
701 ServerName serverName, List<Mutation> metaEntries) throws IOException {
702 List<Mutation> mutations = metaEntries;
703 HRegionInfo copyOfParent = new HRegionInfo(parent);
704 copyOfParent.setOffline(true);
705 copyOfParent.setSplit(true);
706
707
708 Put putParent = MetaEditor.makePutFromRegionInfo(copyOfParent);
709 MetaEditor.addDaughtersToPut(putParent, splitA, splitB);
710 mutations.add(putParent);
711
712
713 Put putA = MetaEditor.makePutFromRegionInfo(splitA);
714 Put putB = MetaEditor.makePutFromRegionInfo(splitB);
715
716 addLocation(putA, serverName, 1);
717 addLocation(putB, serverName, 1);
718 mutations.add(putA);
719 mutations.add(putB);
720 MetaEditor.mutateMetaTable(catalogTracker, mutations);
721 }
722
723 public Put addLocation(final Put p, final ServerName sn, long openSeqNum) {
724 p.addImmutable(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
725 Bytes.toBytes(sn.getHostAndPort()));
726 p.addImmutable(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER,
727 Bytes.toBytes(sn.getStartcode()));
728 p.addImmutable(HConstants.CATALOG_FAMILY, HConstants.SEQNUM_QUALIFIER,
729 Bytes.toBytes(openSeqNum));
730 return p;
731 }
732
733
734
735
736
737 class DaughterOpener extends HasThread {
738 private final Server server;
739 private final HRegion r;
740 private Throwable t = null;
741
742 DaughterOpener(final Server s, final HRegion r) {
743 super((s == null? "null-services": s.getServerName()) +
744 "-daughterOpener=" + r.getRegionInfo().getEncodedName());
745 setDaemon(true);
746 this.server = s;
747 this.r = r;
748 }
749
750
751
752
753
754 Throwable getException() {
755 return this.t;
756 }
757
758 @Override
759 public void run() {
760 try {
761 openDaughterRegion(this.server, r);
762 } catch (Throwable t) {
763 this.t = t;
764 }
765 }
766 }
767
768
769
770
771
772
773
774
775 void openDaughterRegion(final Server server, final HRegion daughter)
776 throws IOException, KeeperException {
777 HRegionInfo hri = daughter.getRegionInfo();
778 LoggingProgressable reporter = server == null ? null
779 : new LoggingProgressable(hri, server.getConfiguration().getLong(
780 "hbase.regionserver.split.daughter.open.log.interval", 10000));
781 daughter.openHRegion(reporter);
782 }
783
784 static class LoggingProgressable implements CancelableProgressable {
785 private final HRegionInfo hri;
786 private long lastLog = -1;
787 private final long interval;
788
789 LoggingProgressable(final HRegionInfo hri, final long interval) {
790 this.hri = hri;
791 this.interval = interval;
792 }
793
794 @Override
795 public boolean progress() {
796 long now = EnvironmentEdgeManager.currentTimeMillis();
797 if (now - lastLog > this.interval) {
798 LOG.info("Opening " + this.hri.getRegionNameAsString());
799 this.lastLog = now;
800 }
801 return true;
802 }
803 }
804
805
806
807
808
809
810
811
812 private Pair<Integer, Integer> splitStoreFiles(
813 final Map<byte[], List<StoreFile>> hstoreFilesToSplit)
814 throws IOException {
815 if (hstoreFilesToSplit == null) {
816
817 throw new IOException("Close returned empty list of StoreFiles");
818 }
819
820
821
822 int nbFiles = 0;
823 for (Map.Entry<byte[], List<StoreFile>> entry: hstoreFilesToSplit.entrySet()) {
824 nbFiles += entry.getValue().size();
825 }
826 if (nbFiles == 0) {
827
828 return new Pair<Integer, Integer>(0,0);
829 }
830
831
832 int defMaxThreads = Math.min(parent.conf.getInt(HStore.BLOCKING_STOREFILES_KEY,
833 HStore.DEFAULT_BLOCKING_STOREFILE_COUNT),
834 Runtime.getRuntime().availableProcessors());
835
836 int maxThreads = Math.min(parent.conf.getInt(HConstants.REGION_SPLIT_THREADS_MAX,
837 defMaxThreads), nbFiles);
838 LOG.info("Preparing to split " + nbFiles + " storefiles for region " + this.parent +
839 " using " + maxThreads + " threads");
840 ThreadFactoryBuilder builder = new ThreadFactoryBuilder();
841 builder.setNameFormat("StoreFileSplitter-%1$d");
842 ThreadFactory factory = builder.build();
843 ThreadPoolExecutor threadPool =
844 (ThreadPoolExecutor) Executors.newFixedThreadPool(maxThreads, factory);
845 List<Future<Pair<Path,Path>>> futures = new ArrayList<Future<Pair<Path,Path>>> (nbFiles);
846
847
848 for (Map.Entry<byte[], List<StoreFile>> entry: hstoreFilesToSplit.entrySet()) {
849 for (StoreFile sf: entry.getValue()) {
850 StoreFileSplitter sfs = new StoreFileSplitter(entry.getKey(), sf);
851 futures.add(threadPool.submit(sfs));
852 }
853 }
854
855 threadPool.shutdown();
856
857
858 try {
859 boolean stillRunning = !threadPool.awaitTermination(
860 this.fileSplitTimeout, TimeUnit.MILLISECONDS);
861 if (stillRunning) {
862 threadPool.shutdownNow();
863
864 while (!threadPool.isTerminated()) {
865 Thread.sleep(50);
866 }
867 throw new IOException("Took too long to split the" +
868 " files and create the references, aborting split");
869 }
870 } catch (InterruptedException e) {
871 throw (InterruptedIOException)new InterruptedIOException().initCause(e);
872 }
873
874 int created_a = 0;
875 int created_b = 0;
876
877 for (Future<Pair<Path, Path>> future : futures) {
878 try {
879 Pair<Path, Path> p = future.get();
880 created_a += p.getFirst() != null ? 1 : 0;
881 created_b += p.getSecond() != null ? 1 : 0;
882 } catch (InterruptedException e) {
883 throw (InterruptedIOException) new InterruptedIOException().initCause(e);
884 } catch (ExecutionException e) {
885 throw new IOException(e);
886 }
887 }
888
889 if (LOG.isDebugEnabled()) {
890 LOG.debug("Split storefiles for region " + this.parent + " Daugther A: " + created_a
891 + " storefiles, Daugther B: " + created_b + " storefiles.");
892 }
893 return new Pair<Integer, Integer>(created_a, created_b);
894 }
895
896 private Pair<Path, Path> splitStoreFile(final byte[] family, final StoreFile sf)
897 throws IOException {
898 if (LOG.isDebugEnabled()) {
899 LOG.debug("Splitting started for store file: " + sf.getPath() + " for region: " +
900 this.parent);
901 }
902 HRegionFileSystem fs = this.parent.getRegionFileSystem();
903 String familyName = Bytes.toString(family);
904 Path path_a =
905 fs.splitStoreFile(this.hri_a, familyName, sf, this.splitrow, false,
906 this.parent.getSplitPolicy());
907 Path path_b =
908 fs.splitStoreFile(this.hri_b, familyName, sf, this.splitrow, true,
909 this.parent.getSplitPolicy());
910 if (LOG.isDebugEnabled()) {
911 LOG.debug("Splitting complete for store file: " + sf.getPath() + " for region: " +
912 this.parent);
913 }
914 return new Pair<Path,Path>(path_a, path_b);
915 }
916
917
918
919
920
921 class StoreFileSplitter implements Callable<Pair<Path,Path>> {
922 private final byte[] family;
923 private final StoreFile sf;
924
925
926
927
928
929
930 public StoreFileSplitter(final byte[] family, final StoreFile sf) {
931 this.sf = sf;
932 this.family = family;
933 }
934
935 public Pair<Path,Path> call() throws IOException {
936 return splitStoreFile(family, sf);
937 }
938 }
939
940
941
942
943
944
945
946
947 @SuppressWarnings("deprecation")
948 public boolean rollback(final Server server, final RegionServerServices services)
949 throws IOException {
950
951 if (this.parent.getCoprocessorHost() != null) {
952 this.parent.getCoprocessorHost().preRollBackSplit();
953 }
954
955 boolean result = true;
956 ListIterator<JournalEntry> iterator =
957 this.journal.listIterator(this.journal.size());
958
959 while (iterator.hasPrevious()) {
960 JournalEntry je = iterator.previous();
961 switch(je.type) {
962
963 case SET_SPLITTING_IN_ZK:
964 if (server != null && server.getZooKeeper() != null && useZKForAssignment) {
965 cleanZK(server, this.parent.getRegionInfo());
966 } else if (services != null
967 && !useZKForAssignment
968 && !services.reportRegionStateTransition(TransitionCode.SPLIT_REVERTED,
969 parent.getRegionInfo(), hri_a, hri_b)) {
970 return false;
971 }
972 break;
973
974 case CREATE_SPLIT_DIR:
975 this.parent.writestate.writesEnabled = true;
976 this.parent.getRegionFileSystem().cleanupSplitsDir();
977 break;
978
979 case CLOSED_PARENT_REGION:
980 try {
981
982
983
984
985
986 this.parent.initialize();
987 } catch (IOException e) {
988 LOG.error("Failed rollbacking CLOSED_PARENT_REGION of region " +
989 this.parent.getRegionNameAsString(), e);
990 throw new RuntimeException(e);
991 }
992 break;
993
994 case STARTED_REGION_A_CREATION:
995 this.parent.getRegionFileSystem().cleanupDaughterRegion(this.hri_a);
996 break;
997
998 case STARTED_REGION_B_CREATION:
999 this.parent.getRegionFileSystem().cleanupDaughterRegion(this.hri_b);
1000 break;
1001
1002 case OFFLINED_PARENT:
1003 if (services != null) services.addToOnlineRegions(this.parent);
1004 break;
1005
1006 case PONR:
1007
1008
1009
1010
1011 return false;
1012
1013
1014 case STARTED:
1015 case PREPARED:
1016 case BEFORE_PRE_SPLIT_HOOK:
1017 case AFTER_PRE_SPLIT_HOOK:
1018 case BEFORE_POST_SPLIT_HOOK:
1019 case AFTER_POST_SPLIT_HOOK:
1020 case OPENED_REGION_A:
1021 case OPENED_REGION_B:
1022 break;
1023
1024 default:
1025 throw new RuntimeException("Unhandled journal entry: " + je);
1026 }
1027 }
1028
1029 if (this.parent.getCoprocessorHost() != null) {
1030 this.parent.getCoprocessorHost().postRollBackSplit();
1031 }
1032 return result;
1033 }
1034
1035 HRegionInfo getFirstDaughter() {
1036 return hri_a;
1037 }
1038
1039 HRegionInfo getSecondDaughter() {
1040 return hri_b;
1041 }
1042
1043 private static void cleanZK(final Server server, final HRegionInfo hri) {
1044 try {
1045
1046 if (!ZKAssign.deleteNode(server.getZooKeeper(), hri.getEncodedName(),
1047 RS_ZK_REQUEST_REGION_SPLIT, server.getServerName())) {
1048 ZKAssign.deleteNode(server.getZooKeeper(), hri.getEncodedName(),
1049 RS_ZK_REGION_SPLITTING, server.getServerName());
1050 }
1051 } catch (KeeperException.NoNodeException e) {
1052 LOG.info("Failed cleanup zk node of " + hri.getRegionNameAsString(), e);
1053 } catch (KeeperException e) {
1054 server.abort("Failed cleanup of " + hri.getRegionNameAsString(), e);
1055 }
1056 }
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071 public static void createNodeSplitting(final ZooKeeperWatcher zkw, final HRegionInfo region,
1072 final ServerName serverName, final HRegionInfo a,
1073 final HRegionInfo b) throws KeeperException, IOException {
1074 LOG.debug(zkw.prefix("Creating ephemeral node for " +
1075 region.getEncodedName() + " in PENDING_SPLIT state"));
1076 byte [] payload = HRegionInfo.toDelimitedByteArray(a, b);
1077 RegionTransition rt = RegionTransition.createRegionTransition(
1078 RS_ZK_REQUEST_REGION_SPLIT, region.getRegionName(), serverName, payload);
1079 String node = ZKAssign.getNodeName(zkw, region.getEncodedName());
1080 if (!ZKUtil.createEphemeralNodeAndWatch(zkw, node, rt.toByteArray())) {
1081 throw new IOException("Failed create of ephemeral " + node);
1082 }
1083 }
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119 public static int transitionSplittingNode(ZooKeeperWatcher zkw,
1120 HRegionInfo parent, HRegionInfo a, HRegionInfo b, ServerName serverName,
1121 final int znodeVersion, final EventType beginState,
1122 final EventType endState) throws KeeperException, IOException {
1123 byte [] payload = HRegionInfo.toDelimitedByteArray(a, b);
1124 return ZKAssign.transitionNode(zkw, parent, serverName,
1125 beginState, endState, znodeVersion, payload);
1126 }
1127
1128 List<JournalEntry> getJournal() {
1129 return journal;
1130 }
1131 }