View Javadoc

1   /**
2    * Copyright The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one or more
5    * contributor license agreements. See the NOTICE file distributed with this
6    * work for additional information regarding copyright ownership. The ASF
7    * licenses this file to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance with the License.
9    * You may obtain a copy of the License at
10   *
11   * http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
15   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
16   * License for the specific language governing permissions and limitationsME
17   * under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import java.io.IOException;
22  import java.util.ArrayList;
23  import java.util.List;
24  import java.util.ListIterator;
25  import java.util.Map;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.hbase.classification.InterfaceAudience;
30  import org.apache.hadoop.fs.Path;
31  import org.apache.hadoop.hbase.HConstants;
32  import org.apache.hadoop.hbase.HRegionInfo;
33  import org.apache.hadoop.hbase.MetaMutationAnnotation;
34  import org.apache.hadoop.hbase.Server;
35  import org.apache.hadoop.hbase.ServerName;
36  import org.apache.hadoop.hbase.MetaTableAccessor;
37  import org.apache.hadoop.hbase.client.Delete;
38  import org.apache.hadoop.hbase.client.HConnection;
39  import org.apache.hadoop.hbase.client.Mutation;
40  import org.apache.hadoop.hbase.client.Put;
41  import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager;
42  import org.apache.hadoop.hbase.coordination.RegionMergeCoordination.RegionMergeDetails;
43  import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
44  import org.apache.hadoop.hbase.regionserver.SplitTransaction.LoggingProgressable;
45  import org.apache.hadoop.hbase.util.Bytes;
46  import org.apache.hadoop.hbase.util.ConfigUtil;
47  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
48  import org.apache.hadoop.hbase.util.Pair;
49  import org.apache.zookeeper.KeeperException;
50  
51  /**
52   * Executes region merge as a "transaction". It is similar with
53   * SplitTransaction. Call {@link #prepare(RegionServerServices)} to setup the
54   * transaction, {@link #execute(Server, RegionServerServices)} to run the
55   * transaction and {@link #rollback(Server, RegionServerServices)} to cleanup if
56   * execute fails.
57   * 
58   * <p>
59   * Here is an example of how you would use this class:
60   * 
61   * <pre>
62   *  RegionMergeTransaction mt = new RegionMergeTransaction(this.conf, parent, midKey)
63   *  if (!mt.prepare(services)) return;
64   *  try {
65   *    mt.execute(server, services);
66   *  } catch (IOException ioe) {
67   *    try {
68   *      mt.rollback(server, services);
69   *      return;
70   *    } catch (RuntimeException e) {
71   *      myAbortable.abort("Failed merge, abort");
72   *    }
73   *  }
74   * </Pre>
75   * <p>
76   * This class is not thread safe. Caller needs ensure merge is run by one thread
77   * only.
78   */
79  @InterfaceAudience.Private
80  public class RegionMergeTransaction {
81    private static final Log LOG = LogFactory.getLog(RegionMergeTransaction.class);
82  
83    // Merged region info
84    private HRegionInfo mergedRegionInfo;
85    // region_a sorts before region_b
86    private final HRegion region_a;
87    private final HRegion region_b;
88    // merges dir is under region_a
89    private final Path mergesdir;
90    // We only merge adjacent regions if forcible is false
91    private final boolean forcible;
92    private boolean useCoordinationForAssignment;
93  
94    /**
95     * Types to add to the transaction journal. Each enum is a step in the merge
96     * transaction. Used to figure how much we need to rollback.
97     */
98    enum JournalEntry {
99      /**
100      * Set region as in transition, set it into MERGING state.
101      */
102     SET_MERGING,
103     /**
104      * We created the temporary merge data directory.
105      */
106     CREATED_MERGE_DIR,
107     /**
108      * Closed the merging region A.
109      */
110     CLOSED_REGION_A,
111     /**
112      * The merging region A has been taken out of the server's online regions list.
113      */
114     OFFLINED_REGION_A,
115     /**
116      * Closed the merging region B.
117      */
118     CLOSED_REGION_B,
119     /**
120      * The merging region B has been taken out of the server's online regions list.
121      */
122     OFFLINED_REGION_B,
123     /**
124      * Started in on creation of the merged region.
125      */
126     STARTED_MERGED_REGION_CREATION,
127     /**
128      * Point of no return. If we got here, then transaction is not recoverable
129      * other than by crashing out the regionserver.
130      */
131     PONR
132   }
133 
134   /*
135    * Journal of how far the merge transaction has progressed.
136    */
137   private final List<JournalEntry> journal = new ArrayList<JournalEntry>();
138 
139   private static IOException closedByOtherException = new IOException(
140       "Failed to close region: already closed by another thread");
141 
142   private RegionServerCoprocessorHost rsCoprocessorHost = null;
143 
144   private RegionMergeDetails rmd;
145 
146   /**
147    * Constructor
148    * @param a region a to merge
149    * @param b region b to merge
150    * @param forcible if false, we will only merge adjacent regions
151    */
152   public RegionMergeTransaction(final HRegion a, final HRegion b,
153       final boolean forcible) {
154     if (a.getRegionInfo().compareTo(b.getRegionInfo()) <= 0) {
155       this.region_a = a;
156       this.region_b = b;
157     } else {
158       this.region_a = b;
159       this.region_b = a;
160     }
161     this.forcible = forcible;
162     this.mergesdir = region_a.getRegionFileSystem().getMergesDir();
163   }
164 
165   /**
166    * Does checks on merge inputs.
167    * @param services
168    * @return <code>true</code> if the regions are mergeable else
169    *         <code>false</code> if they are not (e.g. its already closed, etc.).
170    */
171   public boolean prepare(final RegionServerServices services) {
172     if (!region_a.getTableDesc().getTableName()
173         .equals(region_b.getTableDesc().getTableName())) {
174       LOG.info("Can't merge regions " + region_a + "," + region_b
175           + " because they do not belong to the same table");
176       return false;
177     }
178     if (region_a.getRegionInfo().equals(region_b.getRegionInfo())) {
179       LOG.info("Can't merge the same region " + region_a);
180       return false;
181     }
182     if (!forcible && !HRegionInfo.areAdjacent(region_a.getRegionInfo(),
183             region_b.getRegionInfo())) {
184       String msg = "Skip merging " + this.region_a.getRegionNameAsString()
185           + " and " + this.region_b.getRegionNameAsString()
186           + ", because they are not adjacent.";
187       LOG.info(msg);
188       return false;
189     }
190     if (!this.region_a.isMergeable() || !this.region_b.isMergeable()) {
191       return false;
192     }
193     try {
194       boolean regionAHasMergeQualifier = hasMergeQualifierInMeta(services,
195           region_a.getRegionName());
196       if (regionAHasMergeQualifier ||
197           hasMergeQualifierInMeta(services, region_b.getRegionName())) {
198         LOG.debug("Region " + (regionAHasMergeQualifier ? region_a.getRegionNameAsString()
199                 : region_b.getRegionNameAsString())
200             + " is not mergeable because it has merge qualifier in META");
201         return false;
202       }
203     } catch (IOException e) {
204       LOG.warn("Failed judging whether merge transaction is available for "
205               + region_a.getRegionNameAsString() + " and "
206               + region_b.getRegionNameAsString(), e);
207       return false;
208     }
209 
210     // WARN: make sure there is no parent region of the two merging regions in
211     // hbase:meta If exists, fixing up daughters would cause daughter regions(we
212     // have merged one) online again when we restart master, so we should clear
213     // the parent region to prevent the above case
214     // Since HBASE-7721, we don't need fix up daughters any more. so here do
215     // nothing
216 
217     this.mergedRegionInfo = getMergedRegionInfo(region_a.getRegionInfo(),
218         region_b.getRegionInfo());
219     return true;
220   }
221 
222   /**
223    * Run the transaction.
224    * @param server Hosting server instance. Can be null when testing
225    * @param services Used to online/offline regions.
226    * @throws IOException If thrown, transaction failed. Call
227    *           {@link #rollback(Server, RegionServerServices)}
228    * @return merged region
229    * @throws IOException
230    * @see #rollback(Server, RegionServerServices)
231    */
232   public HRegion execute(final Server server,
233  final RegionServerServices services) throws IOException {
234     useCoordinationForAssignment =
235         server == null ? true : ConfigUtil.useZKForAssignment(server.getConfiguration());
236     if (rmd == null) {
237       rmd =
238           server != null && server.getCoordinatedStateManager() != null ? ((BaseCoordinatedStateManager) server
239               .getCoordinatedStateManager()).getRegionMergeCoordination().getDefaultDetails()
240               : null;
241     }
242     if (rsCoprocessorHost == null) {
243       rsCoprocessorHost = server != null ?
244         ((HRegionServer) server).getRegionServerCoprocessorHost() : null;
245     }
246     HRegion mergedRegion = createMergedRegion(server, services);
247     if (rsCoprocessorHost != null) {
248       rsCoprocessorHost.postMergeCommit(this.region_a, this.region_b, mergedRegion);
249     }
250     return stepsAfterPONR(server, services, mergedRegion);
251   }
252 
253   public HRegion stepsAfterPONR(final Server server, final RegionServerServices services,
254       HRegion mergedRegion) throws IOException {
255     openMergedRegion(server, services, mergedRegion);
256     if (useCoordination(server)) {
257       ((BaseCoordinatedStateManager) server.getCoordinatedStateManager())
258           .getRegionMergeCoordination().completeRegionMergeTransaction(services, mergedRegionInfo,
259             region_a, region_b, rmd, mergedRegion);
260     }
261     if (rsCoprocessorHost != null) {
262       rsCoprocessorHost.postMerge(this.region_a, this.region_b, mergedRegion);
263     }
264     return mergedRegion;
265   }
266 
267   /**
268    * Prepare the merged region and region files.
269    * @param server Hosting server instance. Can be null when testing
270    * @param services Used to online/offline regions.
271    * @return merged region
272    * @throws IOException If thrown, transaction failed. Call
273    *           {@link #rollback(Server, RegionServerServices)}
274    */
275   HRegion createMergedRegion(final Server server,
276       final RegionServerServices services) throws IOException {
277     LOG.info("Starting merge of " + region_a + " and "
278         + region_b.getRegionNameAsString() + ", forcible=" + forcible);
279     if ((server != null && server.isStopped())
280         || (services != null && services.isStopping())) {
281       throw new IOException("Server is stopped or stopping");
282     }
283 
284     if (rsCoprocessorHost != null) {
285       if (rsCoprocessorHost.preMerge(this.region_a, this.region_b)) {
286         throw new IOException("Coprocessor bypassing regions " + this.region_a + " "
287             + this.region_b + " merge.");
288       }
289     }
290 
291     // If true, no cluster to write meta edits to or to use coordination.
292     boolean testing = server == null ? true : server.getConfiguration()
293         .getBoolean("hbase.testing.nocluster", false);
294 
295     HRegion mergedRegion = stepsBeforePONR(server, services, testing);
296 
297     @MetaMutationAnnotation
298     List<Mutation> metaEntries = new ArrayList<Mutation>();
299     if (rsCoprocessorHost != null) {
300       if (rsCoprocessorHost.preMergeCommit(this.region_a, this.region_b, metaEntries)) {
301         throw new IOException("Coprocessor bypassing regions " + this.region_a + " "
302             + this.region_b + " merge.");
303       }
304       try {
305         for (Mutation p : metaEntries) {
306           HRegionInfo.parseRegionName(p.getRow());
307         }
308       } catch (IOException e) {
309         LOG.error("Row key of mutation from coprocessor is not parsable as region name."
310             + "Mutations from coprocessor should only be for hbase:meta table.", e);
311         throw e;
312       }
313     }
314 
315     // This is the point of no return. Similar with SplitTransaction.
316     // IF we reach the PONR then subsequent failures need to crash out this
317     // regionserver
318     this.journal.add(JournalEntry.PONR);
319 
320     // Add merged region and delete region_a and region_b
321     // as an atomic update. See HBASE-7721. This update to hbase:meta makes the region
322     // will determine whether the region is merged or not in case of failures.
323     // If it is successful, master will roll-forward, if not, master will
324     // rollback
325     if (!testing && useCoordinationForAssignment) {
326       if (metaEntries.isEmpty()) {
327         MetaTableAccessor.mergeRegions(server.getConnection(),
328           mergedRegion.getRegionInfo(), region_a.getRegionInfo(), region_b.getRegionInfo(),
329           server.getServerName());
330       } else {
331         mergeRegionsAndPutMetaEntries(server.getConnection(),
332           mergedRegion.getRegionInfo(), region_a.getRegionInfo(), region_b.getRegionInfo(),
333           server.getServerName(), metaEntries);
334       }
335     } else if (services != null && !useCoordinationForAssignment) {
336       if (!services.reportRegionStateTransition(TransitionCode.MERGE_PONR,
337           mergedRegionInfo, region_a.getRegionInfo(), region_b.getRegionInfo())) {
338         // Passed PONR, let SSH clean it up
339         throw new IOException("Failed to notify master that merge passed PONR: "
340           + region_a.getRegionInfo().getRegionNameAsString() + " and "
341           + region_b.getRegionInfo().getRegionNameAsString());
342       }
343     }
344     return mergedRegion;
345   }
346 
347   private void mergeRegionsAndPutMetaEntries(HConnection hConnection,
348       HRegionInfo mergedRegion, HRegionInfo regionA, HRegionInfo regionB,
349       ServerName serverName, List<Mutation> metaEntries) throws IOException {
350     prepareMutationsForMerge(mergedRegion, regionA, regionB, serverName, metaEntries);
351     MetaTableAccessor.mutateMetaTable(hConnection, metaEntries);
352   }
353 
354   public void prepareMutationsForMerge(HRegionInfo mergedRegion, HRegionInfo regionA,
355       HRegionInfo regionB, ServerName serverName, List<Mutation> mutations) throws IOException {
356     HRegionInfo copyOfMerged = new HRegionInfo(mergedRegion);
357 
358     // Put for parent
359     Put putOfMerged = MetaTableAccessor.makePutFromRegionInfo(copyOfMerged);
360     putOfMerged.add(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER, regionA.toByteArray());
361     putOfMerged.add(HConstants.CATALOG_FAMILY, HConstants.MERGEB_QUALIFIER, regionB.toByteArray());
362     mutations.add(putOfMerged);
363     // Deletes for merging regions
364     Delete deleteA = MetaTableAccessor.makeDeleteFromRegionInfo(regionA);
365     Delete deleteB = MetaTableAccessor.makeDeleteFromRegionInfo(regionB);
366     mutations.add(deleteA);
367     mutations.add(deleteB);
368     // The merged is a new region, openSeqNum = 1 is fine.
369     addLocation(putOfMerged, serverName, 1);
370   }
371 
372   public Put addLocation(final Put p, final ServerName sn, long openSeqNum) {
373     p.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, Bytes
374         .toBytes(sn.getHostAndPort()));
375     p.add(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER, Bytes.toBytes(sn
376         .getStartcode()));
377     p.add(HConstants.CATALOG_FAMILY, HConstants.SEQNUM_QUALIFIER, Bytes.toBytes(openSeqNum));
378     return p;
379   }
380 
381   public HRegion stepsBeforePONR(final Server server, final RegionServerServices services,
382       boolean testing) throws IOException {
383     if (rmd == null) {
384       rmd =
385           server != null && server.getCoordinatedStateManager() != null ? ((BaseCoordinatedStateManager) server
386               .getCoordinatedStateManager()).getRegionMergeCoordination().getDefaultDetails()
387               : null;
388     }
389 
390     // If server doesn't have a coordination state manager, don't do coordination actions.
391     if (useCoordination(server)) {
392       try {
393         ((BaseCoordinatedStateManager) server.getCoordinatedStateManager())
394             .getRegionMergeCoordination().startRegionMergeTransaction(mergedRegionInfo,
395               server.getServerName(), region_a.getRegionInfo(), region_b.getRegionInfo());
396       } catch (IOException e) {
397         throw new IOException("Failed to start region merge transaction for "
398             + this.mergedRegionInfo.getRegionNameAsString(), e);
399       }
400     } else if (services != null && !useCoordinationForAssignment) {
401       if (!services.reportRegionStateTransition(TransitionCode.READY_TO_MERGE,
402           mergedRegionInfo, region_a.getRegionInfo(), region_b.getRegionInfo())) {
403         throw new IOException("Failed to get ok from master to merge "
404           + region_a.getRegionInfo().getRegionNameAsString() + " and "
405           + region_b.getRegionInfo().getRegionNameAsString());
406       }
407     }
408     this.journal.add(JournalEntry.SET_MERGING);
409     if (useCoordination(server)) {
410       // After creating the merge node, wait for master to transition it
411       // from PENDING_MERGE to MERGING so that we can move on. We want master
412       // knows about it and won't transition any region which is merging.
413       ((BaseCoordinatedStateManager) server.getCoordinatedStateManager())
414           .getRegionMergeCoordination().waitForRegionMergeTransaction(services, mergedRegionInfo,
415             region_a, region_b, rmd);
416     }
417 
418     this.region_a.getRegionFileSystem().createMergesDir();
419     this.journal.add(JournalEntry.CREATED_MERGE_DIR);
420 
421     Map<byte[], List<StoreFile>> hstoreFilesOfRegionA = closeAndOfflineRegion(
422         services, this.region_a, true, testing);
423     Map<byte[], List<StoreFile>> hstoreFilesOfRegionB = closeAndOfflineRegion(
424         services, this.region_b, false, testing);
425 
426     assert hstoreFilesOfRegionA != null && hstoreFilesOfRegionB != null;
427 
428 
429     //
430     // mergeStoreFiles creates merged region dirs under the region_a merges dir
431     // Nothing to unroll here if failure -- clean up of CREATE_MERGE_DIR will
432     // clean this up.
433     mergeStoreFiles(hstoreFilesOfRegionA, hstoreFilesOfRegionB);
434 
435     if (useCoordination(server)) {
436       try {
437         // Do the final check in case any merging region is moved somehow. If so, the transition
438         // will fail.
439         ((BaseCoordinatedStateManager) server.getCoordinatedStateManager())
440             .getRegionMergeCoordination().confirmRegionMergeTransaction(this.mergedRegionInfo,
441               region_a.getRegionInfo(), region_b.getRegionInfo(), server.getServerName(), rmd);
442       } catch (IOException e) {
443         throw new IOException("Failed setting MERGING on "
444             + this.mergedRegionInfo.getRegionNameAsString(), e);
445       }
446     }
447 
448     // Log to the journal that we are creating merged region. We could fail
449     // halfway through. If we do, we could have left
450     // stuff in fs that needs cleanup -- a storefile or two. Thats why we
451     // add entry to journal BEFORE rather than AFTER the change.
452     this.journal.add(JournalEntry.STARTED_MERGED_REGION_CREATION);
453     HRegion mergedRegion = createMergedRegionFromMerges(this.region_a,
454         this.region_b, this.mergedRegionInfo);
455     return mergedRegion;
456   }
457 
458   /**
459    * Create a merged region from the merges directory under region a. In order
460    * to mock it for tests, place it with a new method.
461    * @param a hri of region a
462    * @param b hri of region b
463    * @param mergedRegion hri of merged region
464    * @return merged HRegion.
465    * @throws IOException
466    */
467   HRegion createMergedRegionFromMerges(final HRegion a, final HRegion b,
468       final HRegionInfo mergedRegion) throws IOException {
469     return a.createMergedRegionFromMerges(mergedRegion, b);
470   }
471 
472   /**
473    * Close the merging region and offline it in regionserver
474    * @param services
475    * @param region
476    * @param isRegionA true if it is merging region a, false if it is region b
477    * @param testing true if it is testing
478    * @return a map of family name to list of store files
479    * @throws IOException
480    */
481   private Map<byte[], List<StoreFile>> closeAndOfflineRegion(
482       final RegionServerServices services, final HRegion region,
483       final boolean isRegionA, final boolean testing) throws IOException {
484     Map<byte[], List<StoreFile>> hstoreFilesToMerge = null;
485     Exception exceptionToThrow = null;
486     try {
487       hstoreFilesToMerge = region.close(false);
488     } catch (Exception e) {
489       exceptionToThrow = e;
490     }
491     if (exceptionToThrow == null && hstoreFilesToMerge == null) {
492       // The region was closed by a concurrent thread. We can't continue
493       // with the merge, instead we must just abandon the merge. If we
494       // reopen or merge this could cause problems because the region has
495       // probably already been moved to a different server, or is in the
496       // process of moving to a different server.
497       exceptionToThrow = closedByOtherException;
498     }
499     if (exceptionToThrow != closedByOtherException) {
500       this.journal.add(isRegionA ? JournalEntry.CLOSED_REGION_A
501           : JournalEntry.CLOSED_REGION_B);
502     }
503     if (exceptionToThrow != null) {
504       if (exceptionToThrow instanceof IOException)
505         throw (IOException) exceptionToThrow;
506       throw new IOException(exceptionToThrow);
507     }
508 
509     if (!testing) {
510       services.removeFromOnlineRegions(region, null);
511     }
512     this.journal.add(isRegionA ? JournalEntry.OFFLINED_REGION_A
513         : JournalEntry.OFFLINED_REGION_B);
514     return hstoreFilesToMerge;
515   }
516 
517   /**
518    * Get merged region info through the specified two regions
519    * @param a merging region A
520    * @param b merging region B
521    * @return the merged region info
522    */
523   public static HRegionInfo getMergedRegionInfo(final HRegionInfo a,
524       final HRegionInfo b) {
525     long rid = EnvironmentEdgeManager.currentTime();
526     // Regionid is timestamp. Merged region's id can't be less than that of
527     // merging regions else will insert at wrong location in hbase:meta
528     if (rid < a.getRegionId() || rid < b.getRegionId()) {
529       LOG.warn("Clock skew; merging regions id are " + a.getRegionId()
530           + " and " + b.getRegionId() + ", but current time here is " + rid);
531       rid = Math.max(a.getRegionId(), b.getRegionId()) + 1;
532     }
533 
534     byte[] startKey = null;
535     byte[] endKey = null;
536     // Choose the smaller as start key
537     if (a.compareTo(b) <= 0) {
538       startKey = a.getStartKey();
539     } else {
540       startKey = b.getStartKey();
541     }
542     // Choose the bigger as end key
543     if (Bytes.equals(a.getEndKey(), HConstants.EMPTY_BYTE_ARRAY)
544         || (!Bytes.equals(b.getEndKey(), HConstants.EMPTY_BYTE_ARRAY)
545             && Bytes.compareTo(a.getEndKey(), b.getEndKey()) > 0)) {
546       endKey = a.getEndKey();
547     } else {
548       endKey = b.getEndKey();
549     }
550 
551     // Merged region is sorted between two merging regions in META
552     HRegionInfo mergedRegionInfo = new HRegionInfo(a.getTable(), startKey,
553         endKey, false, rid);
554     return mergedRegionInfo;
555   }
556 
557   /**
558    * Perform time consuming opening of the merged region.
559    * @param server Hosting server instance. Can be null when testing
560    * @param services Used to online/offline regions.
561    * @param merged the merged region
562    * @throws IOException If thrown, transaction failed. Call
563    *           {@link #rollback(Server, RegionServerServices)}
564    */
565   void openMergedRegion(final Server server,
566       final RegionServerServices services, HRegion merged) throws IOException {
567     boolean stopped = server != null && server.isStopped();
568     boolean stopping = services != null && services.isStopping();
569     if (stopped || stopping) {
570       LOG.info("Not opening merged region  " + merged.getRegionNameAsString()
571           + " because stopping=" + stopping + ", stopped=" + stopped);
572       return;
573     }
574     HRegionInfo hri = merged.getRegionInfo();
575     LoggingProgressable reporter = server == null ? null
576         : new LoggingProgressable(hri, server.getConfiguration().getLong(
577             "hbase.regionserver.regionmerge.open.log.interval", 10000));
578     merged.openHRegion(reporter);
579 
580     if (services != null) {
581       try {
582         if (useCoordinationForAssignment) {
583           services.postOpenDeployTasks(merged);
584         } else if (!services.reportRegionStateTransition(TransitionCode.MERGED,
585             mergedRegionInfo, region_a.getRegionInfo(), region_b.getRegionInfo())) {
586           throw new IOException("Failed to report merged region to master: "
587             + mergedRegionInfo.getShortNameToLog());
588         }
589         services.addToOnlineRegions(merged);
590       } catch (KeeperException ke) {
591         throw new IOException(ke);
592       }
593     }
594 
595   }
596 
597   /**
598    * Create reference file(s) of merging regions under the region_a merges dir
599    * @param hstoreFilesOfRegionA
600    * @param hstoreFilesOfRegionB
601    * @throws IOException
602    */
603   private void mergeStoreFiles(
604       Map<byte[], List<StoreFile>> hstoreFilesOfRegionA,
605       Map<byte[], List<StoreFile>> hstoreFilesOfRegionB)
606       throws IOException {
607     // Create reference file(s) of region A in mergdir
608     HRegionFileSystem fs_a = this.region_a.getRegionFileSystem();
609     for (Map.Entry<byte[], List<StoreFile>> entry : hstoreFilesOfRegionA
610         .entrySet()) {
611       String familyName = Bytes.toString(entry.getKey());
612       for (StoreFile storeFile : entry.getValue()) {
613         fs_a.mergeStoreFile(this.mergedRegionInfo, familyName, storeFile,
614             this.mergesdir);
615       }
616     }
617     // Create reference file(s) of region B in mergedir
618     HRegionFileSystem fs_b = this.region_b.getRegionFileSystem();
619     for (Map.Entry<byte[], List<StoreFile>> entry : hstoreFilesOfRegionB
620         .entrySet()) {
621       String familyName = Bytes.toString(entry.getKey());
622       for (StoreFile storeFile : entry.getValue()) {
623         fs_b.mergeStoreFile(this.mergedRegionInfo, familyName, storeFile,
624             this.mergesdir);
625       }
626     }
627   }
628 
629   /**
630    * @param server Hosting server instance (May be null when testing).
631    * @param services Services of regionserver, used to online regions.
632    * @throws IOException If thrown, rollback failed. Take drastic action.
633    * @return True if we successfully rolled back, false if we got to the point
634    *         of no return and so now need to abort the server to minimize
635    *         damage.
636    */
637   @SuppressWarnings("deprecation")
638   public boolean rollback(final Server server,
639       final RegionServerServices services) throws IOException {
640     assert this.mergedRegionInfo != null;
641     // Coprocessor callback
642     if (rsCoprocessorHost != null) {
643       rsCoprocessorHost.preRollBackMerge(this.region_a, this.region_b);
644     }
645 
646     boolean result = true;
647     ListIterator<JournalEntry> iterator = this.journal
648         .listIterator(this.journal.size());
649     // Iterate in reverse.
650     while (iterator.hasPrevious()) {
651       JournalEntry je = iterator.previous();
652       switch (je) {
653 
654         case SET_MERGING:
655         if (useCoordination(server)) {
656           ((BaseCoordinatedStateManager) server.getCoordinatedStateManager())
657               .getRegionMergeCoordination().clean(this.mergedRegionInfo);
658           } else if (services != null && !useCoordinationForAssignment
659               && !services.reportRegionStateTransition(TransitionCode.MERGE_REVERTED,
660                   mergedRegionInfo, region_a.getRegionInfo(), region_b.getRegionInfo())) {
661             return false;
662         }
663           break;
664 
665         case CREATED_MERGE_DIR:
666           this.region_a.writestate.writesEnabled = true;
667           this.region_b.writestate.writesEnabled = true;
668           this.region_a.getRegionFileSystem().cleanupMergesDir();
669           break;
670 
671         case CLOSED_REGION_A:
672           try {
673             // So, this returns a seqid but if we just closed and then reopened,
674             // we should be ok. On close, we flushed using sequenceid obtained
675             // from hosting regionserver so no need to propagate the sequenceid
676             // returned out of initialize below up into regionserver as we
677             // normally do.
678             this.region_a.initialize();
679           } catch (IOException e) {
680             LOG.error("Failed rollbacking CLOSED_REGION_A of region "
681                 + this.region_a.getRegionNameAsString(), e);
682             throw new RuntimeException(e);
683           }
684           break;
685 
686         case OFFLINED_REGION_A:
687           if (services != null)
688             services.addToOnlineRegions(this.region_a);
689           break;
690 
691         case CLOSED_REGION_B:
692           try {
693             this.region_b.initialize();
694           } catch (IOException e) {
695             LOG.error("Failed rollbacking CLOSED_REGION_A of region "
696                 + this.region_b.getRegionNameAsString(), e);
697             throw new RuntimeException(e);
698           }
699           break;
700 
701         case OFFLINED_REGION_B:
702           if (services != null)
703             services.addToOnlineRegions(this.region_b);
704           break;
705 
706         case STARTED_MERGED_REGION_CREATION:
707           this.region_a.getRegionFileSystem().cleanupMergedRegion(
708               this.mergedRegionInfo);
709           break;
710 
711         case PONR:
712           // We got to the point-of-no-return so we need to just abort. Return
713           // immediately. Do not clean up created merged regions.
714           return false;
715 
716         default:
717           throw new RuntimeException("Unhandled journal entry: " + je);
718       }
719     }
720     // Coprocessor callback
721     if (rsCoprocessorHost != null) {
722       rsCoprocessorHost.postRollBackMerge(this.region_a, this.region_b);
723     }
724 
725     return result;
726   }
727 
728   HRegionInfo getMergedRegionInfo() {
729     return this.mergedRegionInfo;
730   }
731 
732   // For unit testing.
733   Path getMergesDir() {
734     return this.mergesdir;
735   }
736 
737   private boolean useCoordination(final Server server) {
738     return server != null && useCoordinationForAssignment
739         && server.getCoordinatedStateManager() != null;
740   }
741 
742 
743 
744   /**
745    * Checks if the given region has merge qualifier in hbase:meta
746    * @param services
747    * @param regionName name of specified region
748    * @return true if the given region has merge qualifier in META.(It will be
749    *         cleaned by CatalogJanitor)
750    * @throws IOException
751    */
752   boolean hasMergeQualifierInMeta(final RegionServerServices services,
753       final byte[] regionName) throws IOException {
754     if (services == null) return false;
755     // Get merge regions if it is a merged region and already has merge
756     // qualifier
757     Pair<HRegionInfo, HRegionInfo> mergeRegions = MetaTableAccessor
758         .getRegionsFromMergeQualifier(services.getConnection(), regionName);
759     if (mergeRegions != null &&
760         (mergeRegions.getFirst() != null || mergeRegions.getSecond() != null)) {
761       // It has merge qualifier
762       return true;
763     }
764     return false;
765   }
766 }