View Javadoc

1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.regionserver;
21  
22  import java.io.IOException;
23  import java.util.ArrayList;
24  import java.util.List;
25  import java.util.ListIterator;
26  import java.util.concurrent.Callable;
27  import java.util.concurrent.ExecutionException;
28  import java.util.concurrent.Executors;
29  import java.util.concurrent.Future;
30  import java.util.concurrent.ThreadFactory;
31  import java.util.concurrent.ThreadPoolExecutor;
32  import java.util.concurrent.TimeUnit;
33  
34  import org.apache.commons.logging.Log;
35  import org.apache.commons.logging.LogFactory;
36  import org.apache.hadoop.conf.Configuration;
37  import org.apache.hadoop.fs.FileStatus;
38  import org.apache.hadoop.fs.FileSystem;
39  import org.apache.hadoop.fs.Path;
40  import org.apache.hadoop.hbase.HRegionInfo;
41  import org.apache.hadoop.hbase.Server;
42  import org.apache.hadoop.hbase.catalog.MetaEditor;
43  import org.apache.hadoop.hbase.io.Reference.Range;
44  import org.apache.hadoop.hbase.util.Bytes;
45  import org.apache.hadoop.hbase.util.CancelableProgressable;
46  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
47  import org.apache.hadoop.hbase.util.FSUtils;
48  import org.apache.hadoop.hbase.util.PairOfSameType;
49  import org.apache.zookeeper.KeeperException;
50  
51  import com.google.common.util.concurrent.ThreadFactoryBuilder;
52  
53  /**
54   * Executes region split as a "transaction".  Call {@link #prepare()} to setup
55   * the transaction, {@link #execute(Server, RegionServerServices)} to run the
56   * transaction and {@link #rollback(OnlineRegions)} to cleanup if execute fails.
57   *
58   * <p>Here is an example of how you would use this class:
59   * <pre>
60   *  SplitTransaction st = new SplitTransaction(this.conf, parent, midKey)
61   *  if (!st.prepare()) return;
62   *  try {
63   *    st.execute(server, services);
64   *  } catch (IOException ioe) {
65   *    try {
66   *      st.rollback(server, services);
67   *      return;
68   *    } catch (RuntimeException e) {
69   *      myAbortable.abort("Failed split, abort");
70   *    }
71   *  }
72   * </Pre>
73   * <p>This class is not thread safe.  Caller needs ensure split is run by
74   * one thread only.
75   */
76  class SplitTransaction {
77    private static final Log LOG = LogFactory.getLog(SplitTransaction.class);
78    private static final String SPLITDIR = "splits";
79  
80    /*
81     * Region to split
82     */
83    private final HRegion parent;
84    private HRegionInfo hri_a;
85    private HRegionInfo hri_b;
86    private Path splitdir;
87    private long fileSplitTimeout = 30000;
88  
89    /*
90     * Row to split around
91     */
92    private final byte [] splitrow;
93  
94    /**
95     * Types to add to the transaction journal.
96     * Each enum is a step in the split transaction. Used to figure how much
97     * we need to rollback.
98     */
99    enum JournalEntry {
100     /**
101      * We created the temporary split data directory.
102      */
103     CREATE_SPLIT_DIR,
104     /**
105      * Closed the parent region.
106      */
107     CLOSED_PARENT_REGION,
108     /**
109      * The parent has been taken out of the server's online regions list.
110      */
111     OFFLINED_PARENT,
112     /**
113      * Started in on creation of the first daughter region.
114      */
115     STARTED_REGION_A_CREATION,
116     /**
117      * Started in on the creation of the second daughter region.
118      */
119     STARTED_REGION_B_CREATION,
120     /**
121      * Point of no return.
122      * If we got here, then transaction is not recoverable other than by
123      * crashing out the regionserver.
124      */
125     PONR
126   }
127 
128   /*
129    * Journal of how far the split transaction has progressed.
130    */
131   private final List<JournalEntry> journal = new ArrayList<JournalEntry>();
132 
133   /**
134    * Constructor
135    * @param services So we can online new regions.  If null, we'll skip onlining
136    * (Useful testing).
137    * @param c Configuration to use running split
138    * @param r Region to split
139    * @param splitrow Row to split around
140    */
141   SplitTransaction(final HRegion r, final byte [] splitrow) {
142     this.parent = r;
143     this.splitrow = splitrow;
144     this.splitdir = getSplitDir(this.parent);
145   }
146 
147   /**
148    * Does checks on split inputs.
149    * @return <code>true</code> if the region is splittable else
150    * <code>false</code> if it is not (e.g. its already closed, etc.).
151    */
152   public boolean prepare() {
153     if (this.parent.isClosed() || this.parent.isClosing()) return false;
154     // Split key can be null if this region is unsplittable; i.e. has refs.
155     if (this.splitrow == null) return false;
156     HRegionInfo hri = this.parent.getRegionInfo();
157     parent.prepareToSplit();
158     // Check splitrow.
159     byte [] startKey = hri.getStartKey();
160     byte [] endKey = hri.getEndKey();
161     if (Bytes.equals(startKey, splitrow) ||
162         !this.parent.getRegionInfo().containsRow(splitrow)) {
163       LOG.info("Split row is not inside region key range or is equal to " +
164           "startkey: " + Bytes.toStringBinary(this.splitrow));
165       return false;
166     }
167     long rid = getDaughterRegionIdTimestamp(hri);
168     this.hri_a = new HRegionInfo(hri.getTableDesc(), startKey, this.splitrow,
169       false, rid);
170     this.hri_b = new HRegionInfo(hri.getTableDesc(), this.splitrow, endKey,
171       false, rid);
172     return true;
173   }
174 
175   /**
176    * Calculate daughter regionid to use.
177    * @param hri Parent {@link HRegionInfo}
178    * @return Daughter region id (timestamp) to use.
179    */
180   private static long getDaughterRegionIdTimestamp(final HRegionInfo hri) {
181     long rid = EnvironmentEdgeManager.currentTimeMillis();
182     // Regionid is timestamp.  Can't be less than that of parent else will insert
183     // at wrong location in .META. (See HBASE-710).
184     if (rid < hri.getRegionId()) {
185       LOG.warn("Clock skew; parent regions id is " + hri.getRegionId() +
186         " but current time here is " + rid);
187       rid = hri.getRegionId() + 1;
188     }
189     return rid;
190   }
191 
192   /**
193    * Run the transaction.
194    * @param server Hosting server instance.  Can be null when testing (won't try
195    * and update in zk if a null server)
196    * @param services Used to online/offline regions.
197    * @throws IOException If thrown, transaction failed. Call {@link #rollback(Server, RegionServerServices)}
198    * @return Regions created
199    * @see #rollback(Server, RegionServerServices)
200    */
201   PairOfSameType<HRegion> execute(final Server server,
202       final RegionServerServices services)
203   throws IOException {
204     LOG.info("Starting split of region " + this.parent);
205     if ((server != null && server.isStopped()) ||
206         (services != null && services.isStopping())) {
207       throw new IOException("Server is stopped or stopping");
208     }
209     assert !this.parent.lock.writeLock().isHeldByCurrentThread() : "Unsafe to hold write lock while performing RPCs";
210 
211     // If true, no cluster to write meta edits into.
212     boolean testing = server == null? true:
213       server.getConfiguration().getBoolean("hbase.testing.nocluster", false);
214     this.fileSplitTimeout = testing ? this.fileSplitTimeout :
215         server.getConfiguration().getLong(
216             "hbase.regionserver.fileSplitTimeout", this.fileSplitTimeout);
217 
218     createSplitDir(this.parent.getFilesystem(), this.splitdir);
219     this.journal.add(JournalEntry.CREATE_SPLIT_DIR);
220 
221     List<StoreFile> hstoreFilesToSplit = this.parent.close(false);
222     if (hstoreFilesToSplit == null) {
223       // The region was closed by a concurrent thread.  We can't continue
224       // with the split, instead we must just abandon the split.  If we
225       // reopen or split this could cause problems because the region has
226       // probably already been moved to a different server, or is in the
227       // process of moving to a different server.
228       throw new IOException("Failed to close region: already closed by " +
229         "another thread");
230     }
231     this.journal.add(JournalEntry.CLOSED_PARENT_REGION);
232 
233     if (!testing) {
234       services.removeFromOnlineRegions(this.parent.getRegionInfo().getEncodedName());
235     }
236     this.journal.add(JournalEntry.OFFLINED_PARENT);
237 
238     // TODO: If splitStoreFiles were multithreaded would we complete steps in
239     // less elapsed time?  St.Ack 20100920
240     //
241     // splitStoreFiles creates daughter region dirs under the parent splits dir
242     // Nothing to unroll here if failure -- clean up of CREATE_SPLIT_DIR will
243     // clean this up.
244     splitStoreFiles(this.splitdir, hstoreFilesToSplit);
245 
246     // Log to the journal that we are creating region A, the first daughter
247     // region.  We could fail halfway through.  If we do, we could have left
248     // stuff in fs that needs cleanup -- a storefile or two.  Thats why we
249     // add entry to journal BEFORE rather than AFTER the change.
250     this.journal.add(JournalEntry.STARTED_REGION_A_CREATION);
251     HRegion a = createDaughterRegion(this.hri_a, this.parent.flushRequester);
252 
253     // Ditto
254     this.journal.add(JournalEntry.STARTED_REGION_B_CREATION);
255     HRegion b = createDaughterRegion(this.hri_b, this.parent.flushRequester);
256 
257     // Edit parent in meta.  Offlines parent region and adds splita and splitb.
258     if (!testing) {
259       MetaEditor.offlineParentInMeta(server.getCatalogTracker(),
260         this.parent.getRegionInfo(), a.getRegionInfo(), b.getRegionInfo());
261     }
262 
263     // This is the point of no return.  Adding subsequent edits to .META. as we
264     // do below when we do the daugther opens adding each to .META. can fail in
265     // various interesting ways the most interesting of which is a timeout
266     // BUT the edits all go through (See HBASE-3872).  IF we reach the POWR
267     // then subsequent failures need to crash out this regionserver; the
268     // server shutdown processing should be able to fix-up the incomplete split.
269     this.journal.add(JournalEntry.PONR);
270     // Open daughters in parallel.
271     DaughterOpener aOpener = new DaughterOpener(server, services, a);
272     DaughterOpener bOpener = new DaughterOpener(server, services, b);
273     aOpener.start();
274     bOpener.start();
275     try {
276       aOpener.join();
277       bOpener.join();
278     } catch (InterruptedException e) {
279       Thread.currentThread().interrupt();
280       throw new IOException("Interrupted " + e.getMessage());
281     }
282     if (aOpener.getException() != null) {
283       throw new IOException("Failed " +
284         aOpener.getName(), aOpener.getException());
285     }
286     if (bOpener.getException() != null) {
287       throw new IOException("Failed " +
288         bOpener.getName(), bOpener.getException());
289     }
290 
291     // Leaving here, the splitdir with its dross will be in place but since the
292     // split was successful, just leave it; it'll be cleaned when parent is
293     // deleted and cleaned up.
294     return new PairOfSameType<HRegion>(a, b);
295   }
296 
297   /*
298    * Open daughter region in its own thread.
299    * If we fail, abort this hosting server.
300    */
301   class DaughterOpener extends Thread {
302     private final RegionServerServices services;
303     private final Server server;
304     private final HRegion r;
305     private Throwable t = null;
306 
307     DaughterOpener(final Server s, final RegionServerServices services,
308         final HRegion r) {
309       super((s ==  null? "null": s.getServerName()) + "-daughterOpener=" +
310         r.getRegionInfo().getEncodedName());
311       setDaemon(true);
312       this.services = services;
313       this.server = s;
314       this.r = r;
315     }
316 
317     /**
318      * @return Null if open succeeded else exception that causes us fail open.
319      * Call it after this thread exits else you may get wrong view on result.
320      */
321     Throwable getException() {
322       return this.t;
323     }
324 
325     @Override
326     public void run() {
327       try {
328         openDaughterRegion(this.server, this.services, r);
329       } catch (Throwable t) {
330         this.t = t;
331       }
332     }
333   }
334 
335   /**
336    * Open daughter regions, add them to online list and update meta.
337    * @param server
338    * @param services Can be null when testing.
339    * @param daughter
340    * @throws IOException
341    * @throws KeeperException
342    */
343   void openDaughterRegion(final Server server,
344       final RegionServerServices services, final HRegion daughter)
345   throws IOException, KeeperException {
346     boolean stopping = services != null && services.isStopping();
347     boolean stopped = server != null && server.isStopped();
348     if (stopped || stopping) {
349       MetaEditor.addDaughter(server.getCatalogTracker(),
350         daughter.getRegionInfo(), null);
351       LOG.info("Not opening daughter " +
352         daughter.getRegionInfo().getRegionNameAsString() +
353         " because stopping=" + stopping + ", stopped=" + server.isStopped());
354       return;
355     }
356     HRegionInfo hri = daughter.getRegionInfo();
357     LoggingProgressable reporter = (server == null)? null:
358       new LoggingProgressable(hri, server.getConfiguration());
359     HRegion r = daughter.openHRegion(reporter);
360     if (services != null) {
361       services.postOpenDeployTasks(r, server.getCatalogTracker(), true);
362     }
363   }
364 
365   static class LoggingProgressable implements CancelableProgressable {
366     private final HRegionInfo hri;
367     private long lastLog = -1;
368     private final long interval;
369 
370     LoggingProgressable(final HRegionInfo hri, final Configuration c) {
371       this.hri = hri;
372       this.interval = c.getLong("hbase.regionserver.split.daughter.open.log.interval",
373         10000);
374     }
375 
376     @Override
377     public boolean progress() {
378       long now = System.currentTimeMillis();
379       if (now - lastLog > this.interval) {
380         LOG.info("Opening " + this.hri.getRegionNameAsString());
381         this.lastLog = now;
382       }
383       return true;
384     }
385   }
386 
387   private static Path getSplitDir(final HRegion r) {
388     return new Path(r.getRegionDir(), SPLITDIR);
389   }
390 
391   /**
392    * @param fs Filesystem to use
393    * @param splitdir Directory to store temporary split data in
394    * @throws IOException If <code>splitdir</code> already exists or we fail
395    * to create it.
396    * @see #cleanupSplitDir(FileSystem, Path)
397    */
398   private static void createSplitDir(final FileSystem fs, final Path splitdir)
399   throws IOException {
400     if (fs.exists(splitdir)) throw new IOException("Splitdir already exits? " + splitdir);
401     if (!fs.mkdirs(splitdir)) throw new IOException("Failed create of " + splitdir);
402   }
403 
404   private static void cleanupSplitDir(final FileSystem fs, final Path splitdir)
405   throws IOException {
406     // Splitdir may have been cleaned up by reopen of the parent dir.
407     deleteDir(fs, splitdir, false);
408   }
409 
410   /**
411    * @param fs Filesystem to use
412    * @param dir Directory to delete
413    * @param mustPreExist If true, we'll throw exception if <code>dir</code>
414    * does not preexist, else we'll just pass.
415    * @throws IOException Thrown if we fail to delete passed <code>dir</code>
416    */
417   private static void deleteDir(final FileSystem fs, final Path dir,
418       final boolean mustPreExist)
419   throws IOException {
420     if (!fs.exists(dir)) {
421       if (mustPreExist) throw new IOException(dir.toString() + " does not exist!");
422     } else if (!fs.delete(dir, true)) {
423       throw new IOException("Failed delete of " + dir);
424     }
425   }
426 
427   private void splitStoreFiles(final Path splitdir,
428     final List<StoreFile> hstoreFilesToSplit)
429   throws IOException {
430     if (hstoreFilesToSplit == null) {
431       // Could be null because close didn't succeed -- for now consider it fatal
432       throw new IOException("Close returned empty list of StoreFiles");
433     }
434     // The following code sets up a thread pool executor with as many slots as
435     // there's files to split. It then fires up everything, waits for
436     // completion and finally checks for any exception
437     int nbFiles = hstoreFilesToSplit.size();
438     ThreadFactoryBuilder builder = new ThreadFactoryBuilder();
439     builder.setNameFormat("StoreFileSplitter-%1$d");
440     ThreadFactory factory = builder.build();
441     ThreadPoolExecutor threadPool =
442       (ThreadPoolExecutor) Executors.newFixedThreadPool(nbFiles, factory);
443     List<Future<Void>> futures = new ArrayList<Future<Void>>(nbFiles);
444 
445      // Split each store file.
446     for (StoreFile sf: hstoreFilesToSplit) {
447       //splitStoreFile(sf, splitdir);
448       StoreFileSplitter sfs = new StoreFileSplitter(sf, splitdir);
449       futures.add(threadPool.submit(sfs));
450     }
451     // Shutdown the pool
452     threadPool.shutdown();
453 
454     // Wait for all the tasks to finish
455     try {
456       boolean stillRunning = !threadPool.awaitTermination(
457           this.fileSplitTimeout, TimeUnit.MILLISECONDS);
458       if (stillRunning) {
459         threadPool.shutdownNow();
460         throw new IOException("Took too long to split the" +
461             " files and create the references, aborting split");
462       }
463     } catch (InterruptedException e) {
464       Thread.currentThread().interrupt();
465       throw new IOException("Interrupted while waiting for file splitters", e);
466     }
467 
468     // Look for any exception
469     for (Future future : futures) {
470       try {
471         future.get();
472       } catch (InterruptedException e) {
473         Thread.currentThread().interrupt();
474         throw new IOException(
475             "Interrupted while trying to get the results of file splitters", e);
476       } catch (ExecutionException e) {
477         throw new IOException(e);
478       }
479     }
480   }
481 
482   private void splitStoreFile(final StoreFile sf, final Path splitdir)
483   throws IOException {
484     FileSystem fs = this.parent.getFilesystem();
485     byte [] family = sf.getFamily();
486     String encoded = this.hri_a.getEncodedName();
487     Path storedir = Store.getStoreHomedir(splitdir, encoded, family);
488     StoreFile.split(fs, storedir, sf, this.splitrow, Range.bottom);
489     encoded = this.hri_b.getEncodedName();
490     storedir = Store.getStoreHomedir(splitdir, encoded, family);
491     StoreFile.split(fs, storedir, sf, this.splitrow, Range.top);
492   }
493 
494   /**
495    * Utility class used to do the file splitting / reference writing
496    * in parallel instead of sequentially.
497    */
498   class StoreFileSplitter implements Callable<Void> {
499 
500     private final StoreFile sf;
501     private final Path splitdir;
502 
503     /**
504      * Constructor that takes what it needs to split
505      * @param sf which file
506      * @param splitdir where the splitting is done
507      */
508     public StoreFileSplitter(final StoreFile sf, final Path splitdir) {
509       this.sf = sf;
510       this.splitdir = splitdir;
511     }
512 
513     public Void call() throws IOException {
514       splitStoreFile(sf, splitdir);
515       return null;
516     }
517   }
518 
519   /**
520    * @param hri Spec. for daughter region to open.
521    * @param flusher Flusher this region should use.
522    * @return Created daughter HRegion.
523    * @throws IOException
524    * @see #cleanupDaughterRegion(FileSystem, Path, HRegionInfo)
525    */
526   HRegion createDaughterRegion(final HRegionInfo hri,
527       final FlushRequester flusher)
528   throws IOException {
529     // Package private so unit tests have access.
530     FileSystem fs = this.parent.getFilesystem();
531     Path regionDir = getSplitDirForDaughter(this.parent.getFilesystem(),
532       this.splitdir, hri);
533     HRegion r = HRegion.newHRegion(this.parent.getTableDir(),
534       this.parent.getLog(), fs, this.parent.getConf(),
535       hri, flusher);
536     HRegion.moveInitialFilesIntoPlace(fs, regionDir, r.getRegionDir());
537     return r;
538   }
539 
540   private static void cleanupDaughterRegion(final FileSystem fs,
541     final Path tabledir, final String encodedName)
542   throws IOException {
543     Path regiondir = HRegion.getRegionDir(tabledir, encodedName);
544     // Dir may not preexist.
545     deleteDir(fs, regiondir, false);
546   }
547 
548   /*
549    * Get the daughter directories in the splits dir.  The splits dir is under
550    * the parent regions' directory.
551    * @param fs
552    * @param splitdir
553    * @param hri
554    * @return Path to daughter split dir.
555    * @throws IOException
556    */
557   private static Path getSplitDirForDaughter(final FileSystem fs,
558       final Path splitdir, final HRegionInfo hri)
559   throws IOException {
560     return new Path(splitdir, hri.getEncodedName());
561   }
562 
563   /**
564    * @param server Hosting server instance (May be null when testing).
565    * @param services
566    * @throws IOException If thrown, rollback failed.  Take drastic action.
567    * @return True if we successfully rolled back, false if we got to the point
568    * of no return and so now need to abort the server to minimize damage.
569    */
570   public boolean rollback(final Server server, final RegionServerServices services)
571   throws IOException {
572     boolean result = true;
573     FileSystem fs = this.parent.getFilesystem();
574     ListIterator<JournalEntry> iterator =
575       this.journal.listIterator(this.journal.size());
576     // Iterate in reverse.
577     while (iterator.hasPrevious()) {
578       JournalEntry je = iterator.previous();
579       switch(je) {
580       case CREATE_SPLIT_DIR:
581     	this.parent.writestate.writesEnabled = true;
582         cleanupSplitDir(fs, this.splitdir);
583         break;
584 
585       case CLOSED_PARENT_REGION:
586         // So, this returns a seqid but if we just closed and then reopened, we
587         // should be ok. On close, we flushed using sequenceid obtained from
588         // hosting regionserver so no need to propagate the sequenceid returned
589         // out of initialize below up into regionserver as we normally do.
590         // TODO: Verify.
591         this.parent.initialize();
592         break;
593 
594       case STARTED_REGION_A_CREATION:
595         cleanupDaughterRegion(fs, this.parent.getTableDir(),
596           this.hri_a.getEncodedName());
597         break;
598 
599       case STARTED_REGION_B_CREATION:
600         cleanupDaughterRegion(fs, this.parent.getTableDir(),
601           this.hri_b.getEncodedName());
602         break;
603 
604       case OFFLINED_PARENT:
605         if (services != null) services.addToOnlineRegions(this.parent);
606         break;
607 
608       case PONR:
609         // We got to the point-of-no-return so we need to just abort. Return
610         // immediately.
611         return false;
612 
613       default:
614         throw new RuntimeException("Unhandled journal entry: " + je);
615       }
616     }
617     return result;
618   }
619 
620   HRegionInfo getFirstDaughter() {
621     return hri_a;
622   }
623 
624   HRegionInfo getSecondDaughter() {
625     return hri_b;
626   }
627 
628   // For unit testing.
629   Path getSplitDir() {
630     return this.splitdir;
631   }
632 
633   /**
634    * Clean up any split detritus that may have been left around from previous
635    * split attempts.
636    * Call this method on initial region deploy.  Cleans up any mess
637    * left by previous deploys of passed <code>r</code> region.
638    * @param r
639    * @throws IOException
640    */
641   static void cleanupAnySplitDetritus(final HRegion r) throws IOException {
642     Path splitdir = getSplitDir(r);
643     FileSystem fs = r.getFilesystem();
644     if (!fs.exists(splitdir)) return;
645     // Look at the splitdir.  It could have the encoded names of the daughter
646     // regions we tried to make.  See if the daughter regions actually got made
647     // out under the tabledir.  If here under splitdir still, then the split did
648     // not complete.  Try and do cleanup.  This code WILL NOT catch the case
649     // where we successfully created daughter a but regionserver crashed during
650     // the creation of region b.  In this case, there'll be an orphan daughter
651     // dir in the filesystem.  TOOD: Fix.
652     FileStatus [] daughters = fs.listStatus(splitdir, new FSUtils.DirFilter(fs));
653     for (int i = 0; i < daughters.length; i++) {
654       cleanupDaughterRegion(fs, r.getTableDir(),
655         daughters[i].getPath().getName());
656     }
657     cleanupSplitDir(r.getFilesystem(), splitdir);
658     LOG.info("Cleaned up old failed split transaction detritus: " + splitdir);
659   }
660 }