1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.master;
21  
22  import static org.apache.hadoop.hbase.util.HFileArchiveTestingUtil.assertArchiveEqualToOriginal;
23  import static org.junit.Assert.assertEquals;
24  import static org.junit.Assert.assertFalse;
25  import static org.junit.Assert.assertNotNull;
26  import static org.junit.Assert.assertNull;
27  import static org.junit.Assert.assertTrue;
28  import static org.mockito.Mockito.doReturn;
29  import static org.mockito.Mockito.spy;
30  
31  import java.io.IOException;
32  import java.util.ArrayList;
33  import java.util.List;
34  import java.util.Map;
35  import java.util.SortedMap;
36  import java.util.TreeMap;
37  
38  import org.apache.hadoop.conf.Configuration;
39  import org.apache.hadoop.fs.FSDataOutputStream;
40  import org.apache.hadoop.fs.FileStatus;
41  import org.apache.hadoop.fs.FileSystem;
42  import org.apache.hadoop.fs.Path;
43  import org.apache.hadoop.hbase.HBaseTestingUtility;
44  import org.apache.hadoop.hbase.HColumnDescriptor;
45  import org.apache.hadoop.hbase.HConstants;
46  import org.apache.hadoop.hbase.HRegionInfo;
47  import org.apache.hadoop.hbase.HTableDescriptor;
48  import org.apache.hadoop.hbase.KeyValue;
49  import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
50  import org.apache.hadoop.hbase.Server;
51  import org.apache.hadoop.hbase.ServerName;
52  import org.apache.hadoop.hbase.SmallTests;
53  import org.apache.hadoop.hbase.TableDescriptors;
54  import org.apache.hadoop.hbase.catalog.CatalogTracker;
55  import org.apache.hadoop.hbase.client.HConnection;
56  import org.apache.hadoop.hbase.client.HConnectionManager;
57  import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
58  import org.apache.hadoop.hbase.client.Result;
59  import org.apache.hadoop.hbase.executor.ExecutorService;
60  import org.apache.hadoop.hbase.io.Reference;
61  import org.apache.hadoop.hbase.ipc.CoprocessorProtocol;
62  import org.apache.hadoop.hbase.master.CatalogJanitor.SplitParentFirstComparator;
63  import org.apache.hadoop.hbase.ipc.HRegionInterface;
64  import org.apache.hadoop.hbase.regionserver.Store;
65  import org.apache.hadoop.hbase.util.Bytes;
66  import org.apache.hadoop.hbase.util.FSUtils;
67  import org.apache.hadoop.hbase.util.HFileArchiveUtil;
68  import org.apache.hadoop.hbase.util.Pair;
69  import org.apache.hadoop.hbase.util.Writables;
70  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
71  import org.junit.Test;
72  import org.junit.experimental.categories.Category;
73  import org.mockito.Mockito;
74  
75  @Category(SmallTests.class)
76  public class TestCatalogJanitor {
77    /**
78     * Pseudo server for below tests.
79     * Be sure to call stop on the way out else could leave some mess around.
80     */
81    class MockServer implements Server {
82      private final HConnection connection;
83      private final Configuration c;
84      private final CatalogTracker ct;
85  
86      MockServer(final HBaseTestingUtility htu)
87      throws NotAllMetaRegionsOnlineException, IOException, InterruptedException {
88        this.c = htu.getConfiguration();
89        // Mock an HConnection and a HRegionInterface implementation.  Have the
90        // HConnection return the HRI.  Have the HRI return a few mocked up responses
91        // to make our test work.
92        this.connection =
93          HConnectionTestingUtility.getMockedConnectionAndDecorate(this.c,
94            Mockito.mock(HRegionInterface.class),
95            new ServerName("example.org,12345,6789"),
96            HRegionInfo.FIRST_META_REGIONINFO);
97        // Set hbase.rootdir into test dir.
98        FileSystem fs = FileSystem.get(this.c);
99        Path rootdir = fs.makeQualified(new Path(this.c.get(HConstants.HBASE_DIR)));
100       this.c.set(HConstants.HBASE_DIR, rootdir.toString());
101       this.ct = Mockito.mock(CatalogTracker.class);
102       HRegionInterface hri = Mockito.mock(HRegionInterface.class);
103       Mockito.when(this.ct.getConnection()).thenReturn(this.connection);
104       Mockito.when(ct.waitForMetaServerConnection(Mockito.anyLong())).thenReturn(hri);
105     }
106 
107     @Override
108     public CatalogTracker getCatalogTracker() {
109       return this.ct;
110     }
111 
112     @Override
113     public Configuration getConfiguration() {
114       return this.c;
115     }
116 
117     @Override
118     public ServerName getServerName() {
119       return new ServerName("mockserver.example.org", 1234, -1L);
120     }
121 
122     @Override
123     public ZooKeeperWatcher getZooKeeper() {
124       return null;
125     }
126 
127     @Override
128     public void abort(String why, Throwable e) {
129       //no-op
130     }
131 
132     @Override
133     public boolean isAborted() {
134       return false;
135     }
136 
137     @Override
138     public boolean isStopped() {
139       return false;
140     }
141 
142     @Override
143     public void stop(String why) {
144       if (this.ct != null) {
145         this.ct.stop();
146       }
147       if (this.connection != null) {
148         HConnectionManager.deleteConnection(this.connection.getConfiguration());
149       }
150     }
151   }
152 
153   /**
154    * Mock MasterServices for tests below.
155    */
156   class MockMasterServices implements MasterServices {
157     private final MasterFileSystem mfs;
158     private final AssignmentManager asm;
159 
160     MockMasterServices(final Server server) throws IOException {
161       this.mfs = new MasterFileSystem(server, this, null, false);
162       this.asm = Mockito.mock(AssignmentManager.class);
163     }
164 
165     @Override
166     public void checkTableModifiable(byte[] tableName) throws IOException {
167       //no-op
168     }
169 
170     @Override
171     public void createTable(HTableDescriptor desc, byte[][] splitKeys)
172         throws IOException {
173       // no-op
174     }
175 
176     @Override
177     public AssignmentManager getAssignmentManager() {
178       return this.asm;
179     }
180 
181     @Override
182     public ExecutorService getExecutorService() {
183       return null;
184     }
185 
186     @Override
187     public MasterFileSystem getMasterFileSystem() {
188       return this.mfs;
189     }
190 
191     @Override
192     public ServerManager getServerManager() {
193       return null;
194     }
195 
196     @Override
197     public ZooKeeperWatcher getZooKeeper() {
198       return null;
199     }
200 
201     @Override
202     public CatalogTracker getCatalogTracker() {
203       return null;
204     }
205 
206     @Override
207     public Configuration getConfiguration() {
208       return mfs.conf;
209     }
210 
211     @Override
212     public ServerName getServerName() {
213       return null;
214     }
215 
216     @Override
217     public void abort(String why, Throwable e) {
218       //no-op
219     }
220 
221     @Override
222     public boolean isAborted() {
223       return false;
224     }
225 
226     private boolean stopped = false;
227 
228     @Override
229     public void stop(String why) {
230       stopped = true;
231     }
232 
233     @Override
234     public boolean isStopped() {
235       return stopped;
236     }
237 
238     @Override
239     public TableDescriptors getTableDescriptors() {
240       return new TableDescriptors() {
241         @Override
242         public HTableDescriptor remove(String tablename) throws IOException {
243           return null;
244         }
245         
246         @Override
247         public Map<String, HTableDescriptor> getAll() throws IOException {
248           return null;
249         }
250         
251         @Override
252         public HTableDescriptor get(byte[] tablename)
253         throws IOException {
254           return get(Bytes.toString(tablename));
255         }
256         
257         @Override
258         public HTableDescriptor get(String tablename)
259         throws IOException {
260           return createHTableDescriptor();
261         }
262         
263         @Override
264         public void add(HTableDescriptor htd) throws IOException {
265         }
266       };
267     }
268 
269     @Override
270     public boolean isServerShutdownHandlerEnabled() {
271       return true;
272     }
273 
274     @Override
275     public MasterCoprocessorHost getCoprocessorHost() {
276       return null;
277     }
278 
279     @Override
280     public <T extends CoprocessorProtocol> boolean registerProtocol(Class<T> protocol, T handler) {
281       return false;
282     }
283 
284     @Override
285     public void deleteTable(byte[] tableName) throws IOException {
286     }
287 
288     @Override
289     public void modifyTable(byte[] tableName, HTableDescriptor descriptor) throws IOException {
290     }
291 
292     @Override
293     public void enableTable(byte[] tableName) throws IOException {
294     }
295 
296     @Override
297     public void disableTable(byte[] tableName) throws IOException {
298     }
299 
300     @Override
301     public void addColumn(byte[] tableName, HColumnDescriptor column) throws IOException {
302     }
303 
304     @Override
305     public void modifyColumn(byte[] tableName, HColumnDescriptor descriptor) throws IOException {
306     }
307 
308     @Override
309     public void deleteColumn(byte[] tableName, byte[] columnName) throws IOException {
310     }
311   }
312 
313   @Test
314   public void testGetHRegionInfo() throws IOException {
315     assertNull(CatalogJanitor.getHRegionInfo(new Result()));
316     List<KeyValue> kvs = new ArrayList<KeyValue>();
317     Result r = new Result(kvs);
318     assertNull(CatalogJanitor.getHRegionInfo(r));
319     byte [] f = HConstants.CATALOG_FAMILY;
320     // Make a key value that doesn't have the expected qualifier.
321     kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY, f,
322       HConstants.SERVER_QUALIFIER, f));
323     r = new Result(kvs);
324     assertNull(CatalogJanitor.getHRegionInfo(r));
325     // Make a key that does not have a regioninfo value.
326     kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY, f,
327       HConstants.REGIONINFO_QUALIFIER, f));
328     HRegionInfo hri = CatalogJanitor.getHRegionInfo(new Result(kvs));
329     assertTrue(hri == null);
330     // OK, give it what it expects
331     kvs.clear();
332     kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY, f,
333       HConstants.REGIONINFO_QUALIFIER,
334       Writables.getBytes(HRegionInfo.FIRST_META_REGIONINFO)));
335     hri = CatalogJanitor.getHRegionInfo(new Result(kvs));
336     assertNotNull(hri);
337     assertTrue(hri.equals(HRegionInfo.FIRST_META_REGIONINFO));
338   }
339 
340   @Test
341   public void testCleanParent() throws IOException, InterruptedException {
342     HBaseTestingUtility htu = new HBaseTestingUtility();
343     setRootDirAndCleanIt(htu, "testCleanParent");
344     Server server = new MockServer(htu);
345     try {
346       MasterServices services = new MockMasterServices(server);
347       CatalogJanitor janitor = new CatalogJanitor(server, services);
348       // Create regions.
349       HTableDescriptor htd = new HTableDescriptor("table");
350       htd.addFamily(new HColumnDescriptor("f"));
351       HRegionInfo parent =
352         new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"),
353             Bytes.toBytes("eee"));
354       HRegionInfo splita =
355         new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"),
356             Bytes.toBytes("ccc"));
357       HRegionInfo splitb =
358         new HRegionInfo(htd.getName(), Bytes.toBytes("ccc"),
359             Bytes.toBytes("eee"));
360       // Test that when both daughter regions are in place, that we do not
361       // remove the parent.
362       List<KeyValue> kvs = new ArrayList<KeyValue>();
363       kvs.add(new KeyValue(parent.getRegionName(), HConstants.CATALOG_FAMILY,
364           HConstants.SPLITA_QUALIFIER, Writables.getBytes(splita)));
365       kvs.add(new KeyValue(parent.getRegionName(), HConstants.CATALOG_FAMILY,
366           HConstants.SPLITB_QUALIFIER, Writables.getBytes(splitb)));
367       Result r = new Result(kvs);
368       // Add a reference under splitA directory so we don't clear out the parent.
369       Path rootdir = services.getMasterFileSystem().getRootDir();
370       Path tabledir =
371         HTableDescriptor.getTableDir(rootdir, htd.getName());
372       Path storedir = Store.getStoreHomedir(tabledir, splita.getEncodedName(),
373           htd.getColumnFamilies()[0].getName());
374       Reference ref = new Reference(Bytes.toBytes("ccc"), Reference.Range.top);
375       long now = System.currentTimeMillis();
376       // Reference name has this format: StoreFile#REF_NAME_PARSER
377       Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName());
378       FileSystem fs = services.getMasterFileSystem().getFileSystem();
379       Path path = ref.write(fs, p);
380       assertTrue(fs.exists(path));
381       assertFalse(janitor.cleanParent(parent, r));
382       // Remove the reference file and try again.
383       assertTrue(fs.delete(p, true));
384       assertTrue(janitor.cleanParent(parent, r));
385     } finally {
386       server.stop("shutdown");
387     }
388   }
389 
390   /**
391    * Make sure parent gets cleaned up even if daughter is cleaned up before it.
392    * @throws IOException
393    * @throws InterruptedException
394    */
395   @Test
396   public void testParentCleanedEvenIfDaughterGoneFirst()
397   throws IOException, InterruptedException {
398     parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(
399       "testParentCleanedEvenIfDaughterGoneFirst", Bytes.toBytes("eee"));
400   }
401 
402   /**
403    * Make sure last parent with empty end key gets cleaned up even if daughter is cleaned up before it.
404    * @throws IOException
405    * @throws InterruptedException
406    */
407   @Test
408   public void testLastParentCleanedEvenIfDaughterGoneFirst()
409   throws IOException, InterruptedException {
410     parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(
411       "testLastParentCleanedEvenIfDaughterGoneFirst", new byte[0]);
412   }
413 
414   /**
415    * Make sure parent with specified end key gets cleaned up even if daughter is cleaned up before it.
416    *
417    * @param rootDir the test case name, used as the HBase testing utility root
418    * @param lastEndKey the end key of the split parent
419    * @throws IOException
420    * @throws InterruptedException
421    */
422   private void parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(
423   final String rootDir, final byte[] lastEndKey)
424   throws IOException, InterruptedException {
425     HBaseTestingUtility htu = new HBaseTestingUtility();
426     setRootDirAndCleanIt(htu, rootDir);
427     Server server = new MockServer(htu);
428     MasterServices services = new MockMasterServices(server);
429     CatalogJanitor janitor = new CatalogJanitor(server, services);
430     final HTableDescriptor htd = createHTableDescriptor();
431 
432     // Create regions: aaa->{lastEndKey}, aaa->ccc, aaa->bbb, bbb->ccc, etc.
433 
434     // Parent
435     HRegionInfo parent = new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"),
436       lastEndKey);
437     // Sleep a second else the encoded name on these regions comes out
438     // same for all with same start key and made in same second.
439     Thread.sleep(1001);
440 
441     // Daughter a
442     HRegionInfo splita = new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"),
443       Bytes.toBytes("ccc"));
444     Thread.sleep(1001);
445     // Make daughters of daughter a; splitaa and splitab.
446     HRegionInfo splitaa = new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"),
447       Bytes.toBytes("bbb"));
448     HRegionInfo splitab = new HRegionInfo(htd.getName(), Bytes.toBytes("bbb"),
449       Bytes.toBytes("ccc"));
450 
451     // Daughter b
452     HRegionInfo splitb = new HRegionInfo(htd.getName(), Bytes.toBytes("ccc"),
453       lastEndKey);
454     Thread.sleep(1001);
455     // Make Daughters of daughterb; splitba and splitbb.
456     HRegionInfo splitba = new HRegionInfo(htd.getName(), Bytes.toBytes("ccc"),
457       Bytes.toBytes("ddd"));
458     HRegionInfo splitbb = new HRegionInfo(htd.getName(), Bytes.toBytes("ddd"),
459     lastEndKey);
460 
461     // First test that our Comparator works right up in CatalogJanitor.
462     // Just fo kicks.
463     SortedMap<HRegionInfo, Result> regions =
464       new TreeMap<HRegionInfo, Result>(new CatalogJanitor.SplitParentFirstComparator());
465     // Now make sure that this regions map sorts as we expect it to.
466     regions.put(parent, createResult(parent, splita, splitb));
467     regions.put(splitb, createResult(splitb, splitba, splitbb));
468     regions.put(splita, createResult(splita, splitaa, splitab));
469     // Assert its properly sorted.
470     int index = 0;
471     for (Map.Entry<HRegionInfo, Result> e: regions.entrySet()) {
472       if (index == 0) {
473         assertTrue(e.getKey().getEncodedName().equals(parent.getEncodedName()));
474       } else if (index == 1) {
475         assertTrue(e.getKey().getEncodedName().equals(splita.getEncodedName()));
476       } else if (index == 2) {
477         assertTrue(e.getKey().getEncodedName().equals(splitb.getEncodedName()));
478       }
479       index++;
480     }
481 
482     // Now play around with the cleanParent function.  Create a ref from splita
483     // up to the parent.
484     Path splitaRef =
485       createReferences(services, htd, parent, splita, Bytes.toBytes("ccc"), false);
486     // Make sure actual super parent sticks around because splita has a ref.
487     assertFalse(janitor.cleanParent(parent, regions.get(parent)));
488 
489     //splitba, and split bb, do not have dirs in fs.  That means that if
490     // we test splitb, it should get cleaned up.
491     assertTrue(janitor.cleanParent(splitb, regions.get(splitb)));
492 
493     // Now remove ref from splita to parent... so parent can be let go and so
494     // the daughter splita can be split (can't split if still references).
495     // BUT make the timing such that the daughter gets cleaned up before we
496     // can get a chance to let go of the parent.
497     FileSystem fs = FileSystem.get(htu.getConfiguration());
498     assertTrue(fs.delete(splitaRef, true));
499     // Create the refs from daughters of splita.
500     Path splitaaRef =
501       createReferences(services, htd, splita, splitaa, Bytes.toBytes("bbb"), false);
502     Path splitabRef =
503       createReferences(services, htd, splita, splitab, Bytes.toBytes("bbb"), true);
504 
505     // Test splita.  It should stick around because references from splitab, etc.
506     assertFalse(janitor.cleanParent(splita, regions.get(splita)));
507 
508     // Now clean up parent daughter first.  Remove references from its daughters.
509     assertTrue(fs.delete(splitaaRef, true));
510     assertTrue(fs.delete(splitabRef, true));
511     assertTrue(janitor.cleanParent(splita, regions.get(splita)));
512 
513     // Super parent should get cleaned up now both splita and splitb are gone.
514     assertTrue(janitor.cleanParent(parent, regions.get(parent)));
515 
516     services.stop("test finished");
517     janitor.join();
518   }
519 
520   /**
521    * CatalogJanitor.scan() should not clean parent regions if their own
522    * parents are still referencing them. This ensures that grandfather regions
523    * do not point to deleted parent regions.
524    */
525   @Test
526   public void testScanDoesNotCleanRegionsWithExistingParents() throws Exception {
527     HBaseTestingUtility htu = new HBaseTestingUtility();
528     setRootDirAndCleanIt(htu, "testScanDoesNotCleanRegionsWithExistingParents");
529     Server server = new MockServer(htu);
530     MasterServices services = new MockMasterServices(server);
531 
532     final HTableDescriptor htd = createHTableDescriptor();
533 
534     // Create regions: aaa->{lastEndKey}, aaa->ccc, aaa->bbb, bbb->ccc, etc.
535 
536     // Parent
537     HRegionInfo parent = new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"),
538       new byte[0], true);
539     // Sleep a second else the encoded name on these regions comes out
540     // same for all with same start key and made in same second.
541     Thread.sleep(1001);
542 
543     // Daughter a
544     HRegionInfo splita = new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"),
545       Bytes.toBytes("ccc"), true);
546     Thread.sleep(1001);
547     // Make daughters of daughter a; splitaa and splitab.
548     HRegionInfo splitaa = new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"),
549       Bytes.toBytes("bbb"), false);
550     HRegionInfo splitab = new HRegionInfo(htd.getName(), Bytes.toBytes("bbb"),
551       Bytes.toBytes("ccc"), false);
552 
553     // Daughter b
554     HRegionInfo splitb = new HRegionInfo(htd.getName(), Bytes.toBytes("ccc"),
555         new byte[0]);
556     Thread.sleep(1001);
557 
558     final Map<HRegionInfo, Result> splitParents =
559         new TreeMap<HRegionInfo, Result>(new SplitParentFirstComparator());
560     splitParents.put(parent, makeResultFromHRegionInfo(parent, splita, splitb));
561     splita.setOffline(true);//simulate that splita goes offline when it is split
562     splitParents.put(splita, makeResultFromHRegionInfo(splita, splitaa, splitab));
563 
564     CatalogJanitor janitor = spy(new CatalogJanitor(server, services));
565     doReturn(new Pair<Integer, Map<HRegionInfo, Result>>(
566         10, splitParents)).when(janitor).getSplitParents();
567 
568     //create ref from splita to parent
569     Path splitaRef =
570         createReferences(services, htd, parent, splita, Bytes.toBytes("ccc"), false);
571 
572     //parent and A should not be removed
573     assertEquals(0, janitor.scan());
574 
575     //now delete the ref
576     FileSystem fs = FileSystem.get(htu.getConfiguration());
577     assertTrue(fs.delete(splitaRef, true));
578 
579     //now, both parent, and splita can be deleted
580     assertEquals(2, janitor.scan());
581 
582     services.stop("test finished");
583     janitor.join();
584   }
585 
586   @Test
587   public void testArchiveOldRegion() throws Exception {
588     String table = "table";
589     HBaseTestingUtility htu = new HBaseTestingUtility();
590     setRootDirAndCleanIt(htu, "testCleanParent");
591     Server server = new MockServer(htu);
592     MasterServices services = new MockMasterServices(server);
593 
594     // create the janitor
595     CatalogJanitor janitor = new CatalogJanitor(server, services);
596 
597     // Create regions.
598     HTableDescriptor htd = new HTableDescriptor(table);
599     htd.addFamily(new HColumnDescriptor("f"));
600     HRegionInfo parent = new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"), Bytes.toBytes("eee"));
601     HRegionInfo splita = new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
602     HRegionInfo splitb = new HRegionInfo(htd.getName(), Bytes.toBytes("ccc"), Bytes.toBytes("eee"));
603     // Test that when both daughter regions are in place, that we do not
604     // remove the parent.
605     List<KeyValue> kvs = new ArrayList<KeyValue>();
606     kvs.add(new KeyValue(parent.getRegionName(), HConstants.CATALOG_FAMILY,
607         HConstants.SPLITA_QUALIFIER, Writables.getBytes(splita)));
608     kvs.add(new KeyValue(parent.getRegionName(), HConstants.CATALOG_FAMILY,
609         HConstants.SPLITB_QUALIFIER, Writables.getBytes(splitb)));
610     Result r = new Result(kvs);
611 
612     FileSystem fs = FileSystem.get(htu.getConfiguration());
613     Path rootdir = services.getMasterFileSystem().getRootDir();
614     // have to set the root directory since we use it in HFileDisposer to figure out to get to the
615     // archive directory. Otherwise, it just seems to pick the first root directory it can find (so
616     // the single test passes, but when the full suite is run, things get borked).
617     FSUtils.setRootDir(fs.getConf(), rootdir);
618     Path tabledir = HTableDescriptor.getTableDir(rootdir, htd.getName());
619     Path storedir = Store.getStoreHomedir(tabledir, parent.getEncodedName(),
620       htd.getColumnFamilies()[0].getName());
621 
622     // delete the file and ensure that the files have been archived
623     Path storeArchive = HFileArchiveUtil.getStoreArchivePath(services.getConfiguration(), parent,
624       tabledir, htd.getColumnFamilies()[0].getName());
625 
626     // enable archiving, make sure that files get archived
627     addMockStoreFiles(2, services, storedir);
628     // get the current store files for comparison
629     FileStatus[] storeFiles = fs.listStatus(storedir);
630     for (FileStatus file : storeFiles) {
631       System.out.println("Have store file:" + file.getPath());
632     }
633 
634     // do the cleaning of the parent
635     assertTrue(janitor.cleanParent(parent, r));
636 
637     // and now check to make sure that the files have actually been archived
638     FileStatus[] archivedStoreFiles = fs.listStatus(storeArchive);
639     assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs);
640 
641     // cleanup
642     services.stop("Test finished");
643     server.stop("shutdown");
644     janitor.join();
645   }
646 
647   /**
648    * Test that if a store file with the same name is present as those already backed up cause the
649    * already archived files to be timestamped backup
650    */
651   @Test
652   public void testDuplicateHFileResolution() throws Exception {
653     String table = "table";
654     HBaseTestingUtility htu = new HBaseTestingUtility();
655     setRootDirAndCleanIt(htu, "testCleanParent");
656     Server server = new MockServer(htu);
657     MasterServices services = new MockMasterServices(server);
658 
659     // create the janitor
660     CatalogJanitor janitor = new CatalogJanitor(server, services);
661 
662     // Create regions.
663     HTableDescriptor htd = new HTableDescriptor(table);
664     htd.addFamily(new HColumnDescriptor("f"));
665     HRegionInfo parent = new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"), Bytes.toBytes("eee"));
666     HRegionInfo splita = new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
667     HRegionInfo splitb = new HRegionInfo(htd.getName(), Bytes.toBytes("ccc"), Bytes.toBytes("eee"));
668     // Test that when both daughter regions are in place, that we do not
669     // remove the parent.
670     List<KeyValue> kvs = new ArrayList<KeyValue>();
671     kvs.add(new KeyValue(parent.getRegionName(), HConstants.CATALOG_FAMILY,
672         HConstants.SPLITA_QUALIFIER, Writables.getBytes(splita)));
673     kvs.add(new KeyValue(parent.getRegionName(), HConstants.CATALOG_FAMILY,
674         HConstants.SPLITB_QUALIFIER, Writables.getBytes(splitb)));
675     Result r = new Result(kvs);
676 
677     FileSystem fs = FileSystem.get(htu.getConfiguration());
678 
679     Path rootdir = services.getMasterFileSystem().getRootDir();
680     // have to set the root directory since we use it in HFileDisposer to figure out to get to the
681     // archive directory. Otherwise, it just seems to pick the first root directory it can find (so
682     // the single test passes, but when the full suite is run, things get borked).
683     FSUtils.setRootDir(fs.getConf(), rootdir);
684     Path tabledir = HTableDescriptor.getTableDir(rootdir, parent.getTableName());
685     Path storedir = Store.getStoreHomedir(tabledir, parent.getEncodedName(),
686       htd.getColumnFamilies()[0].getName());
687     System.out.println("Old root:" + rootdir);
688     System.out.println("Old table:" + tabledir);
689     System.out.println("Old store:" + storedir);
690 
691     Path storeArchive = HFileArchiveUtil.getStoreArchivePath(services.getConfiguration(), parent,
692       tabledir, htd.getColumnFamilies()[0].getName());
693     System.out.println("Old archive:" + storeArchive);
694 
695     // enable archiving, make sure that files get archived
696     addMockStoreFiles(2, services, storedir);
697     // get the current store files for comparison
698     FileStatus[] storeFiles = fs.listStatus(storedir);
699 
700     // do the cleaning of the parent
701     assertTrue(janitor.cleanParent(parent, r));
702 
703     // and now check to make sure that the files have actually been archived
704     FileStatus[] archivedStoreFiles = fs.listStatus(storeArchive);
705     assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs);
706 
707     // now add store files with the same names as before to check backup
708     // enable archiving, make sure that files get archived
709     addMockStoreFiles(2, services, storedir);
710 
711     // do the cleaning of the parent
712     assertTrue(janitor.cleanParent(parent, r));
713 
714     // and now check to make sure that the files have actually been archived
715     archivedStoreFiles = fs.listStatus(storeArchive);
716     assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs, true);
717 
718     // cleanup
719     services.stop("Test finished");
720     server.stop("shutdown");
721     janitor.join();
722   }
723 
724   private void addMockStoreFiles(int count, MasterServices services, Path storedir)
725       throws IOException {
726     // get the existing store files
727     FileSystem fs = services.getMasterFileSystem().getFileSystem();
728     fs.mkdirs(storedir);
729     // create the store files in the parent
730     for (int i = 0; i < count; i++) {
731       Path storeFile = new Path(storedir, "_store" + i);
732       FSDataOutputStream dos = fs.create(storeFile, true);
733       dos.writeBytes("Some data: " + i);
734       dos.close();
735     }
736     // make sure the mock store files are there
737     FileStatus[] storeFiles = fs.listStatus(storedir);
738     assertEquals(count, storeFiles.length);
739   }
740 
741   private Result makeResultFromHRegionInfo(HRegionInfo region, HRegionInfo splita,
742       HRegionInfo splitb) throws IOException {
743     List<KeyValue> kvs = new ArrayList<KeyValue>();
744     kvs.add(new KeyValue(
745         region.getRegionName(),
746         HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
747         Writables.getBytes(region)));
748 
749     if (splita != null) {
750       kvs.add(new KeyValue(
751           region.getRegionName(),
752           HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER,
753           Writables.getBytes(splita)));
754     }
755 
756     if (splitb != null) {
757       kvs.add(new KeyValue(
758           region.getRegionName(),
759           HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER,
760           Writables.getBytes(splitb)));
761     }
762 
763     return new Result(kvs);
764   }
765 
766   private String setRootDirAndCleanIt(final HBaseTestingUtility htu,
767       final String subdir)
768   throws IOException {
769     Path testdir = htu.getDataTestDir(subdir);
770     FileSystem fs = FileSystem.get(htu.getConfiguration());
771     if (fs.exists(testdir)) assertTrue(fs.delete(testdir, true));
772     htu.getConfiguration().set(HConstants.HBASE_DIR, testdir.toString());
773     return htu.getConfiguration().get(HConstants.HBASE_DIR);
774   }
775 
776   /**
777    * @param services Master services instance.
778    * @param htd
779    * @param parent
780    * @param daughter
781    * @param midkey
782    * @param top True if we are to write a 'top' reference.
783    * @return Path to reference we created.
784    * @throws IOException
785    */
786   private Path createReferences(final MasterServices services,
787       final HTableDescriptor htd, final HRegionInfo parent,
788       final HRegionInfo daughter, final byte [] midkey, final boolean top)
789   throws IOException {
790     Path rootdir = services.getMasterFileSystem().getRootDir();
791     Path tabledir = HTableDescriptor.getTableDir(rootdir, parent.getTableName());
792     Path storedir = Store.getStoreHomedir(tabledir, daughter.getEncodedName(),
793       htd.getColumnFamilies()[0].getName());
794     Reference ref = new Reference(midkey,
795       top? Reference.Range.top: Reference.Range.bottom);
796     long now = System.currentTimeMillis();
797     // Reference name has this format: StoreFile#REF_NAME_PARSER
798     Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName());
799     FileSystem fs = services.getMasterFileSystem().getFileSystem();
800     ref.write(fs, p);
801     return p;
802   }
803 
804   private Result createResult(final HRegionInfo parent, final HRegionInfo a,
805       final HRegionInfo b)
806   throws IOException {
807     List<KeyValue> kvs = new ArrayList<KeyValue>();
808     kvs.add(new KeyValue(parent.getRegionName(), HConstants.CATALOG_FAMILY,
809       HConstants.SPLITA_QUALIFIER, Writables.getBytes(a)));
810     kvs.add(new KeyValue(parent.getRegionName(), HConstants.CATALOG_FAMILY,
811       HConstants.SPLITB_QUALIFIER, Writables.getBytes(b)));
812     return new Result(kvs);
813   }
814 
815   private HTableDescriptor createHTableDescriptor() {
816     HTableDescriptor htd = new HTableDescriptor("t");
817     htd.addFamily(new HColumnDescriptor("f"));
818     return htd;
819   }
820 
821   @org.junit.Rule
822   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
823     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
824 }
825