1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.master;
21  
22  import static org.junit.Assert.assertFalse;
23  import static org.junit.Assert.assertNotNull;
24  import static org.junit.Assert.assertNull;
25  import static org.junit.Assert.assertTrue;
26  
27  import java.io.IOException;
28  import java.util.ArrayList;
29  import java.util.List;
30  
31  import org.apache.hadoop.conf.Configuration;
32  import org.apache.hadoop.fs.FileSystem;
33  import org.apache.hadoop.fs.Path;
34  import org.apache.hadoop.hbase.HBaseTestingUtility;
35  import org.apache.hadoop.hbase.HColumnDescriptor;
36  import org.apache.hadoop.hbase.HConstants;
37  import org.apache.hadoop.hbase.HRegionInfo;
38  import org.apache.hadoop.hbase.HTableDescriptor;
39  import org.apache.hadoop.hbase.KeyValue;
40  import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
41  import org.apache.hadoop.hbase.Server;
42  import org.apache.hadoop.hbase.catalog.CatalogTracker;
43  import org.apache.hadoop.hbase.client.Result;
44  import org.apache.hadoop.hbase.executor.ExecutorService;
45  import org.apache.hadoop.hbase.io.Reference;
46  import org.apache.hadoop.hbase.ipc.HRegionInterface;
47  import org.apache.hadoop.hbase.regionserver.Store;
48  import org.apache.hadoop.hbase.util.Bytes;
49  import org.apache.hadoop.hbase.util.Writables;
50  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
51  import org.junit.Test;
52  import org.mockito.Mockito;
53  
54  public class TestCatalogJanitor {
55  
56    /**
57     * Pseudo server for below tests.
58     */
59    class MockServer implements Server {
60      private final Configuration c;
61      private final CatalogTracker ct;
62  
63      MockServer(final HBaseTestingUtility htu)
64      throws NotAllMetaRegionsOnlineException, IOException {
65        this.c = htu.getConfiguration();
66        // Set hbase.rootdir into test dir.
67        FileSystem fs = FileSystem.get(this.c);
68        Path rootdir =
69          fs.makeQualified(HBaseTestingUtility.getTestDir(HConstants.HBASE_DIR));
70        this.c.set(HConstants.HBASE_DIR, rootdir.toString());
71        this.ct = Mockito.mock(CatalogTracker.class);
72        HRegionInterface hri = Mockito.mock(HRegionInterface.class);
73        Mockito.when(ct.waitForMetaServerConnectionDefault()).thenReturn(hri);
74      }
75  
76      @Override
77      public CatalogTracker getCatalogTracker() {
78        return this.ct;
79      }
80  
81      @Override
82      public Configuration getConfiguration() {
83        return this.c;
84      }
85  
86      @Override
87      public String getServerName() {
88        // TODO Auto-generated method stub
89        return null;
90      }
91  
92      @Override
93      public ZooKeeperWatcher getZooKeeper() {
94        // TODO Auto-generated method stub
95        return null;
96      }
97  
98      @Override
99      public void abort(String why, Throwable e) {
100       // TODO Auto-generated method stub
101     }
102 
103     @Override
104     public boolean isStopped() {
105       // TODO Auto-generated method stub
106       return false;
107     }
108 
109     @Override
110     public void stop(String why) {
111       // TODO Auto-generated method stub
112     }
113     
114   }
115 
116   /**
117    * Mock MasterServices for tests below.
118    */
119   class MockMasterServices implements MasterServices {
120     private final MasterFileSystem mfs;
121 
122     MockMasterServices(final Server server) throws IOException {
123       this.mfs = new MasterFileSystem(server, null);
124     }
125 
126     @Override
127     public void checkTableModifiable(byte[] tableName) throws IOException {
128       // TODO Auto-generated method stub
129     }
130 
131     @Override
132     public AssignmentManager getAssignmentManager() {
133       // TODO Auto-generated method stub
134       return null;
135     }
136 
137     @Override
138     public ExecutorService getExecutorService() {
139       // TODO Auto-generated method stub
140       return null;
141     }
142 
143     @Override
144     public MasterFileSystem getMasterFileSystem() {
145       return this.mfs;
146     }
147 
148     @Override
149     public ServerManager getServerManager() {
150       // TODO Auto-generated method stub
151       return null;
152     }
153     
154   }
155 
156   @Test
157   public void testGetHRegionInfo() throws IOException {
158     assertNull(CatalogJanitor.getHRegionInfo(new Result()));
159     List<KeyValue> kvs = new ArrayList<KeyValue>();
160     Result r = new Result(kvs);
161     assertNull(CatalogJanitor.getHRegionInfo(r));
162     byte [] f = HConstants.CATALOG_FAMILY;
163     // Make a key value that doesn't have the expected qualifier.
164     kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY, f,
165       HConstants.SERVER_QUALIFIER, f));
166     r = new Result(kvs);
167     assertNull(CatalogJanitor.getHRegionInfo(r));
168     // Make a key that does not have a regioninfo value.
169     kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY, f,
170       HConstants.REGIONINFO_QUALIFIER, f));
171     HRegionInfo hri = CatalogJanitor.getHRegionInfo(new Result(kvs));
172     assertTrue(hri == null);
173     // OK, give it what it expects
174     kvs.clear();
175     kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY, f,
176       HConstants.REGIONINFO_QUALIFIER,
177       Writables.getBytes(HRegionInfo.FIRST_META_REGIONINFO)));
178     hri = CatalogJanitor.getHRegionInfo(new Result(kvs));
179     assertNotNull(hri);
180     assertTrue(hri.equals(HRegionInfo.FIRST_META_REGIONINFO));
181   }
182 
183   @Test
184   public void testCleanParent() throws IOException {
185     HBaseTestingUtility htu = new HBaseTestingUtility();
186     Server server = new MockServer(htu);
187     MasterServices services = new MockMasterServices(server);
188     CatalogJanitor janitor = new CatalogJanitor(server, services);
189     // Create regions.
190     HTableDescriptor htd = new HTableDescriptor("table");
191     htd.addFamily(new HColumnDescriptor("family"));
192     HRegionInfo parent =
193       new HRegionInfo(htd, Bytes.toBytes("aaa"), Bytes.toBytes("eee"));
194     HRegionInfo splita =
195       new HRegionInfo(htd, Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
196     HRegionInfo splitb =
197       new HRegionInfo(htd, Bytes.toBytes("ccc"), Bytes.toBytes("eee"));
198     // Test that when both daughter regions are in place, that we do not
199     // remove the parent.
200     List<KeyValue> kvs = new ArrayList<KeyValue>();
201     kvs.add(new KeyValue(parent.getRegionName(), HConstants.CATALOG_FAMILY,
202       HConstants.SPLITA_QUALIFIER, Writables.getBytes(splita)));
203     kvs.add(new KeyValue(parent.getRegionName(), HConstants.CATALOG_FAMILY,
204       HConstants.SPLITB_QUALIFIER, Writables.getBytes(splitb)));
205     Result r = new Result(kvs);
206     // Add a reference under splitA directory so we don't clear out the parent.
207     Path rootdir = services.getMasterFileSystem().getRootDir();
208     Path tabledir =
209       HTableDescriptor.getTableDir(rootdir, htd.getName());
210     Path storedir = Store.getStoreHomedir(tabledir, splita.getEncodedName(),
211       htd.getColumnFamilies()[0].getName());
212     Reference ref = new Reference(Bytes.toBytes("ccc"), Reference.Range.top);
213     long now = System.currentTimeMillis();
214     // Reference name has this format: StoreFile#REF_NAME_PARSER
215     Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName());
216     FileSystem fs = services.getMasterFileSystem().getFileSystem();
217     ref.write(fs, p);
218     assertFalse(janitor.cleanParent(parent, r));
219     // Remove the reference file and try again.
220     assertTrue(fs.delete(p, true));
221     // We will fail!!! Because split b is empty, which is right... we should
222     // not remove parent if daughters do not exist in fs.
223     assertFalse(janitor.cleanParent(parent, r));
224     // Put in place daughter dir for b... that should make it so parent gets
225     // cleaned up.
226     storedir = Store.getStoreHomedir(tabledir, splitb.getEncodedName(),
227       htd.getColumnFamilies()[0].getName());
228     assertTrue(fs.mkdirs(storedir));
229     assertTrue(janitor.cleanParent(parent, r));
230   }
231 }