View Javadoc

1   /**
2    * Copyright 2008 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.master;
21  
22  import java.io.FileNotFoundException;
23  import java.io.IOException;
24  import java.util.Comparator;
25  import java.util.HashSet;
26  import java.util.Map;
27  import java.util.TreeMap;
28  import java.util.concurrent.atomic.AtomicInteger;
29  
30  import org.apache.commons.logging.Log;
31  import org.apache.commons.logging.LogFactory;
32  import org.apache.hadoop.fs.FileStatus;
33  import org.apache.hadoop.fs.FileSystem;
34  import org.apache.hadoop.fs.Path;
35  import org.apache.hadoop.fs.PathFilter;
36  import org.apache.hadoop.hbase.Chore;
37  import org.apache.hadoop.hbase.HColumnDescriptor;
38  import org.apache.hadoop.hbase.HConstants;
39  import org.apache.hadoop.hbase.HRegionInfo;
40  import org.apache.hadoop.hbase.HTableDescriptor;
41  import org.apache.hadoop.hbase.Server;
42  import org.apache.hadoop.hbase.backup.HFileArchiver;
43  import org.apache.hadoop.hbase.catalog.MetaEditor;
44  import org.apache.hadoop.hbase.catalog.MetaReader;
45  import org.apache.hadoop.hbase.client.Result;
46  import org.apache.hadoop.hbase.regionserver.Store;
47  import org.apache.hadoop.hbase.regionserver.StoreFile;
48  import org.apache.hadoop.hbase.util.Bytes;
49  import org.apache.hadoop.hbase.util.FSUtils;
50  import org.apache.hadoop.hbase.util.Pair;
51  import org.apache.hadoop.hbase.util.Writables;
52  
53  /**
54   * A janitor for the catalog tables.  Scans the <code>.META.</code> catalog
55   * table on a period looking for unused regions to garbage collect.
56   */
57  class CatalogJanitor extends Chore {
58    private static final Log LOG = LogFactory.getLog(CatalogJanitor.class.getName());
59    private final Server server;
60    private final MasterServices services;
61    private boolean enabled = true;
62  
63    CatalogJanitor(final Server server, final MasterServices services) {
64      super(server.getServerName() + "-CatalogJanitor",
65        server.getConfiguration().getInt("hbase.catalogjanitor.interval", 300000),
66        server);
67      this.server = server;
68      this.services = services;
69    }
70  
71    @Override
72    protected boolean initialChore() {
73      try {
74        if (this.enabled) scan();
75      } catch (IOException e) {
76        LOG.warn("Failed initial scan of catalog table", e);
77        return false;
78      }
79      return true;
80    }
81  
82    /**
83     * @param enabled
84     */
85    public void setEnabled(final boolean enabled) {
86      this.enabled = enabled;
87    }
88  
89    @Override
90    protected void chore() {
91      try {
92        scan();
93      } catch (IOException e) {
94        LOG.warn("Failed scan of catalog table", e);
95      }
96    }
97  
98    /**
99     * Scans META and returns a number of scanned rows, and
100    * an ordered map of split parents.
101    */
102   Pair<Integer, Map<HRegionInfo, Result>> getSplitParents() throws IOException {
103     // TODO: Only works with single .META. region currently.  Fix.
104     final AtomicInteger count = new AtomicInteger(0);
105     // Keep Map of found split parents.  There are candidates for cleanup.
106     // Use a comparator that has split parents come before its daughters.
107     final Map<HRegionInfo, Result> splitParents =
108       new TreeMap<HRegionInfo, Result>(new SplitParentFirstComparator());
109     // This visitor collects split parents and counts rows in the .META. table
110     MetaReader.Visitor visitor = new MetaReader.Visitor() {
111       @Override
112       public boolean visit(Result r) throws IOException {
113         if (r == null || r.isEmpty()) return true;
114         count.incrementAndGet();
115         HRegionInfo info = getHRegionInfo(r);
116         if (info == null) return true; // Keep scanning
117         if (info.isSplitParent()) splitParents.put(info, r);
118         // Returning true means "keep scanning"
119         return true;
120       }
121     };
122     // Run full scan of .META. catalog table passing in our custom visitor
123     MetaReader.fullScan(this.server.getCatalogTracker(), visitor);
124 
125     return new Pair<Integer, Map<HRegionInfo, Result>>(count.get(), splitParents);
126   }
127 
128   /**
129    * Run janitorial scan of catalog <code>.META.</code> table looking for
130    * garbage to collect.
131    * @throws IOException
132    */
133   int scan() throws IOException {
134     Pair<Integer, Map<HRegionInfo, Result>> pair = getSplitParents();
135     int count = pair.getFirst();
136     Map<HRegionInfo, Result> splitParents = pair.getSecond();
137 
138     // Now work on our list of found parents. See if any we can clean up.
139     int cleaned = 0;
140     HashSet<String> parentNotCleaned = new HashSet<String>(); //regions whose parents are still around
141     for (Map.Entry<HRegionInfo, Result> e : splitParents.entrySet()) {
142       if (!parentNotCleaned.contains(e.getKey().getEncodedName()) && cleanParent(e.getKey(), e.getValue())) {
143         cleaned++;
144       } else {
145         // We could not clean the parent, so it's daughters should not be cleaned either (HBASE-6160)
146         parentNotCleaned.add(getDaughterRegionInfo(
147               e.getValue(), HConstants.SPLITA_QUALIFIER).getEncodedName());
148         parentNotCleaned.add(getDaughterRegionInfo(
149               e.getValue(), HConstants.SPLITB_QUALIFIER).getEncodedName());
150       }
151     }
152     if (cleaned != 0) {
153       LOG.info("Scanned " + count + " catalog row(s) and gc'd " + cleaned +
154         " unreferenced parent region(s)");
155     } else if (LOG.isDebugEnabled()) {
156       LOG.debug("Scanned " + count + " catalog row(s) and gc'd " + cleaned +
157       " unreferenced parent region(s)");
158     }
159     return cleaned;
160   }
161 
162   /**
163    * Compare HRegionInfos in a way that has split parents sort BEFORE their
164    * daughters.
165    */
166   static class SplitParentFirstComparator implements Comparator<HRegionInfo> {
167     @Override
168     public int compare(HRegionInfo left, HRegionInfo right) {
169       // This comparator differs from the one HRegionInfo in that it sorts
170       // parent before daughters.
171       if (left == null) return -1;
172       if (right == null) return 1;
173       // Same table name.
174       int result = Bytes.compareTo(left.getTableName(),
175           right.getTableName());
176       if (result != 0) return result;
177       // Compare start keys.
178       result = Bytes.compareTo(left.getStartKey(), right.getStartKey());
179       if (result != 0) return result;
180       // Compare end keys.
181       result = Bytes.compareTo(left.getEndKey(), right.getEndKey());
182       if (result != 0) {
183         if (left.getStartKey().length != 0
184                 && left.getEndKey().length == 0) {
185             return -1;  // left is last region
186         }
187         if (right.getStartKey().length != 0
188                 && right.getEndKey().length == 0) {
189             return 1;  // right is the last region
190         }
191         return -result; // Flip the result so parent comes first.
192       }
193       return result;
194     }
195   }
196 
197   /**
198    * Get HRegionInfo from passed Map of row values.
199    * @param result Map to do lookup in.
200    * @return Null if not found (and logs fact that expected COL_REGIONINFO
201    * was missing) else deserialized {@link HRegionInfo}
202    * @throws IOException
203    */
204   static HRegionInfo getHRegionInfo(final Result result)
205   throws IOException {
206     byte [] bytes =
207       result.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
208     if (bytes == null) {
209       LOG.warn("REGIONINFO_QUALIFIER is empty in " + result);
210       return null;
211     }
212     return Writables.getHRegionInfo(bytes);
213   }
214 
215   /**
216    * If daughters no longer hold reference to the parents, delete the parent.
217    * @param server HRegionInterface of meta server to talk to
218    * @param parent HRegionInfo of split offlined parent
219    * @param rowContent Content of <code>parent</code> row in
220    * <code>metaRegionName</code>
221    * @return True if we removed <code>parent</code> from meta table and from
222    * the filesystem.
223    * @throws IOException
224    */
225   boolean cleanParent(final HRegionInfo parent, Result rowContent)
226   throws IOException {
227     boolean result = false;
228     // Run checks on each daughter split.
229     HRegionInfo a_region = getDaughterRegionInfo(rowContent, HConstants.SPLITA_QUALIFIER);
230     HRegionInfo b_region = getDaughterRegionInfo(rowContent, HConstants.SPLITB_QUALIFIER);
231     Pair<Boolean, Boolean> a =
232       checkDaughterInFs(parent, a_region, HConstants.SPLITA_QUALIFIER);
233     Pair<Boolean, Boolean> b =
234       checkDaughterInFs(parent, b_region, HConstants.SPLITB_QUALIFIER);
235     if (hasNoReferences(a) && hasNoReferences(b)) {
236       LOG.debug("Deleting region " + parent.getRegionNameAsString() +
237         " because daughter splits no longer hold references");
238       // wipe out daughter references from parent region in meta
239       removeDaughtersFromParent(parent);
240 
241       // This latter regionOffline should not be necessary but is done for now
242       // until we let go of regionserver to master heartbeats.  See HBASE-3368.
243       if (this.services.getAssignmentManager() != null) {
244         // The mock used in testing catalogjanitor returns null for getAssignmnetManager.
245         // Allow for null result out of getAssignmentManager.
246         this.services.getAssignmentManager().regionOffline(parent);
247       }
248       FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
249       HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, parent);
250       MetaEditor.deleteRegion(this.server.getCatalogTracker(), parent);
251       result = true;
252     }
253     return result;
254   }
255 
256   /**
257    * @param p A pair where the first boolean says whether or not the daughter
258    * region directory exists in the filesystem and then the second boolean says
259    * whether the daughter has references to the parent.
260    * @return True the passed <code>p</code> signifies no references.
261    */
262   private boolean hasNoReferences(final Pair<Boolean, Boolean> p) {
263     return !p.getFirst() || !p.getSecond();
264   }
265 
266   /**
267    * Get daughter HRegionInfo out of parent info:splitA/info:splitB columns.
268    * @param result
269    * @param which Whether "info:splitA" or "info:splitB" column
270    * @return Deserialized content of the info:splitA or info:splitB as a
271    * HRegionInfo
272    * @throws IOException
273    */
274   private HRegionInfo getDaughterRegionInfo(final Result result,
275     final byte [] which)
276   throws IOException {
277     byte [] bytes = result.getValue(HConstants.CATALOG_FAMILY, which);
278     return Writables.getHRegionInfoOrNull(bytes);
279   }
280 
281   /**
282    * Remove mention of daughters from parent row.
283    * @param parent
284    * @throws IOException
285    */
286   private void removeDaughtersFromParent(final HRegionInfo parent)
287   throws IOException {
288     MetaEditor.deleteDaughtersReferencesInParent(this.server.getCatalogTracker(), parent);
289   }
290 
291   /**
292    * Checks if a daughter region -- either splitA or splitB -- still holds
293    * references to parent.
294    * @param parent Parent region name.
295    * @param split Which column family.
296    * @param qualifier Which of the daughters to look at, splitA or splitB.
297    * @return A pair where the first boolean says whether or not the daughter
298    * region directory exists in the filesystem and then the second boolean says
299    * whether the daughter has references to the parent.
300    * @throws IOException
301    */
302   Pair<Boolean, Boolean> checkDaughterInFs(final HRegionInfo parent,
303     final HRegionInfo split,
304     final byte [] qualifier)
305   throws IOException {
306     boolean references = false;
307     boolean exists = false;
308     if (split == null)  {
309       return new Pair<Boolean, Boolean>(Boolean.FALSE, Boolean.FALSE);
310     }
311     FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
312     Path rootdir = this.services.getMasterFileSystem().getRootDir();
313     Path tabledir = new Path(rootdir, split.getTableNameAsString());
314     Path regiondir = new Path(tabledir, split.getEncodedName());
315     exists = fs.exists(regiondir);
316     if (!exists) {
317       LOG.warn("Daughter regiondir does not exist: " + regiondir.toString());
318       return new Pair<Boolean, Boolean>(exists, Boolean.FALSE);
319     }
320     HTableDescriptor parentDescriptor = getTableDescriptor(parent.getTableName());
321 
322     for (HColumnDescriptor family: parentDescriptor.getFamilies()) {
323       Path p = Store.getStoreHomedir(tabledir, split.getEncodedName(),
324         family.getName());
325       if (!fs.exists(p)) continue;
326       // Look for reference files.  Call listStatus with anonymous instance of PathFilter.
327       FileStatus [] ps = FSUtils.listStatus(fs, p,
328           new PathFilter () {
329             public boolean accept(Path path) {
330               return StoreFile.isReference(path);
331             }
332           }
333       );
334 
335       if (ps != null && ps.length > 0) {
336         references = true;
337         break;
338       }
339     }
340     return new Pair<Boolean, Boolean>(Boolean.valueOf(exists),
341       Boolean.valueOf(references));
342   }
343 
344   private HTableDescriptor getTableDescriptor(byte[] tableName)
345   throws FileNotFoundException, IOException {
346     return this.services.getTableDescriptors().get(Bytes.toString(tableName));
347   }
348 }