View Javadoc

1   /**
2    * Copyright 2011 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.regionserver.compactions;
21  
22  import java.util.ArrayList;
23  import java.util.Calendar;
24  import java.util.GregorianCalendar;
25  import java.util.List;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.hbase.regionserver.StoreFile;
31  
32  public class CompactSelection {
33    private static final long serialVersionUID = 1L;
34    static final Log LOG = LogFactory.getLog(CompactSelection.class);
35    // the actual list - this is needed to handle methods like "sublist"
36    // correctly
37    List<StoreFile> filesToCompact = new ArrayList<StoreFile>();
38  
39    /**
40     * Number of off peak compactions either in the compaction queue or
41     * happening now. Please lock compactionCountLock before modifying.
42     */
43    static long numOutstandingOffPeakCompactions = 0;
44  
45    /**
46     * Lock object for numOutstandingOffPeakCompactions
47     */
48    private final static Object compactionCountLock = new Object();
49  
50    // HBase conf object
51    Configuration conf;
52    // was this compaction promoted to an off-peak
53    boolean isOffPeakCompaction = false;
54    // compactRatio: double on purpose!  Float.MAX < Long.MAX < Double.MAX
55    // With float, java will downcast your long to float for comparisons (bad)
56    private double compactRatio;
57    // compaction ratio off-peak
58    private double compactRatioOffPeak;
59    // offpeak start time
60    private int offPeakStartHour = -1;
61    // off peak end time
62    private int offPeakEndHour = -1;
63  
64    public CompactSelection(Configuration conf, List<StoreFile> filesToCompact) {
65      this.filesToCompact = filesToCompact;
66      this.conf = conf;
67      this.compactRatio = conf.getFloat("hbase.hstore.compaction.ratio", 1.2F);
68      this.compactRatioOffPeak = conf.getFloat("hbase.hstore.compaction.ratio.offpeak", 5.0F);
69  
70      // Peak time is from [offPeakStartHour, offPeakEndHour). Valid numbers are [0, 23]
71      this.offPeakStartHour = conf.getInt("hbase.offpeak.start.hour", -1);
72      this.offPeakEndHour = conf.getInt("hbase.offpeak.end.hour", -1);
73      if (!isValidHour(this.offPeakStartHour) || !isValidHour(this.offPeakEndHour)) {
74        if (!(this.offPeakStartHour == -1 && this.offPeakEndHour == -1)) {
75          LOG.warn("Invalid start/end hour for peak hour : start = " +
76              this.offPeakStartHour + " end = " + this.offPeakEndHour +
77              ". Valid numbers are [0-23]");
78        }
79        this.offPeakStartHour = this.offPeakEndHour = -1;
80      }
81    }
82  
83    /**
84     * Select the expired store files to compact
85     * 
86     * @param maxExpiredTimeStamp
87     *          The store file will be marked as expired if its max time stamp is
88     *          less than this maxExpiredTimeStamp.
89     * @return A CompactSelection contains the expired store files as
90     *         filesToCompact
91     */
92    public CompactSelection selectExpiredStoreFilesToCompact(
93        long maxExpiredTimeStamp) {
94      if (filesToCompact == null || filesToCompact.size() == 0)
95        return null;
96      ArrayList<StoreFile> expiredStoreFiles = null;
97      boolean hasExpiredStoreFiles = false;
98      CompactSelection expiredSFSelection = null;
99  
100     for (StoreFile storeFile : this.filesToCompact) {
101       if (storeFile.getReader().getMaxTimestamp() < maxExpiredTimeStamp) {
102         LOG.info("Deleting the expired store file by compaction: "
103             + storeFile.getPath() + " whose maxTimeStamp is "
104             + storeFile.getReader().getMaxTimestamp()
105             + " while the max expired timestamp is " + maxExpiredTimeStamp);
106         if (!hasExpiredStoreFiles) {
107           expiredStoreFiles = new ArrayList<StoreFile>();
108           hasExpiredStoreFiles = true;
109         }
110         expiredStoreFiles.add(storeFile);
111       }
112     }
113 
114     if (hasExpiredStoreFiles) {
115       expiredSFSelection = new CompactSelection(conf, expiredStoreFiles);
116     }
117     return expiredSFSelection;
118   }
119 
120   /**
121    * If the current hour falls in the off peak times and there are no 
122    * outstanding off peak compactions, the current compaction is 
123    * promoted to an off peak compaction. Currently only one off peak 
124    * compaction is present in the compaction queue.
125    *
126    * @param currentHour
127    * @return
128    */
129   public double getCompactSelectionRatio() {
130     double r = this.compactRatio;
131     synchronized(compactionCountLock) {
132       if (isOffPeakHour() && numOutstandingOffPeakCompactions == 0) {
133         r = this.compactRatioOffPeak;
134         numOutstandingOffPeakCompactions++;
135         isOffPeakCompaction = true;
136       }
137     }
138     if(isOffPeakCompaction) {
139       LOG.info("Running an off-peak compaction, selection ratio = " +
140           compactRatioOffPeak + ", numOutstandingOffPeakCompactions is now " +
141           numOutstandingOffPeakCompactions);
142     }
143     return r;
144   }
145 
146   /**
147    * The current compaction finished, so reset the off peak compactions count
148    * if this was an off peak compaction.
149    */
150   public void finishRequest() {
151     if (isOffPeakCompaction) {
152       synchronized(compactionCountLock) {
153         numOutstandingOffPeakCompactions--;
154         isOffPeakCompaction = false;
155       }
156       LOG.info("Compaction done, numOutstandingOffPeakCompactions is now " +
157           numOutstandingOffPeakCompactions);
158     }
159   }
160 
161   public List<StoreFile> getFilesToCompact() {
162     return filesToCompact;
163   }
164 
165   /**
166    * Removes all files from the current compaction list, and resets off peak
167    * compactions is set.
168    */
169   public void emptyFileList() {
170     filesToCompact.clear();
171     if (isOffPeakCompaction) {
172       synchronized(compactionCountLock) {
173         // reset the off peak count
174         numOutstandingOffPeakCompactions--;
175         isOffPeakCompaction = false;
176       }
177       LOG.info("Nothing to compact, numOutstandingOffPeakCompactions is now " +
178           numOutstandingOffPeakCompactions);
179     }
180   }
181 
182   public boolean isOffPeakCompaction() {
183     return this.isOffPeakCompaction;
184   }
185 
186   private boolean isOffPeakHour() {
187     int currentHour = (new GregorianCalendar()).get(Calendar.HOUR_OF_DAY);
188     // If offpeak time checking is disabled just return false.
189     if (this.offPeakStartHour == this.offPeakEndHour) {
190       return false;
191     }
192     if (this.offPeakStartHour < this.offPeakEndHour) {
193       return (currentHour >= this.offPeakStartHour && currentHour < this.offPeakEndHour);
194     }
195     return (currentHour >= this.offPeakStartHour || currentHour < this.offPeakEndHour);
196   }
197 
198   public CompactSelection subList(int start, int end) {
199     throw new UnsupportedOperationException();
200   }
201 
202   public CompactSelection getSubList(int start, int end) {
203     filesToCompact = filesToCompact.subList(start, end);
204     return this;
205   }
206 
207   public void clearSubList(int start, int end) {
208     filesToCompact.subList(start, end).clear();
209   }
210 
211   private boolean isValidHour(int hour) {
212     return (hour >= 0 && hour <= 23);
213   }
214 }