View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.regionserver.compactions;
21  
22  import org.apache.commons.logging.Log;
23  import org.apache.commons.logging.LogFactory;
24  import org.apache.hadoop.conf.Configuration;
25  import org.apache.hadoop.hbase.HConstants;
26  import org.apache.hadoop.hbase.classification.InterfaceAudience;
27  import org.apache.hadoop.hbase.regionserver.StoreConfigInformation;
28  
29  /**
30   * Compaction configuration for a particular instance of HStore.
31   * Takes into account both global settings and ones set on the column family/store.
32   * Control knobs for default compaction algorithm:
33   * <p/>
34   * maxCompactSize - upper bound on file size to be included in minor compactions
35   * minCompactSize - lower bound below which compaction is selected without ratio test
36   * minFilesToCompact - lower bound on number of files in any minor compaction
37   * maxFilesToCompact - upper bound on number of files in any minor compaction
38   * compactionRatio - Ratio used for compaction
39   * <p/>
40   * Set parameter as "hbase.hstore.compaction.<attribute>"
41   */
42  
43  //TODO: revisit this class for online parameter updating (both in xml and on the CF)
44  @InterfaceAudience.Private
45  public class CompactionConfiguration {
46  
47    static final Log LOG = LogFactory.getLog(CompactionConfiguration.class);
48  
49    private static final String CONFIG_PREFIX = "hbase.hstore.compaction.";
50    public static final String HBASE_HSTORE_MIN_LOCALITY_TO_SKIP_MAJOR_COMPACT =
51        "hbase.hstore.min.locality.to.skip.major.compact";
52    public static final String RATIO_KEY = CONFIG_PREFIX + "ratio";
53    public static final String MIN_KEY = CONFIG_PREFIX + "min";
54    public static final String MAX_KEY = CONFIG_PREFIX + "max";
55    
56    /*
57     * The epoch time length for the windows we no longer compact
58     */
59    public static final String DATE_TIERED_MAX_AGE_MILLIS_KEY =
60      CONFIG_PREFIX + "date.tiered.max.storefile.age.millis";
61    public static final String DATE_TIERED_INCOMING_WINDOW_MIN_KEY =
62      CONFIG_PREFIX + "date.tiered.incoming.window.min";
63    public static final String COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS_KEY =
64      CONFIG_PREFIX + "date.tiered.window.policy.class";
65    public static final String DATE_TIERED_SINGLE_OUTPUT_FOR_MINOR_COMPACTION_KEY =
66      CONFIG_PREFIX + "date.tiered.single.output.for.minor.compaction";
67  
68    private static final Class<? extends RatioBasedCompactionPolicy>
69      DEFAULT_COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS = ExploringCompactionPolicy.class;
70  
71    public static final String DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS_KEY =
72      CONFIG_PREFIX + "date.tiered.window.factory.class";
73  
74    private static final Class<? extends CompactionWindowFactory>
75      DEFAULT_DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS = ExponentialCompactionWindowFactory.class;
76  
77    Configuration conf;
78    StoreConfigInformation storeConfigInfo;
79  
80    long maxCompactSize;
81    long minCompactSize;
82    int minFilesToCompact;
83    int maxFilesToCompact;
84    double compactionRatio;
85    double offPeekCompactionRatio;
86    long throttlePoint;
87    long majorCompactionPeriod;
88    float majorCompactionJitter;
89    final float minLocalityToForceCompact;
90    private final long dateTieredMaxStoreFileAgeMillis;
91    private final int dateTieredIncomingWindowMin;
92    private final String compactionPolicyForDateTieredWindow;
93    private final boolean dateTieredSingleOutputForMinorCompaction;
94    private final String dateTieredCompactionWindowFactory;
95  
96    CompactionConfiguration(Configuration conf, StoreConfigInformation storeConfigInfo) {
97      this.conf = conf;
98      this.storeConfigInfo = storeConfigInfo;
99  
100     maxCompactSize = conf.getLong(CONFIG_PREFIX + "max.size", Long.MAX_VALUE);
101     minCompactSize = conf.getLong(CONFIG_PREFIX + "min.size",
102         storeConfigInfo.getMemstoreFlushSize());
103     minFilesToCompact = Math.max(2, conf.getInt(MIN_KEY,
104           /*old name*/ conf.getInt("hbase.hstore.compactionThreshold", 3)));
105     maxFilesToCompact = conf.getInt(MAX_KEY, 10);
106     compactionRatio = conf.getFloat(RATIO_KEY, 1.2F);
107     offPeekCompactionRatio = conf.getFloat(CONFIG_PREFIX + "ratio.offpeak", 5.0F);
108 
109     throttlePoint =  conf.getLong("hbase.regionserver.thread.compaction.throttle",
110           2 * maxFilesToCompact * storeConfigInfo.getMemstoreFlushSize());
111     majorCompactionPeriod = conf.getLong(HConstants.MAJOR_COMPACTION_PERIOD, 1000*60*60*24*7);
112     // Make it 0.5 so jitter has us fall evenly either side of when the compaction should run
113     majorCompactionJitter = conf.getFloat("hbase.hregion.majorcompaction.jitter", 0.50F);
114     minLocalityToForceCompact = conf.getFloat(HBASE_HSTORE_MIN_LOCALITY_TO_SKIP_MAJOR_COMPACT, 0f);
115 
116     dateTieredMaxStoreFileAgeMillis = conf.getLong(DATE_TIERED_MAX_AGE_MILLIS_KEY, Long.MAX_VALUE);
117     dateTieredIncomingWindowMin = conf.getInt(DATE_TIERED_INCOMING_WINDOW_MIN_KEY, 6);
118     compactionPolicyForDateTieredWindow = conf.get(
119       COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS_KEY,
120       DEFAULT_COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS.getName());
121     dateTieredSingleOutputForMinorCompaction = conf
122         .getBoolean(DATE_TIERED_SINGLE_OUTPUT_FOR_MINOR_COMPACTION_KEY, true);
123     this.dateTieredCompactionWindowFactory = conf.get(
124       DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS_KEY,
125       DEFAULT_DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS.getName());
126     LOG.info(this);
127   }
128 
129   @Override
130   public String toString() {
131     return String.format(
132       "size [%d, %d); files [%d, %d); ratio %f; off-peak ratio %f; throttle point %d;"
133       + " major period %d, major jitter %f, min locality to compact %f;"
134       + " tiered compaction: max_age %d, incoming window min %d,"
135       + " compaction policy for tiered window %s, single output for minor %b,"
136       + " compaction window factory %s",
137       minCompactSize,
138       maxCompactSize,
139       minFilesToCompact,
140       maxFilesToCompact,
141       compactionRatio,
142       offPeekCompactionRatio,
143       throttlePoint,
144       majorCompactionPeriod,
145       majorCompactionJitter,
146       minLocalityToForceCompact,
147       dateTieredMaxStoreFileAgeMillis,
148       dateTieredIncomingWindowMin,
149       compactionPolicyForDateTieredWindow,
150       dateTieredSingleOutputForMinorCompaction,
151       dateTieredCompactionWindowFactory
152       );
153   }
154 
155   /**
156    * @return lower bound below which compaction is selected without ratio test
157    */
158   long getMinCompactSize() {
159     return minCompactSize;
160   }
161 
162   /**
163    * @return upper bound on file size to be included in minor compactions
164    */
165   long getMaxCompactSize() {
166     return maxCompactSize;
167   }
168 
169   /**
170    * @return upper bound on number of files to be included in minor compactions
171    */
172   public int getMinFilesToCompact() {
173     return minFilesToCompact;
174   }
175 
176   /**
177    * Set upper bound on number of files to be included in minor compactions
178    * @param threshold
179    */
180   public void setMinFilesToCompact(int threshold) {
181     minFilesToCompact = threshold;
182   }
183   
184   /**
185    * @return upper bound on number of files to be included in minor compactions
186    */
187   int getMaxFilesToCompact() {
188     return maxFilesToCompact;
189   }
190 
191   /**
192    * @return Ratio used for compaction
193    */
194   double getCompactionRatio() {
195     return compactionRatio;
196   }
197 
198   /**
199    * @return Off peak Ratio used for compaction
200    */
201   double getCompactionRatioOffPeak() {
202     return offPeekCompactionRatio;
203   }
204 
205   /**
206    * @return ThrottlePoint used for classifying small and large compactions
207    */
208   long getThrottlePoint() {
209     return throttlePoint;
210   }
211 
212   /**
213    * @return Major compaction period from compaction.
214    *   Major compactions are selected periodically according to this parameter plus jitter
215    */
216   long getMajorCompactionPeriod() {
217     return majorCompactionPeriod;
218   }
219 
220   /**
221    * @return Major the jitter fraction, the fraction within which the major compaction
222    *    period is randomly chosen from the majorCompactionPeriod in each store.
223    */
224   float getMajorCompactionJitter() {
225     return majorCompactionJitter;
226   }
227 
228   /**
229    * @return Block locality ratio, the ratio at which we will include old regions with a single
230    *   store file for major compaction.  Used to improve block locality for regions that
231    *   haven't had writes in a while but are still being read.
232    */
233   float getMinLocalityToForceCompact() {
234     return minLocalityToForceCompact;
235   }
236 
237   public long getDateTieredMaxStoreFileAgeMillis() {
238     return dateTieredMaxStoreFileAgeMillis;
239   }
240 
241   public int getDateTieredIncomingWindowMin() {
242     return dateTieredIncomingWindowMin;
243   }
244 
245   public String getCompactionPolicyForDateTieredWindow() {
246     return compactionPolicyForDateTieredWindow;
247   }
248 
249   public boolean useDateTieredSingleOutputForMinorCompaction() {
250     return dateTieredSingleOutputForMinorCompaction;
251   }
252 
253   public String getDateTieredCompactionWindowFactory() {
254     return dateTieredCompactionWindowFactory;
255   }
256 }