View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.regionserver.compactions;
21  
22  import org.apache.commons.logging.Log;
23  import org.apache.commons.logging.LogFactory;
24  import org.apache.hadoop.conf.Configuration;
25  import org.apache.hadoop.hbase.HConstants;
26  import org.apache.hadoop.hbase.classification.InterfaceAudience;
27  import org.apache.hadoop.hbase.regionserver.StoreConfigInformation;
28  
29  /**
30   * Compaction configuration for a particular instance of HStore.
31   * Takes into account both global settings and ones set on the column family/store.
32   * Control knobs for default compaction algorithm:
33   * <p/>
34   * maxCompactSize - upper bound on file size to be included in minor compactions
35   * minCompactSize - lower bound below which compaction is selected without ratio test
36   * minFilesToCompact - lower bound on number of files in any minor compaction
37   * maxFilesToCompact - upper bound on number of files in any minor compaction
38   * compactionRatio - Ratio used for compaction
39   * <p/>
40   * Set parameter as "hbase.hstore.compaction.<attribute>"
41   */
42  
43  //TODO: revisit this class for online parameter updating (both in xml and on the CF)
44  @InterfaceAudience.Private
45  public class CompactionConfiguration {
46  
47    static final Log LOG = LogFactory.getLog(CompactionConfiguration.class);
48  
49    private static final String CONFIG_PREFIX = "hbase.hstore.compaction.";
50    public static final String HBASE_HSTORE_MIN_LOCALITY_TO_SKIP_MAJOR_COMPACT =
51        "hbase.hstore.min.locality.to.skip.major.compact";
52    public static final String RATIO_KEY = CONFIG_PREFIX + "ratio";
53    public static final String MIN_KEY = CONFIG_PREFIX + "min";
54    public static final String MAX_KEY = CONFIG_PREFIX + "max";
55  
56    /*
57     * The epoch time length for the windows we no longer compact
58     */
59    public static final String MAX_AGE_KEY =CONFIG_PREFIX + "date.tiered.max.storefile.age.millis";
60    public static final String BASE_WINDOW_MILLIS_KEY =
61      CONFIG_PREFIX + "date.tiered.base.window.millis";
62    public static final String WINDOWS_PER_TIER_KEY = CONFIG_PREFIX + "date.tiered.windows.per.tier";
63    public static final String INCOMING_WINDOW_MIN_KEY =
64      CONFIG_PREFIX + "date.tiered.incoming.window.min";
65    public static final String COMPACTION_POLICY_CLASS_FOR_TIERED_WINDOWS_KEY =
66        CONFIG_PREFIX + "date.tiered.window.policy.class";
67  
68    private static final Class<? extends RatioBasedCompactionPolicy>
69      DEFAULT_TIER_COMPACTION_POLICY_CLASS = ExploringCompactionPolicy.class;
70  
71    Configuration conf;
72    StoreConfigInformation storeConfigInfo;
73  
74    long maxCompactSize;
75    long minCompactSize;
76    int minFilesToCompact;
77    int maxFilesToCompact;
78    double compactionRatio;
79    double offPeekCompactionRatio;
80    long throttlePoint;
81    long majorCompactionPeriod;
82    float majorCompactionJitter;
83    final float minLocalityToForceCompact;
84    private final long maxStoreFileAgeMillis;
85    private final long baseWindowMillis;
86    private final int windowsPerTier;
87    private final int incomingWindowMin;
88    private final String compactionPolicyForTieredWindow;
89  
90    CompactionConfiguration(Configuration conf, StoreConfigInformation storeConfigInfo) {
91      this.conf = conf;
92      this.storeConfigInfo = storeConfigInfo;
93  
94      maxCompactSize = conf.getLong(CONFIG_PREFIX + "max.size", Long.MAX_VALUE);
95      minCompactSize = conf.getLong(CONFIG_PREFIX + "min.size",
96          storeConfigInfo.getMemstoreFlushSize());
97      minFilesToCompact = Math.max(2, conf.getInt(MIN_KEY,
98            /*old name*/ conf.getInt("hbase.hstore.compactionThreshold", 3)));
99      maxFilesToCompact = conf.getInt(MAX_KEY, 10);
100     compactionRatio = conf.getFloat(RATIO_KEY, 1.2F);
101     offPeekCompactionRatio = conf.getFloat(CONFIG_PREFIX + "ratio.offpeak", 5.0F);
102 
103     throttlePoint =  conf.getLong("hbase.regionserver.thread.compaction.throttle",
104           2 * maxFilesToCompact * storeConfigInfo.getMemstoreFlushSize());
105     majorCompactionPeriod = conf.getLong(HConstants.MAJOR_COMPACTION_PERIOD, 1000*60*60*24*7);
106     // Make it 0.5 so jitter has us fall evenly either side of when the compaction should run
107     majorCompactionJitter = conf.getFloat("hbase.hregion.majorcompaction.jitter", 0.50F);
108     minLocalityToForceCompact = conf.getFloat(HBASE_HSTORE_MIN_LOCALITY_TO_SKIP_MAJOR_COMPACT, 0f);
109 
110     maxStoreFileAgeMillis = conf.getLong(MAX_AGE_KEY, Long.MAX_VALUE);
111     baseWindowMillis = conf.getLong(BASE_WINDOW_MILLIS_KEY, 3600000 * 6);
112     windowsPerTier = conf.getInt(WINDOWS_PER_TIER_KEY, 4);
113     incomingWindowMin = conf.getInt(INCOMING_WINDOW_MIN_KEY, 6);
114     compactionPolicyForTieredWindow = conf.get(COMPACTION_POLICY_CLASS_FOR_TIERED_WINDOWS_KEY,
115         DEFAULT_TIER_COMPACTION_POLICY_CLASS.getName());
116     LOG.info(this);
117   }
118 
119   @Override
120   public String toString() {
121     return String.format(
122       "size [%d, %d); files [%d, %d); ratio %f; off-peak ratio %f; throttle point %d;"
123       + " major period %d, major jitter %f, min locality to compact %f;"
124       + " tiered compaction: max_age %d, base window in milliseconds %d, windows per tier %d, "
125       + "incoming window threshold %d",
126       minCompactSize,
127       maxCompactSize,
128       minFilesToCompact,
129       maxFilesToCompact,
130       compactionRatio,
131       offPeekCompactionRatio,
132       throttlePoint,
133       majorCompactionPeriod,
134       majorCompactionJitter,
135       minLocalityToForceCompact,
136       maxStoreFileAgeMillis,
137       baseWindowMillis,
138       windowsPerTier,
139       incomingWindowMin);
140   }
141 
142   /**
143    * @return lower bound below which compaction is selected without ratio test
144    */
145   long getMinCompactSize() {
146     return minCompactSize;
147   }
148 
149   /**
150    * @return upper bound on file size to be included in minor compactions
151    */
152   long getMaxCompactSize() {
153     return maxCompactSize;
154   }
155 
156   /**
157    * @return upper bound on number of files to be included in minor compactions
158    */
159   public int getMinFilesToCompact() {
160     return minFilesToCompact;
161   }
162 
163   /**
164    * Set upper bound on number of files to be included in minor compactions
165    * @param threshold
166    */
167   public void setMinFilesToCompact(int threshold) {
168     minFilesToCompact = threshold;
169   }
170 
171   /**
172    * @return upper bound on number of files to be included in minor compactions
173    */
174   int getMaxFilesToCompact() {
175     return maxFilesToCompact;
176   }
177 
178   /**
179    * @return Ratio used for compaction
180    */
181   double getCompactionRatio() {
182     return compactionRatio;
183   }
184 
185   /**
186    * @return Off peak Ratio used for compaction
187    */
188   double getCompactionRatioOffPeak() {
189     return offPeekCompactionRatio;
190   }
191 
192   /**
193    * @return ThrottlePoint used for classifying small and large compactions
194    */
195   long getThrottlePoint() {
196     return throttlePoint;
197   }
198 
199   /**
200    * @return Major compaction period from compaction.
201    *   Major compactions are selected periodically according to this parameter plus jitter
202    */
203   long getMajorCompactionPeriod() {
204     return majorCompactionPeriod;
205   }
206 
207   /**
208    * @return Major the jitter fraction, the fraction within which the major compaction
209    *    period is randomly chosen from the majorCompactionPeriod in each store.
210    */
211   float getMajorCompactionJitter() {
212     return majorCompactionJitter;
213   }
214 
215   /**
216    * @return Block locality ratio, the ratio at which we will include old regions with a single
217    *   store file for major compaction.  Used to improve block locality for regions that
218    *   haven't had writes in a while but are still being read.
219    */
220   float getMinLocalityToForceCompact() {
221     return minLocalityToForceCompact;
222   }
223 
224   public long getMaxStoreFileAgeMillis() {
225     return maxStoreFileAgeMillis;
226   }
227 
228   public long getBaseWindowMillis() {
229     return baseWindowMillis;
230   }
231 
232   public int getWindowsPerTier() {
233     return windowsPerTier;
234   }
235 
236   public int getIncomingWindowMin() {
237     return incomingWindowMin;
238   }
239 
240   public String getCompactionPolicyForTieredWindow() {
241     return compactionPolicyForTieredWindow;
242   }
243 }