1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver.compactions;
20
21 import java.io.IOException;
22 import java.util.ArrayList;
23 import java.util.Collection;
24 import java.util.Collections;
25 import java.util.List;
26
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.hbase.regionserver.StoreConfigInformation;
31 import org.apache.hadoop.hbase.regionserver.StoreFile;
32 import org.apache.hadoop.hbase.util.Pair;
33 import org.apache.hadoop.hbase.util.ReflectionUtils;
34
35 import com.google.common.annotations.VisibleForTesting;
36 import com.google.common.base.Predicate;
37 import com.google.common.collect.ImmutableList;
38 import com.google.common.collect.Iterables;
39 import com.google.common.collect.Iterators;
40 import com.google.common.collect.Lists;
41 import com.google.common.collect.PeekingIterator;
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57 public class DateTieredCompactionPolicy extends RatioBasedCompactionPolicy {
58 private static final Log LOG = LogFactory.getLog(DateTieredCompactionPolicy.class);
59
60 private RatioBasedCompactionPolicy compactionPolicyPerWindow;
61
62 public DateTieredCompactionPolicy(Configuration conf, StoreConfigInformation storeConfigInfo)
63 throws IOException {
64 super(conf, storeConfigInfo);
65 try {
66 compactionPolicyPerWindow =
67 ReflectionUtils.instantiateWithCustomCtor(comConf.getCompactionPolicyForTieredWindow(),
68 new Class[] { Configuration.class, StoreConfigInformation.class }, new Object[] { conf,
69 storeConfigInfo });
70 } catch (Exception e) {
71 throw new IOException("Unable to load configured compaction policy '"
72 + comConf.getCompactionPolicyForTieredWindow() + "'", e);
73 }
74 }
75
76 @Override
77 public boolean isMajorCompaction(Collection<StoreFile> filesToCompact) throws IOException {
78
79 return false;
80 }
81
82 @Override
83
84
85
86 public boolean needsCompaction(final Collection<StoreFile> storeFiles,
87 final List<StoreFile> filesCompacting) {
88 return needsCompaction(storeFiles, filesCompacting, System.currentTimeMillis());
89 }
90
91 @VisibleForTesting
92 public boolean needsCompaction(final Collection<StoreFile> storeFiles,
93 final List<StoreFile> filesCompacting, long now) {
94 ArrayList<StoreFile> candidates = new ArrayList<StoreFile>(storeFiles);
95 candidates = filterBulk(candidates);
96 candidates = skipLargeFiles(candidates);
97 try {
98 candidates = applyCompactionPolicy(candidates, true, false, now);
99 } catch (Exception e) {
100 LOG.error("Can not check for compaction: ", e);
101 return false;
102 }
103 return candidates != null;
104 }
105
106 @Override
107
108
109
110
111 public ArrayList<StoreFile> applyCompactionPolicy(ArrayList<StoreFile> candidates,
112 boolean mayUseOffPeak, boolean mayBeStuck) throws IOException {
113 return applyCompactionPolicy(candidates, mayUseOffPeak, mayBeStuck,
114 System.currentTimeMillis());
115 }
116
117 @VisibleForTesting
118 public ArrayList<StoreFile> applyCompactionPolicy(ArrayList<StoreFile> candidates,
119 boolean mayUseOffPeak, boolean mayBeStuck, long now) throws IOException {
120 Iterable<StoreFile> candidatesInWindow =
121 filterOldStoreFiles(Lists.newArrayList(candidates), comConf.getMaxStoreFileAgeMillis(), now);
122
123 List<ArrayList<StoreFile>> buckets =
124 partitionFilesToBuckets(candidatesInWindow, comConf.getBaseWindowMillis(),
125 comConf.getWindowsPerTier(), now);
126 LOG.debug("Compaction buckets are: " + buckets);
127 if (buckets.size() >= storeConfigInfo.getBlockingFileCount()) {
128 LOG.warn("Number of compaction buckets:" + buckets.size()
129 + ", exceeds blocking file count setting: "
130 + storeConfigInfo.getBlockingFileCount()
131 + ", either increase hbase.hstore.blockingStoreFiles or "
132 + "reduce the number of tiered compaction windows");
133 }
134
135 return newestBucket(buckets, comConf.getIncomingWindowMin(), now, comConf.getBaseWindowMillis(),
136 mayUseOffPeak);
137 }
138
139
140
141
142
143
144
145
146
147
148 private ArrayList<StoreFile> newestBucket(List<ArrayList<StoreFile>> buckets,
149 int incomingWindowThreshold, long now, long baseWindowMillis, boolean mayUseOffPeak)
150 throws IOException {
151 Window incomingWindow = getInitialWindow(now, baseWindowMillis);
152 for (ArrayList<StoreFile> bucket : buckets) {
153 int minThreshold = incomingWindow.compareToTimestamp(bucket.get(0).getMaximumTimestamp())
154 <= 0? comConf.getIncomingWindowMin() : comConf.minFilesToCompact;
155 compactionPolicyPerWindow.setMinThreshold(minThreshold);
156 ArrayList<StoreFile> candidates = compactionPolicyPerWindow.applyCompactionPolicy(bucket,
157 mayUseOffPeak, false);
158 if (candidates != null && !candidates.isEmpty()) {
159 return candidates;
160 }
161 }
162 return null;
163 }
164
165
166
167
168
169
170
171
172
173 private static List<ArrayList<StoreFile>> partitionFilesToBuckets(Iterable<StoreFile> storeFiles,
174 long baseWindowSizeMillis, int windowsPerTier, long now) {
175 List<ArrayList<StoreFile>> buckets = Lists.newArrayList();
176 Window window = getInitialWindow(now, baseWindowSizeMillis);
177
178 List<Pair<StoreFile, Long>> storefileMaxTimestampPairs =
179 Lists.newArrayListWithCapacity(Iterables.size(storeFiles));
180 long maxTimestampSeen = Long.MIN_VALUE;
181 for (StoreFile storeFile : storeFiles) {
182
183
184 maxTimestampSeen = Math.max(maxTimestampSeen, storeFile.getMaximumTimestamp());
185 storefileMaxTimestampPairs.add(new Pair<StoreFile, Long>(storeFile, maxTimestampSeen));
186 }
187
188 Collections.reverse(storefileMaxTimestampPairs);
189 PeekingIterator<Pair<StoreFile, Long>> it =
190 Iterators.peekingIterator(storefileMaxTimestampPairs.iterator());
191
192 while (it.hasNext()) {
193 int compResult = window.compareToTimestamp(it.peek().getSecond());
194 if (compResult > 0) {
195
196 window = window.nextWindow(windowsPerTier);
197 } else {
198
199 ArrayList<StoreFile> bucket = Lists.newArrayList();
200
201
202 while (it.hasNext() && window.compareToTimestamp(it.peek().getSecond()) <= 0) {
203 bucket.add(it.next().getFirst());
204 }
205 if (!bucket.isEmpty()) {
206 buckets.add(bucket);
207 }
208 }
209 }
210
211 return buckets;
212 }
213
214
215
216
217
218
219
220
221 private static Iterable<StoreFile> filterOldStoreFiles(List<StoreFile> storeFiles, long maxAge,
222 long now) {
223 if (maxAge == 0) return ImmutableList.of();
224 final long cutoff = now - maxAge;
225 return Iterables.filter(storeFiles, new Predicate<StoreFile>() {
226 @Override
227 public boolean apply(StoreFile storeFile) {
228
229 if (storeFile == null) {
230 return false;
231 }
232 return storeFile.getMaximumTimestamp() >= cutoff;
233 }
234 });
235 }
236
237
238
239
240
241 private static Window getInitialWindow(long now, long timeUnit) {
242 return new Window(timeUnit, now / timeUnit);
243 }
244
245 private static class Window {
246
247
248
249 private final long windowMillis;
250
251
252
253 private final long divPosition;
254
255 public Window(long baseWindowMillis, long divPosition) {
256 this.windowMillis = baseWindowMillis;
257 this.divPosition = divPosition;
258 }
259
260
261
262
263
264
265
266 public int compareToTimestamp(long timestamp) {
267 long pos = timestamp / windowMillis;
268 return divPosition == pos ? 0 : divPosition < pos ? -1 : 1;
269 }
270
271
272
273
274
275
276
277
278 public Window nextWindow(int windowsPerTier) {
279 if (divPosition % windowsPerTier > 0) return new Window(windowMillis, divPosition - 1);
280 else return new Window(windowMillis * windowsPerTier, divPosition / windowsPerTier - 1);
281 }
282 }
283 }