1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver.compactions;
21
22 import java.io.IOException;
23 import java.util.ArrayList;
24 import java.util.List;
25
26 import org.apache.commons.logging.Log;
27 import org.apache.commons.logging.LogFactory;
28 import org.apache.hadoop.classification.InterfaceAudience;
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.hbase.regionserver.StoreConfigInformation;
31 import org.apache.hadoop.hbase.regionserver.StoreFile;
32
33
34
35
36
37
38
39 @InterfaceAudience.Private
40 public class ExploringCompactionPolicy extends RatioBasedCompactionPolicy {
41 private static final Log LOG = LogFactory.getLog(ExploringCompactionPolicy.class);
42
43
44
45
46
47
48 public ExploringCompactionPolicy(final Configuration conf,
49 final StoreConfigInformation storeConfigInfo) {
50 super(conf, storeConfigInfo);
51 }
52
53 @Override
54 final ArrayList<StoreFile> applyCompactionPolicy(final ArrayList<StoreFile> candidates,
55 final boolean mayUseOffPeak, final boolean mightBeStuck) throws IOException {
56
57 List<StoreFile> bestSelection = new ArrayList<StoreFile>(0);
58 List<StoreFile> smallest = new ArrayList<StoreFile>(0);
59 long bestSize = 0;
60 long smallestSize = Long.MAX_VALUE;
61
62 int opts = 0, optsInRatio = 0, bestStart = -1;
63
64 for (int start = 0; start < candidates.size(); start++) {
65
66 for (int currentEnd = start + comConf.getMinFilesToCompact() - 1;
67 currentEnd < candidates.size(); currentEnd++) {
68 List<StoreFile> potentialMatchFiles = candidates.subList(start, currentEnd + 1);
69
70
71 if (potentialMatchFiles.size() < comConf.getMinFilesToCompact()) {
72 continue;
73 }
74 if (potentialMatchFiles.size() > comConf.getMaxFilesToCompact()) {
75 continue;
76 }
77
78
79
80 long size = getTotalStoreSize(potentialMatchFiles);
81
82
83
84 if (size < smallestSize) {
85 smallest = potentialMatchFiles;
86 smallestSize = size;
87 }
88
89 if (size > comConf.getMaxCompactSize()) {
90 continue;
91 }
92
93 ++opts;
94 if (size >= comConf.getMinCompactSize()
95 && !filesInRatio(potentialMatchFiles, mayUseOffPeak)) {
96 continue;
97 }
98
99 ++optsInRatio;
100 if (isBetterSelection(bestSelection, bestSize, potentialMatchFiles, size, mightBeStuck)) {
101 bestSelection = potentialMatchFiles;
102 bestSize = size;
103 bestStart = start;
104 }
105 }
106 }
107 if (bestSelection.size() == 0 && mightBeStuck) {
108 LOG.debug("Exploring compaction algorithm has selected " + smallest.size()
109 + " files of size "+ smallestSize + " because the store might be stuck");
110 return new ArrayList<StoreFile>(smallest);
111 }
112 LOG.debug("Exploring compaction algorithm has selected " + bestSelection.size()
113 + " files of size " + bestSize + " starting at candidate #" + bestStart +
114 " after considering " + opts + " permutations with " + optsInRatio + " in ratio");
115 return new ArrayList<StoreFile>(bestSelection);
116 }
117
118 private boolean isBetterSelection(List<StoreFile> bestSelection,
119 long bestSize, List<StoreFile> selection, long size, boolean mightBeStuck) {
120 if (mightBeStuck && bestSize > 0 && size > 0) {
121
122
123
124
125 final double REPLACE_IF_BETTER_BY = 1.05;
126 double thresholdQuality = ((double)bestSelection.size() / bestSize) * REPLACE_IF_BETTER_BY;
127 return thresholdQuality < ((double)selection.size() / size);
128 }
129
130 return selection.size() > bestSelection.size()
131 || (selection.size() == bestSelection.size() && size < bestSize);
132 }
133
134
135
136
137
138
139 private long getTotalStoreSize(final List<StoreFile> potentialMatchFiles) {
140 long size = 0;
141
142 for (StoreFile s:potentialMatchFiles) {
143 size += s.getReader().length();
144 }
145 return size;
146 }
147
148
149
150
151
152
153
154
155
156 private boolean filesInRatio(final List<StoreFile> files, final boolean isOffPeak) {
157 if (files.size() < 2) {
158 return true;
159 }
160 final double currentRatio =
161 isOffPeak ? comConf.getCompactionRatioOffPeak() : comConf.getCompactionRatio();
162
163 long totalFileSize = getTotalStoreSize(files);
164
165 for (StoreFile file : files) {
166 long singleFileSize = file.getReader().length();
167 long sumAllOtherFileSizes = totalFileSize - singleFileSize;
168
169 if (singleFileSize > sumAllOtherFileSizes * currentRatio) {
170 return false;
171 }
172 }
173 return true;
174 }
175 }