View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.regionserver.compactions;
19  
20  import java.io.IOException;
21  import java.util.ArrayList;
22  import java.util.Collection;
23  import java.util.List;
24  import java.util.Map;
25  
26  import org.apache.commons.logging.Log;
27  import org.apache.commons.logging.LogFactory;
28  import org.apache.hadoop.classification.InterfaceAudience;
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.fs.Path;
31  import org.apache.hadoop.hbase.Cell;
32  import org.apache.hadoop.hbase.HConstants;
33  import org.apache.hadoop.hbase.KeyValue;
34  import org.apache.hadoop.hbase.KeyValueUtil;
35  import org.apache.hadoop.hbase.client.Scan;
36  import org.apache.hadoop.hbase.io.CellOutputStream;
37  import org.apache.hadoop.hbase.io.compress.Compression;
38  import org.apache.hadoop.hbase.io.hfile.HFileWriterV2;
39  import org.apache.hadoop.hbase.regionserver.HStore;
40  import org.apache.hadoop.hbase.regionserver.InternalScanner;
41  import org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl;
42  import org.apache.hadoop.hbase.regionserver.ScanType;
43  import org.apache.hadoop.hbase.regionserver.Store;
44  import org.apache.hadoop.hbase.regionserver.StoreFile;
45  import org.apache.hadoop.hbase.regionserver.StoreFileScanner;
46  import org.apache.hadoop.hbase.regionserver.StoreScanner;
47  import org.apache.hadoop.hbase.util.Bytes;
48  import org.apache.hadoop.util.StringUtils;
49  
50  /**
51   * A compactor is a compaction algorithm associated a given policy. Base class also contains
52   * reusable parts for implementing compactors (what is common and what isn't is evolving).
53   */
54  @InterfaceAudience.Private
55  public abstract class Compactor {
56    private static final Log LOG = LogFactory.getLog(Compactor.class);
57    protected CompactionProgress progress;
58    protected Configuration conf;
59    protected Store store;
60  
61    private int compactionKVMax;
62    protected Compression.Algorithm compactionCompression;
63  
64    //TODO: depending on Store is not good but, realistically, all compactors currently do.
65    Compactor(final Configuration conf, final Store store) {
66      this.conf = conf;
67      this.store = store;
68      this.compactionKVMax =
69        this.conf.getInt(HConstants.COMPACTION_KV_MAX, HConstants.COMPACTION_KV_MAX_DEFAULT);
70      this.compactionCompression = (this.store.getFamily() == null) ?
71          Compression.Algorithm.NONE : this.store.getFamily().getCompactionCompression();
72    }
73  
74    /**
75     * TODO: Replace this with {@link CellOutputStream} when StoreFile.Writer uses cells.
76     */
77    public interface CellSink {
78      void append(KeyValue kv) throws IOException;
79    }
80  
81    /**
82     * Do a minor/major compaction on an explicit set of storefiles from a Store.
83     * @param request the requested compaction
84     * @return Product of compaction or an empty list if all cells expired or deleted and nothing made
85     *         it through the compaction.
86     * @throws IOException
87     */
88    public abstract List<Path> compact(final CompactionRequest request) throws IOException;
89  
90    /**
91     * Compact a list of files for testing. Creates a fake {@link CompactionRequest} to pass to
92     * {@link #compact(CompactionRequest)};
93     * @param filesToCompact the files to compact. These are used as the compactionSelection for the
94     *          generated {@link CompactionRequest}.
95     * @param isMajor true to major compact (prune all deletes, max versions, etc)
96     * @return Product of compaction or an empty list if all cells expired or deleted and nothing made
97     *         it through the compaction.
98     * @throws IOException
99     */
100   public List<Path> compactForTesting(final Collection<StoreFile> filesToCompact, boolean isMajor)
101       throws IOException {
102     CompactionRequest cr = new CompactionRequest(filesToCompact);
103     cr.setIsMajor(isMajor);
104     return this.compact(cr);
105   }
106 
107   public CompactionProgress getProgress() {
108     return this.progress;
109   }
110 
111   /** The sole reason this class exists is that java has no ref/out/pointer parameters. */
112   protected static class FileDetails {
113     /** Maximum key count after compaction (for blooms) */
114     public long maxKeyCount = 0;
115     /** Earliest put timestamp if major compaction */
116     public long earliestPutTs = HConstants.LATEST_TIMESTAMP;
117     /** The last key in the files we're compacting. */
118     public long maxSeqId = 0;
119     /** Latest memstore read point found in any of the involved files */
120     public long maxMVCCReadpoint = 0;
121   }
122 
123   protected FileDetails getFileDetails(
124       Collection<StoreFile> filesToCompact, boolean calculatePutTs) throws IOException {
125     FileDetails fd = new FileDetails();
126 
127     for (StoreFile file : filesToCompact) {
128       long seqNum = file.getMaxSequenceId();
129       fd.maxSeqId = Math.max(fd.maxSeqId, seqNum);
130       StoreFile.Reader r = file.getReader();
131       if (r == null) {
132         LOG.warn("Null reader for " + file.getPath());
133         continue;
134       }
135       // NOTE: getFilterEntries could cause under-sized blooms if the user
136       // switches bloom type (e.g. from ROW to ROWCOL)
137       long keyCount = (r.getBloomFilterType() == store.getFamily().getBloomFilterType())
138           ? r.getFilterEntries() : r.getEntries();
139       fd.maxKeyCount += keyCount;
140       // calculate the latest MVCC readpoint in any of the involved store files
141       Map<byte[], byte[]> fileInfo = r.loadFileInfo();
142       byte tmp[] = fileInfo.get(HFileWriterV2.MAX_MEMSTORE_TS_KEY);
143       if (tmp != null) {
144         fd.maxMVCCReadpoint = Math.max(fd.maxMVCCReadpoint, Bytes.toLong(tmp));
145       }
146       // If required, calculate the earliest put timestamp of all involved storefiles.
147       // This is used to remove family delete marker during compaction.
148       long earliestPutTs = 0;
149       if (calculatePutTs) {
150         tmp = fileInfo.get(StoreFile.EARLIEST_PUT_TS);
151         if (tmp == null) {
152           // There's a file with no information, must be an old one
153           // assume we have very old puts
154           fd.earliestPutTs = earliestPutTs = HConstants.OLDEST_TIMESTAMP;
155         } else {
156           earliestPutTs = Bytes.toLong(tmp);
157           fd.earliestPutTs = Math.min(fd.earliestPutTs, earliestPutTs);
158         }
159       }
160       if (LOG.isDebugEnabled()) {
161         LOG.debug("Compacting " + file +
162           ", keycount=" + keyCount +
163           ", bloomtype=" + r.getBloomFilterType().toString() +
164           ", size=" + StringUtils.humanReadableInt(r.length()) +
165           ", encoding=" + r.getHFileReader().getDataBlockEncoding() +
166           ", seqNum=" + seqNum +
167           (calculatePutTs ? ", earliestPutTs=" + earliestPutTs: ""));
168       }
169     }
170     return fd;
171   }
172 
173   protected List<StoreFileScanner> createFileScanners(
174       final Collection<StoreFile> filesToCompact) throws IOException {
175     return StoreFileScanner.getScannersForStoreFiles(filesToCompact, false, false, true);
176   }
177 
178   protected long setSmallestReadPoint() {
179     long smallestReadPoint = store.getSmallestReadPoint();
180     MultiVersionConsistencyControl.setThreadReadPoint(smallestReadPoint);
181     return smallestReadPoint;
182   }
183 
184   protected InternalScanner preCreateCoprocScanner(final CompactionRequest request,
185       ScanType scanType, long earliestPutTs,  List<StoreFileScanner> scanners) throws IOException {
186     if (store.getCoprocessorHost() == null) return null;
187     return store.getCoprocessorHost()
188         .preCompactScannerOpen(store, scanners, scanType, earliestPutTs, request);
189   }
190 
191   protected InternalScanner postCreateCoprocScanner(final CompactionRequest request,
192       ScanType scanType, InternalScanner scanner) throws IOException {
193     if (store.getCoprocessorHost() == null) return scanner;
194     return store.getCoprocessorHost().preCompact(store, scanner, scanType, request);
195   }
196 
197   /**
198    * Performs the compaction.
199    * @param scanner Where to read from.
200    * @param writer Where to write to.
201    * @param smallestReadPoint Smallest read point.
202    * @return Whether compaction ended; false if it was interrupted for some reason.
203    */
204   @SuppressWarnings("deprecation")
205   protected boolean performCompaction(InternalScanner scanner,
206       CellSink writer, long smallestReadPoint) throws IOException {
207     int bytesWritten = 0;
208     // Since scanner.next() can return 'false' but still be delivering data,
209     // we have to use a do/while loop.
210     List<Cell> kvs = new ArrayList<Cell>();
211     // Limit to "hbase.hstore.compaction.kv.max" (default 10) to avoid OOME
212     int closeCheckInterval = HStore.getCloseCheckInterval();
213     boolean hasMore;
214     do {
215       hasMore = scanner.next(kvs, compactionKVMax);
216       // output to writer:
217       for (Cell c : kvs) {
218         KeyValue kv = KeyValueUtil.ensureKeyValue(c);
219         if (kv.getMvccVersion() <= smallestReadPoint) {
220           kv.setMvccVersion(0);
221         }
222         writer.append(kv);
223         ++progress.currentCompactedKVs;
224 
225         // check periodically to see if a system stop is requested
226         if (closeCheckInterval > 0) {
227           bytesWritten += kv.getLength();
228           if (bytesWritten > closeCheckInterval) {
229             bytesWritten = 0;
230             if (!store.areWritesEnabled()) {
231               progress.cancel();
232               return false;
233             }
234           }
235         }
236       }
237       kvs.clear();
238     } while (hasMore);
239     progress.complete();
240     return true;
241   }
242 
243   protected void abortWriter(final StoreFile.Writer writer) throws IOException {
244     writer.close();
245     store.getFileSystem().delete(writer.getPath(), false);
246   }
247 
248   /**
249    * @param scanners Store file scanners.
250    * @param scanType Scan type.
251    * @param smallestReadPoint Smallest MVCC read point.
252    * @param earliestPutTs Earliest put across all files.
253    * @return A compaction scanner.
254    */
255   protected InternalScanner createScanner(Store store, List<StoreFileScanner> scanners,
256       ScanType scanType, long smallestReadPoint, long earliestPutTs) throws IOException {
257     Scan scan = new Scan();
258     scan.setMaxVersions(store.getFamily().getMaxVersions());
259     return new StoreScanner(store, store.getScanInfo(), scan, scanners,
260         scanType, smallestReadPoint, earliestPutTs);
261   }
262 }