View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import static org.apache.hadoop.hbase.regionserver.StripeStoreFileManager.OPEN_KEY;
22  
23  import java.io.IOException;
24  import java.util.ArrayList;
25  import java.util.List;
26  import java.util.SortedSet;
27  import java.util.concurrent.atomic.AtomicLong;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.conf.Configuration;
32  import org.apache.hadoop.fs.Path;
33  import org.apache.hadoop.hbase.KeyValue;
34  import org.apache.hadoop.hbase.KeyValue.KVComparator;
35  import org.apache.hadoop.hbase.classification.InterfaceAudience;
36  import org.apache.hadoop.hbase.monitoring.MonitoredTask;
37  import org.apache.hadoop.hbase.regionserver.StoreFile.Writer;
38  import org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy;
39  
40  import com.google.common.annotations.VisibleForTesting;
41  
42  /**
43   * Stripe implementation of StoreFlusher. Flushes files either into L0 file w/o metadata, or
44   * into separate striped files, avoiding L0.
45   */
46  @InterfaceAudience.Private
47  public class StripeStoreFlusher extends StoreFlusher {
48    private static final Log LOG = LogFactory.getLog(StripeStoreFlusher.class);
49    private final Object flushLock = new Object();
50    private final StripeCompactionPolicy policy;
51    private final StripeCompactionPolicy.StripeInformationProvider stripes;
52  
53    public StripeStoreFlusher(Configuration conf, Store store,
54        StripeCompactionPolicy policy, StripeStoreFileManager stripes) {
55      super(conf, store);
56      this.policy = policy;
57      this.stripes = stripes;
58    }
59  
60    @Override
61    public List<Path> flushSnapshot(SortedSet<KeyValue> snapshot, long cacheFlushSeqNum,
62        final TimeRangeTracker tracker, AtomicLong flushedSize, MonitoredTask status)
63            throws IOException {
64      List<Path> result = new ArrayList<Path>();
65      int kvCount = snapshot.size();
66      if (kvCount == 0) return result; // don't flush if there are no entries
67  
68      long smallestReadPoint = store.getSmallestReadPoint();
69      InternalScanner scanner = createScanner(snapshot, smallestReadPoint);
70      if (scanner == null) {
71        return result; // NULL scanner returned from coprocessor hooks means skip normal processing
72      }
73  
74      // Let policy select flush method.
75      StripeFlushRequest req = this.policy.selectFlush(store.getComparator(), this.stripes,
76        kvCount);
77  
78      long flushedBytes = 0;
79      boolean success = false;
80      StripeMultiFileWriter mw = null;
81      try {
82        mw = req.createWriter(); // Writer according to the policy.
83        StripeMultiFileWriter.WriterFactory factory = createWriterFactory(tracker, kvCount);
84        StoreScanner storeScanner = (scanner instanceof StoreScanner) ? (StoreScanner)scanner : null;
85        mw.init(storeScanner, factory);
86  
87        synchronized (flushLock) {
88          flushedBytes = performFlush(scanner, mw, smallestReadPoint);
89          result = mw.commitWriters(cacheFlushSeqNum, false);
90          success = true;
91        }
92      } finally {
93        if (!success && (mw != null)) {
94          for (Path leftoverFile : mw.abortWriters()) {
95            try {
96              store.getFileSystem().delete(leftoverFile, false);
97            } catch (Exception e) {
98              LOG.error("Failed to delete a file after failed flush: " + e);
99            }
100         }
101       }
102       flushedSize.set(flushedBytes);
103       try {
104         scanner.close();
105       } catch (IOException ex) {
106         LOG.warn("Failed to close flush scanner, ignoring", ex);
107       }
108     }
109     return result;
110   }
111 
112   private StripeMultiFileWriter.WriterFactory createWriterFactory(
113       final TimeRangeTracker tracker, final long kvCount) {
114     return new StripeMultiFileWriter.WriterFactory() {
115       @Override
116       public Writer createWriter() throws IOException {
117         StoreFile.Writer writer = store.createWriterInTmp(
118             kvCount, store.getFamily().getCompression(),
119             /* isCompaction = */ false,
120             /* includeMVCCReadpoint = */ true,
121             /* includesTags = */ true,
122             /* shouldDropBehind = */ false);
123         writer.setTimeRangeTracker(tracker);
124         return writer;
125       }
126     };
127   }
128 
129   /** Stripe flush request wrapper that writes a non-striped file. */
130   public static class StripeFlushRequest {
131 
132     protected final KVComparator comparator;
133 
134     public StripeFlushRequest(KVComparator comparator) {
135       this.comparator = comparator;
136     }
137 
138     @VisibleForTesting
139     public StripeMultiFileWriter createWriter() throws IOException {
140       StripeMultiFileWriter writer = new StripeMultiFileWriter.SizeMultiWriter(comparator, 1,
141           Long.MAX_VALUE, OPEN_KEY, OPEN_KEY);
142       writer.setNoStripeMetadata();
143       return writer;
144     }
145   }
146 
147   /** Stripe flush request wrapper based on boundaries. */
148   public static class BoundaryStripeFlushRequest extends StripeFlushRequest {
149     private final List<byte[]> targetBoundaries;
150 
151     /** @param targetBoundaries New files should be written with these boundaries. */
152     public BoundaryStripeFlushRequest(KVComparator comparator, List<byte[]> targetBoundaries) {
153       super(comparator);
154       this.targetBoundaries = targetBoundaries;
155     }
156 
157     @Override
158     public StripeMultiFileWriter createWriter() throws IOException {
159       return new StripeMultiFileWriter.BoundaryMultiWriter(comparator, targetBoundaries, null,
160           null);
161     }
162   }
163 
164   /** Stripe flush request wrapper based on size. */
165   public static class SizeStripeFlushRequest extends StripeFlushRequest {
166     private final int targetCount;
167     private final long targetKvs;
168 
169     /**
170      * @param targetCount The maximum number of stripes to flush into.
171      * @param targetKvs The KV count of each segment. If targetKvs*targetCount is less than
172      *                  total number of kvs, all the overflow data goes into the last stripe.
173      */
174     public SizeStripeFlushRequest(KVComparator comparator, int targetCount, long targetKvs) {
175       super(comparator);
176       this.targetCount = targetCount;
177       this.targetKvs = targetKvs;
178     }
179 
180     @Override
181     public StripeMultiFileWriter createWriter() throws IOException {
182       return new StripeMultiFileWriter.SizeMultiWriter(comparator, this.targetCount, this.targetKvs,
183           OPEN_KEY, OPEN_KEY);
184     }
185   }
186 }