View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.coprocessor;
21  
22  import static org.junit.Assert.assertEquals;
23  import static org.junit.Assert.assertNull;
24  
25  import java.io.IOException;
26  import java.util.Collection;
27  import java.util.Collections;
28  import java.util.List;
29  import java.util.NavigableSet;
30  import java.util.concurrent.CountDownLatch;
31  
32  import org.apache.hadoop.conf.Configuration;
33  import org.apache.hadoop.fs.FileSystem;
34  import org.apache.hadoop.fs.Path;
35  import org.apache.hadoop.hbase.Cell;
36  import org.apache.hadoop.hbase.Coprocessor;
37  import org.apache.hadoop.hbase.HBaseConfiguration;
38  import org.apache.hadoop.hbase.HBaseTestingUtility;
39  import org.apache.hadoop.hbase.HColumnDescriptor;
40  import org.apache.hadoop.hbase.HConstants;
41  import org.apache.hadoop.hbase.HRegionInfo;
42  import org.apache.hadoop.hbase.HTableDescriptor;
43  import org.apache.hadoop.hbase.MediumTests;
44  import org.apache.hadoop.hbase.TableName;
45  import org.apache.hadoop.hbase.client.Get;
46  import org.apache.hadoop.hbase.client.HBaseAdmin;
47  import org.apache.hadoop.hbase.client.HTable;
48  import org.apache.hadoop.hbase.client.Put;
49  import org.apache.hadoop.hbase.client.Result;
50  import org.apache.hadoop.hbase.client.Scan;
51  import org.apache.hadoop.hbase.filter.FilterBase;
52  import org.apache.hadoop.hbase.regionserver.HRegion;
53  import org.apache.hadoop.hbase.regionserver.HRegionServer;
54  import org.apache.hadoop.hbase.regionserver.InternalScanner;
55  import org.apache.hadoop.hbase.regionserver.KeyValueScanner;
56  import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
57  import org.apache.hadoop.hbase.regionserver.RegionServerServices;
58  import org.apache.hadoop.hbase.regionserver.ScanType;
59  import org.apache.hadoop.hbase.regionserver.Store;
60  import org.apache.hadoop.hbase.regionserver.StoreScanner;
61  import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
62  import org.apache.hadoop.hbase.regionserver.wal.HLog;
63  import org.apache.hadoop.hbase.util.Bytes;
64  import org.junit.Test;
65  import org.junit.experimental.categories.Category;
66  
67  @Category(MediumTests.class)
68  public class TestRegionObserverScannerOpenHook {
69    private static HBaseTestingUtility UTIL = new HBaseTestingUtility();
70    static final Path DIR = UTIL.getDataTestDir();
71  
72    public static class NoDataFilter extends FilterBase {
73  
74      @Override
75      public ReturnCode filterKeyValue(Cell ignored) throws IOException {
76        return ReturnCode.SKIP;
77      }
78  
79      @Override
80      public boolean filterAllRemaining() throws IOException {
81        return true;
82      }
83  
84      @Override
85      public boolean filterRow() throws IOException {
86        return true;
87      }
88    }
89  
90    /**
91     * Do the same logic as the {@link BaseRegionObserver}. Needed since {@link BaseRegionObserver} is
92     * an abstract class.
93     */
94    public static class EmptyRegionObsever extends BaseRegionObserver {
95    }
96  
97    /**
98     * Don't return any data from a scan by creating a custom {@link StoreScanner}.
99     */
100   public static class NoDataFromScan extends BaseRegionObserver {
101     @Override
102     public KeyValueScanner preStoreScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c,
103         Store store, Scan scan, NavigableSet<byte[]> targetCols, KeyValueScanner s)
104         throws IOException {
105       scan.setFilter(new NoDataFilter());
106       return new StoreScanner(store, store.getScanInfo(), scan, targetCols);
107     }
108   }
109 
110   /**
111    * Don't allow any data in a flush by creating a custom {@link StoreScanner}.
112    */
113   public static class NoDataFromFlush extends BaseRegionObserver {
114     @Override
115     public InternalScanner preFlushScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c,
116         Store store, KeyValueScanner memstoreScanner, InternalScanner s) throws IOException {
117       Scan scan = new Scan();
118       scan.setFilter(new NoDataFilter());
119       return new StoreScanner(store, store.getScanInfo(), scan,
120           Collections.singletonList(memstoreScanner), ScanType.COMPACT_RETAIN_DELETES,
121           store.getSmallestReadPoint(), HConstants.OLDEST_TIMESTAMP);
122     }
123   }
124 
125   /**
126    * Don't allow any data to be written out in the compaction by creating a custom
127    * {@link StoreScanner}.
128    */
129   public static class NoDataFromCompaction extends BaseRegionObserver {
130     @Override
131     public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c,
132         Store store, List<? extends KeyValueScanner> scanners, ScanType scanType,
133         long earliestPutTs, InternalScanner s) throws IOException {
134       Scan scan = new Scan();
135       scan.setFilter(new NoDataFilter());
136       return new StoreScanner(store, store.getScanInfo(), scan, scanners,
137           ScanType.COMPACT_RETAIN_DELETES, store.getSmallestReadPoint(),
138           HConstants.OLDEST_TIMESTAMP);
139     }
140   }
141 
142   HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
143       byte[]... families) throws IOException {
144     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
145     for (byte[] family : families) {
146       htd.addFamily(new HColumnDescriptor(family));
147     }
148     HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
149     Path path = new Path(DIR + callingMethod);
150     HRegion r = HRegion.createHRegion(info, path, conf, htd);
151     // this following piece is a hack. currently a coprocessorHost
152     // is secretly loaded at OpenRegionHandler. we don't really
153     // start a region server here, so just manually create cphost
154     // and set it to region.
155     RegionCoprocessorHost host = new RegionCoprocessorHost(r, null, conf);
156     r.setCoprocessorHost(host);
157     return r;
158   }
159 
160   @Test
161   public void testRegionObserverScanTimeStacking() throws Exception {
162     byte[] ROW = Bytes.toBytes("testRow");
163     byte[] TABLE = Bytes.toBytes(getClass().getName());
164     byte[] A = Bytes.toBytes("A");
165     byte[][] FAMILIES = new byte[][] { A };
166 
167     Configuration conf = HBaseConfiguration.create();
168     HRegion region = initHRegion(TABLE, getClass().getName(), conf, FAMILIES);
169     RegionCoprocessorHost h = region.getCoprocessorHost();
170     h.load(NoDataFromScan.class, Coprocessor.PRIORITY_HIGHEST, conf);
171     h.load(EmptyRegionObsever.class, Coprocessor.PRIORITY_USER, conf);
172 
173     Put put = new Put(ROW);
174     put.add(A, A, A);
175     region.put(put);
176 
177     Get get = new Get(ROW);
178     Result r = region.get(get);
179     assertNull(
180       "Got an unexpected number of rows - no data should be returned with the NoDataFromScan coprocessor. Found: "
181           + r, r.listCells());
182   }
183 
184   @Test
185   public void testRegionObserverFlushTimeStacking() throws Exception {
186     byte[] ROW = Bytes.toBytes("testRow");
187     byte[] TABLE = Bytes.toBytes(getClass().getName());
188     byte[] A = Bytes.toBytes("A");
189     byte[][] FAMILIES = new byte[][] { A };
190 
191     Configuration conf = HBaseConfiguration.create();
192     HRegion region = initHRegion(TABLE, getClass().getName(), conf, FAMILIES);
193     RegionCoprocessorHost h = region.getCoprocessorHost();
194     h.load(NoDataFromFlush.class, Coprocessor.PRIORITY_HIGHEST, conf);
195     h.load(EmptyRegionObsever.class, Coprocessor.PRIORITY_USER, conf);
196 
197     // put a row and flush it to disk
198     Put put = new Put(ROW);
199     put.add(A, A, A);
200     region.put(put);
201     region.flushcache();
202     Get get = new Get(ROW);
203     Result r = region.get(get);
204     assertNull(
205       "Got an unexpected number of rows - no data should be returned with the NoDataFromScan coprocessor. Found: "
206           + r, r.listCells());
207   }
208 
209   /*
210    * Custom HRegion which uses CountDownLatch to signal the completion of compaction
211    */
212   public static class CompactionCompletionNotifyingRegion extends HRegion {
213     private static volatile CountDownLatch compactionStateChangeLatch = null;
214     
215     @SuppressWarnings("deprecation")
216     public CompactionCompletionNotifyingRegion(Path tableDir, HLog log,
217         FileSystem fs, Configuration confParam, HRegionInfo info,
218         HTableDescriptor htd, RegionServerServices rsServices) {
219       super(tableDir, log, fs, confParam, info, htd, rsServices);
220     }
221     
222     public CountDownLatch getCompactionStateChangeLatch() {
223       if (compactionStateChangeLatch == null) compactionStateChangeLatch = new CountDownLatch(1);
224       return compactionStateChangeLatch;
225     }
226     @Override
227     public boolean compact(CompactionContext compaction, Store store) throws IOException {
228       boolean ret = super.compact(compaction, store);
229       if (ret) compactionStateChangeLatch.countDown();
230       return ret;
231     }    
232   }
233   
234   /**
235    * Unfortunately, the easiest way to test this is to spin up a mini-cluster since we want to do
236    * the usual compaction mechanism on the region, rather than going through the backdoor to the
237    * region
238    */
239   @Test
240   public void testRegionObserverCompactionTimeStacking() throws Exception {
241     // setup a mini cluster so we can do a real compaction on a region
242     Configuration conf = UTIL.getConfiguration();
243     conf.setClass(HConstants.REGION_IMPL, CompactionCompletionNotifyingRegion.class, HRegion.class);
244     conf.setInt("hbase.hstore.compaction.min", 2);
245     UTIL.startMiniCluster();
246     String tableName = "testRegionObserverCompactionTimeStacking";
247     byte[] ROW = Bytes.toBytes("testRow");
248     byte[] A = Bytes.toBytes("A");
249     HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
250     desc.addFamily(new HColumnDescriptor(A));
251     desc.addCoprocessor(EmptyRegionObsever.class.getName(), null, Coprocessor.PRIORITY_USER, null);
252     desc.addCoprocessor(NoDataFromCompaction.class.getName(), null, Coprocessor.PRIORITY_HIGHEST,
253       null);
254 
255     HBaseAdmin admin = UTIL.getHBaseAdmin();
256     admin.createTable(desc);
257 
258     HTable table = new HTable(conf, desc.getTableName());
259 
260     // put a row and flush it to disk
261     Put put = new Put(ROW);
262     put.add(A, A, A);
263     table.put(put);
264     table.flushCommits();
265 
266     HRegionServer rs = UTIL.getRSForFirstRegionInTable(desc.getTableName());
267     List<HRegion> regions = rs.getOnlineRegions(desc.getTableName());
268     assertEquals("More than 1 region serving test table with 1 row", 1, regions.size());
269     HRegion region = regions.get(0);
270     admin.flush(region.getRegionName());
271     CountDownLatch latch = ((CompactionCompletionNotifyingRegion)region)
272         .getCompactionStateChangeLatch();
273     
274     // put another row and flush that too
275     put = new Put(Bytes.toBytes("anotherrow"));
276     put.add(A, A, A);
277     table.put(put);
278     table.flushCommits();
279     admin.flush(region.getRegionName());
280 
281     // run a compaction, which normally would should get rid of the data
282     // wait for the compaction checker to complete
283     latch.await();
284     // check both rows to ensure that they aren't there
285     Get get = new Get(ROW);
286     Result r = table.get(get);
287     assertNull(
288       "Got an unexpected number of rows - no data should be returned with the NoDataFromScan coprocessor. Found: "
289           + r, r.listCells());
290 
291     get = new Get(Bytes.toBytes("anotherrow"));
292     r = table.get(get);
293     assertNull(
294       "Got an unexpected number of rows - no data should be returned with the NoDataFromScan coprocessor Found: "
295           + r, r.listCells());
296 
297     table.close();
298     UTIL.shutdownMiniCluster();
299   }
300 }