1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.coprocessor;
21
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertNull;
24
25 import java.io.IOException;
26 import java.util.Collections;
27 import java.util.List;
28 import java.util.NavigableSet;
29 import java.util.concurrent.CountDownLatch;
30
31 import org.apache.hadoop.conf.Configuration;
32 import org.apache.hadoop.fs.FileSystem;
33 import org.apache.hadoop.fs.Path;
34 import org.apache.hadoop.hbase.Cell;
35 import org.apache.hadoop.hbase.Coprocessor;
36 import org.apache.hadoop.hbase.HBaseConfiguration;
37 import org.apache.hadoop.hbase.HBaseTestingUtility;
38 import org.apache.hadoop.hbase.HColumnDescriptor;
39 import org.apache.hadoop.hbase.HConstants;
40 import org.apache.hadoop.hbase.HRegionInfo;
41 import org.apache.hadoop.hbase.HTableDescriptor;
42 import org.apache.hadoop.hbase.testclassification.MediumTests;
43 import org.apache.hadoop.hbase.TableName;
44 import org.apache.hadoop.hbase.client.Admin;
45 import org.apache.hadoop.hbase.client.Get;
46 import org.apache.hadoop.hbase.client.HTable;
47 import org.apache.hadoop.hbase.client.IsolationLevel;
48 import org.apache.hadoop.hbase.client.Put;
49 import org.apache.hadoop.hbase.client.Result;
50 import org.apache.hadoop.hbase.client.Scan;
51 import org.apache.hadoop.hbase.client.Table;
52 import org.apache.hadoop.hbase.filter.FilterBase;
53 import org.apache.hadoop.hbase.regionserver.HRegion;
54 import org.apache.hadoop.hbase.regionserver.HRegionServer;
55 import org.apache.hadoop.hbase.regionserver.HStore;
56 import org.apache.hadoop.hbase.regionserver.InternalScanner;
57 import org.apache.hadoop.hbase.regionserver.KeyValueScanner;
58 import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
59 import org.apache.hadoop.hbase.regionserver.RegionServerServices;
60 import org.apache.hadoop.hbase.regionserver.ScanType;
61 import org.apache.hadoop.hbase.regionserver.Store;
62 import org.apache.hadoop.hbase.regionserver.StoreScanner;
63 import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
64 import org.apache.hadoop.hbase.wal.WAL;
65 import org.apache.hadoop.hbase.util.Bytes;
66 import org.junit.Test;
67 import org.junit.experimental.categories.Category;
68
69 @Category(MediumTests.class)
70 public class TestRegionObserverScannerOpenHook {
71 private static HBaseTestingUtility UTIL = new HBaseTestingUtility();
72 static final Path DIR = UTIL.getDataTestDir();
73
74 public static class NoDataFilter extends FilterBase {
75
76 @Override
77 public ReturnCode filterKeyValue(Cell ignored) throws IOException {
78 return ReturnCode.SKIP;
79 }
80
81 @Override
82 public boolean filterAllRemaining() throws IOException {
83 return true;
84 }
85
86 @Override
87 public boolean filterRow() throws IOException {
88 return true;
89 }
90 }
91
92
93
94
95
96 public static class EmptyRegionObsever extends BaseRegionObserver {
97 }
98
99
100
101
102 public static class NoDataFromScan extends BaseRegionObserver {
103 @Override
104 public KeyValueScanner preStoreScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c,
105 Store store, Scan scan, NavigableSet<byte[]> targetCols, KeyValueScanner s)
106 throws IOException {
107 scan.setFilter(new NoDataFilter());
108 return new StoreScanner(store, store.getScanInfo(), scan, targetCols,
109 ((HStore)store).getHRegion().getReadpoint(IsolationLevel.READ_COMMITTED));
110 }
111 }
112
113
114
115
116 public static class NoDataFromFlush extends BaseRegionObserver {
117 @Override
118 public InternalScanner preFlushScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c,
119 Store store, KeyValueScanner memstoreScanner, InternalScanner s) throws IOException {
120 Scan scan = new Scan();
121 scan.setFilter(new NoDataFilter());
122 return new StoreScanner(store, store.getScanInfo(), scan,
123 Collections.singletonList(memstoreScanner), ScanType.COMPACT_RETAIN_DELETES,
124 store.getSmallestReadPoint(), HConstants.OLDEST_TIMESTAMP);
125 }
126 }
127
128
129
130
131
132 public static class NoDataFromCompaction extends BaseRegionObserver {
133 @Override
134 public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c,
135 Store store, List<? extends KeyValueScanner> scanners, ScanType scanType,
136 long earliestPutTs, InternalScanner s) throws IOException {
137 Scan scan = new Scan();
138 scan.setFilter(new NoDataFilter());
139 return new StoreScanner(store, store.getScanInfo(), scan, scanners,
140 ScanType.COMPACT_RETAIN_DELETES, store.getSmallestReadPoint(),
141 HConstants.OLDEST_TIMESTAMP);
142 }
143 }
144
145 HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
146 byte[]... families) throws IOException {
147 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
148 for (byte[] family : families) {
149 htd.addFamily(new HColumnDescriptor(family));
150 }
151 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
152 Path path = new Path(DIR + callingMethod);
153 HRegion r = HRegion.createHRegion(info, path, conf, htd);
154
155
156
157
158 RegionCoprocessorHost host = new RegionCoprocessorHost(r, null, conf);
159 r.setCoprocessorHost(host);
160 return r;
161 }
162
163 @Test
164 public void testRegionObserverScanTimeStacking() throws Exception {
165 byte[] ROW = Bytes.toBytes("testRow");
166 byte[] TABLE = Bytes.toBytes(getClass().getName());
167 byte[] A = Bytes.toBytes("A");
168 byte[][] FAMILIES = new byte[][] { A };
169
170 Configuration conf = HBaseConfiguration.create();
171 HRegion region = initHRegion(TABLE, getClass().getName(), conf, FAMILIES);
172 RegionCoprocessorHost h = region.getCoprocessorHost();
173 h.load(NoDataFromScan.class, Coprocessor.PRIORITY_HIGHEST, conf);
174 h.load(EmptyRegionObsever.class, Coprocessor.PRIORITY_USER, conf);
175
176 Put put = new Put(ROW);
177 put.add(A, A, A);
178 region.put(put);
179
180 Get get = new Get(ROW);
181 Result r = region.get(get);
182 assertNull(
183 "Got an unexpected number of rows - no data should be returned with the NoDataFromScan coprocessor. Found: "
184 + r, r.listCells());
185 }
186
187 @Test
188 public void testRegionObserverFlushTimeStacking() throws Exception {
189 byte[] ROW = Bytes.toBytes("testRow");
190 byte[] TABLE = Bytes.toBytes(getClass().getName());
191 byte[] A = Bytes.toBytes("A");
192 byte[][] FAMILIES = new byte[][] { A };
193
194 Configuration conf = HBaseConfiguration.create();
195 HRegion region = initHRegion(TABLE, getClass().getName(), conf, FAMILIES);
196 RegionCoprocessorHost h = region.getCoprocessorHost();
197 h.load(NoDataFromFlush.class, Coprocessor.PRIORITY_HIGHEST, conf);
198 h.load(EmptyRegionObsever.class, Coprocessor.PRIORITY_USER, conf);
199
200
201 Put put = new Put(ROW);
202 put.add(A, A, A);
203 region.put(put);
204 region.flushcache();
205 Get get = new Get(ROW);
206 Result r = region.get(get);
207 assertNull(
208 "Got an unexpected number of rows - no data should be returned with the NoDataFromScan coprocessor. Found: "
209 + r, r.listCells());
210 }
211
212
213
214
215 public static class CompactionCompletionNotifyingRegion extends HRegion {
216 private static volatile CountDownLatch compactionStateChangeLatch = null;
217
218 @SuppressWarnings("deprecation")
219 public CompactionCompletionNotifyingRegion(Path tableDir, WAL log,
220 FileSystem fs, Configuration confParam, HRegionInfo info,
221 HTableDescriptor htd, RegionServerServices rsServices) {
222 super(tableDir, log, fs, confParam, info, htd, rsServices);
223 }
224
225 public CountDownLatch getCompactionStateChangeLatch() {
226 if (compactionStateChangeLatch == null) compactionStateChangeLatch = new CountDownLatch(1);
227 return compactionStateChangeLatch;
228 }
229 @Override
230 public boolean compact(CompactionContext compaction, Store store) throws IOException {
231 boolean ret = super.compact(compaction, store);
232 if (ret) compactionStateChangeLatch.countDown();
233 return ret;
234 }
235 }
236
237
238
239
240
241
242 @Test
243 public void testRegionObserverCompactionTimeStacking() throws Exception {
244
245 Configuration conf = UTIL.getConfiguration();
246 conf.setClass(HConstants.REGION_IMPL, CompactionCompletionNotifyingRegion.class, HRegion.class);
247 conf.setInt("hbase.hstore.compaction.min", 2);
248 UTIL.startMiniCluster();
249 String tableName = "testRegionObserverCompactionTimeStacking";
250 byte[] ROW = Bytes.toBytes("testRow");
251 byte[] A = Bytes.toBytes("A");
252 HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
253 desc.addFamily(new HColumnDescriptor(A));
254 desc.addCoprocessor(EmptyRegionObsever.class.getName(), null, Coprocessor.PRIORITY_USER, null);
255 desc.addCoprocessor(NoDataFromCompaction.class.getName(), null, Coprocessor.PRIORITY_HIGHEST,
256 null);
257
258 Admin admin = UTIL.getHBaseAdmin();
259 admin.createTable(desc);
260
261 Table table = new HTable(conf, desc.getTableName());
262
263
264 Put put = new Put(ROW);
265 put.add(A, A, A);
266 table.put(put);
267
268 HRegionServer rs = UTIL.getRSForFirstRegionInTable(desc.getTableName());
269 List<HRegion> regions = rs.getOnlineRegions(desc.getTableName());
270 assertEquals("More than 1 region serving test table with 1 row", 1, regions.size());
271 HRegion region = regions.get(0);
272 admin.flushRegion(region.getRegionName());
273 CountDownLatch latch = ((CompactionCompletionNotifyingRegion)region)
274 .getCompactionStateChangeLatch();
275
276
277 put = new Put(Bytes.toBytes("anotherrow"));
278 put.add(A, A, A);
279 table.put(put);
280 admin.flushRegion(region.getRegionName());
281
282
283
284 latch.await();
285
286 Get get = new Get(ROW);
287 Result r = table.get(get);
288 assertNull(
289 "Got an unexpected number of rows - no data should be returned with the NoDataFromScan coprocessor. Found: "
290 + r, r.listCells());
291
292 get = new Get(Bytes.toBytes("anotherrow"));
293 r = table.get(get);
294 assertNull(
295 "Got an unexpected number of rows - no data should be returned with the NoDataFromScan coprocessor Found: "
296 + r, r.listCells());
297
298 table.close();
299 UTIL.shutdownMiniCluster();
300 }
301 }