View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import java.util.ArrayList;
22  import java.util.List;
23  
24  import org.apache.hadoop.hbase.*;
25  import org.apache.hadoop.hbase.client.Delete;
26  import org.apache.hadoop.hbase.client.Get;
27  import org.apache.hadoop.hbase.client.Put;
28  import org.apache.hadoop.hbase.client.Result;
29  import org.apache.hadoop.hbase.filter.TimestampsFilter;
30  import org.apache.hadoop.hbase.util.Bytes;
31  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
32  import org.junit.experimental.categories.Category;
33  
34  /**
35   * Test Minimum Versions feature (HBASE-4071).
36   */
37  @Category(SmallTests.class)
38  public class TestMinVersions extends HBaseTestCase {
39    private final byte[] T0 = Bytes.toBytes("0");
40    private final byte[] T1 = Bytes.toBytes("1");
41    private final byte[] T2 = Bytes.toBytes("2");
42    private final byte[] T3 = Bytes.toBytes("3");
43    private final byte[] T4 = Bytes.toBytes("4");
44    private final byte[] T5 = Bytes.toBytes("5");
45  
46    private final byte[] c0 = COLUMNS[0];
47  
48    /**
49     * Verify behavior of getClosestBefore(...)
50     */
51    public void testGetClosestBefore() throws Exception {
52      HTableDescriptor htd = createTableDescriptor(getName(), 1, 1000, 1, false);
53      HRegion region = createNewHRegion(htd, null, null);
54      try {
55  
56        // 2s in the past
57        long ts = EnvironmentEdgeManager.currentTimeMillis() - 2000;
58  
59        Put p = new Put(T1, ts);
60        p.add(c0, c0, T1);
61        region.put(p);
62  
63        p = new Put(T1, ts+1);
64        p.add(c0, c0, T4);
65        region.put(p);
66  
67        p = new Put(T3, ts);
68        p.add(c0, c0, T3);
69        region.put(p);
70  
71        // now make sure that getClosestBefore(...) get can
72        // rows that would be expired without minVersion.
73        // also make sure it gets the latest version
74        Result r = region.getClosestRowBefore(T1, c0);
75        checkResult(r, c0, T4);
76  
77        r = region.getClosestRowBefore(T2, c0);
78        checkResult(r, c0, T4);
79  
80        // now flush/compact
81        region.flushcache();
82        region.compactStores(true);
83  
84        r = region.getClosestRowBefore(T1, c0);
85        checkResult(r, c0, T4);
86  
87        r = region.getClosestRowBefore(T2, c0);
88        checkResult(r, c0, T4);
89      } finally {
90        HRegion.closeHRegion(region);
91      }
92    }
93  
94    /**
95     * Test mixed memstore and storefile scanning
96     * with minimum versions.
97     */
98    public void testStoreMemStore() throws Exception {
99      // keep 3 versions minimum
100     HTableDescriptor htd = createTableDescriptor(getName(), 3, 1000, 1, false);
101     HRegion region = createNewHRegion(htd, null, null);
102     // 2s in the past
103     long ts = EnvironmentEdgeManager.currentTimeMillis() - 2000;
104 
105     try {
106       Put p = new Put(T1, ts-1);
107       p.add(c0, c0, T2);
108       region.put(p);
109 
110       p = new Put(T1, ts-3);
111       p.add(c0, c0, T0);
112       region.put(p);
113 
114       // now flush/compact
115       region.flushcache();
116       region.compactStores(true);
117 
118       p = new Put(T1, ts);
119       p.add(c0, c0, T3);
120       region.put(p);
121 
122       p = new Put(T1, ts-2);
123       p.add(c0, c0, T1);
124       region.put(p);
125 
126       p = new Put(T1, ts-3);
127       p.add(c0, c0, T0);
128       region.put(p);
129 
130       // newest version in the memstore
131       // the 2nd oldest in the store file
132       // and the 3rd, 4th oldest also in the memstore
133 
134       Get g = new Get(T1);
135       g.setMaxVersions();
136       Result r = region.get(g); // this'll use ScanWildcardColumnTracker
137       checkResult(r, c0, T3,T2,T1);
138 
139       g = new Get(T1);
140       g.setMaxVersions();
141       g.addColumn(c0, c0);
142       r = region.get(g);  // this'll use ExplicitColumnTracker
143       checkResult(r, c0, T3,T2,T1);
144     } finally {
145       HRegion.closeHRegion(region);
146     }
147   }
148 
149   /**
150    * Make sure the Deletes behave as expected with minimum versions
151    */
152   public void testDelete() throws Exception {
153     HTableDescriptor htd = createTableDescriptor(getName(), 3, 1000, 1, false);
154     HRegion region = createNewHRegion(htd, null, null);
155 
156     // 2s in the past
157     long ts = EnvironmentEdgeManager.currentTimeMillis() - 2000;
158 
159     try {
160       Put p = new Put(T1, ts-2);
161       p.add(c0, c0, T1);
162       region.put(p);
163 
164       p = new Put(T1, ts-1);
165       p.add(c0, c0, T2);
166       region.put(p);
167 
168       p = new Put(T1, ts);
169       p.add(c0, c0, T3);
170       region.put(p);
171 
172       Delete d = new Delete(T1, ts-1);
173       region.delete(d);
174 
175       Get g = new Get(T1);
176       g.setMaxVersions();
177       Result r = region.get(g);  // this'll use ScanWildcardColumnTracker
178       checkResult(r, c0, T3);
179 
180       g = new Get(T1);
181       g.setMaxVersions();
182       g.addColumn(c0, c0);
183       r = region.get(g);  // this'll use ExplicitColumnTracker
184       checkResult(r, c0, T3);
185 
186       // now flush/compact
187       region.flushcache();
188       region.compactStores(true);
189 
190       // try again
191       g = new Get(T1);
192       g.setMaxVersions();
193       r = region.get(g);  // this'll use ScanWildcardColumnTracker
194       checkResult(r, c0, T3);
195 
196       g = new Get(T1);
197       g.setMaxVersions();
198       g.addColumn(c0, c0);
199       r = region.get(g);  // this'll use ExplicitColumnTracker
200       checkResult(r, c0, T3);
201     } finally {
202       HRegion.closeHRegion(region);
203     }
204   }
205 
206   /**
207    * Make sure the memstor behaves correctly with minimum versions
208    */
209   public void testMemStore() throws Exception {
210     HTableDescriptor htd = createTableDescriptor(getName(), 2, 1000, 1, false);
211     HRegion region = createNewHRegion(htd, null, null);
212 
213     // 2s in the past
214     long ts = EnvironmentEdgeManager.currentTimeMillis() - 2000;
215 
216     try {
217       // 2nd version
218       Put p = new Put(T1, ts-2);
219       p.add(c0, c0, T2);
220       region.put(p);
221 
222       // 3rd version
223       p = new Put(T1, ts-1);
224       p.add(c0, c0, T3);
225       region.put(p);
226 
227       // 4th version
228       p = new Put(T1, ts);
229       p.add(c0, c0, T4);
230       region.put(p);
231 
232       // now flush/compact
233       region.flushcache();
234       region.compactStores(true);
235 
236       // now put the first version (backdated)
237       p = new Put(T1, ts-3);
238       p.add(c0, c0, T1);
239       region.put(p);
240 
241       // now the latest change is in the memstore,
242       // but it is not the latest version
243 
244       Result r = region.get(new Get(T1));
245       checkResult(r, c0, T4);
246 
247       Get g = new Get(T1);
248       g.setMaxVersions();
249       r = region.get(g); // this'll use ScanWildcardColumnTracker
250       checkResult(r, c0, T4,T3);
251 
252       g = new Get(T1);
253       g.setMaxVersions();
254       g.addColumn(c0, c0);
255       r = region.get(g);  // this'll use ExplicitColumnTracker
256       checkResult(r, c0, T4,T3);
257 
258       p = new Put(T1, ts+1);
259       p.add(c0, c0, T5);
260       region.put(p);
261 
262       // now the latest version is in the memstore
263 
264       g = new Get(T1);
265       g.setMaxVersions();
266       r = region.get(g);  // this'll use ScanWildcardColumnTracker
267       checkResult(r, c0, T5,T4);
268 
269       g = new Get(T1);
270       g.setMaxVersions();
271       g.addColumn(c0, c0);
272       r = region.get(g);  // this'll use ExplicitColumnTracker
273       checkResult(r, c0, T5,T4);
274     } finally {
275       HRegion.closeHRegion(region);
276     }
277   }
278 
279   /**
280    * Verify basic minimum versions functionality
281    */
282   public void testBaseCase() throws Exception {
283     // 1 version minimum, 1000 versions maximum, ttl = 1s
284     HTableDescriptor htd = createTableDescriptor(getName(), 2, 1000, 1, false);
285     HRegion region = createNewHRegion(htd, null, null);
286     try {
287 
288       // 2s in the past
289       long ts = EnvironmentEdgeManager.currentTimeMillis() - 2000;
290 
291        // 1st version
292       Put p = new Put(T1, ts-3);
293       p.add(c0, c0, T1);
294       region.put(p);
295 
296       // 2nd version
297       p = new Put(T1, ts-2);
298       p.add(c0, c0, T2);
299       region.put(p);
300 
301       // 3rd version
302       p = new Put(T1, ts-1);
303       p.add(c0, c0, T3);
304       region.put(p);
305 
306       // 4th version
307       p = new Put(T1, ts);
308       p.add(c0, c0, T4);
309       region.put(p);
310 
311       Result r = region.get(new Get(T1));
312       checkResult(r, c0, T4);
313 
314       Get g = new Get(T1);
315       g.setTimeRange(0L, ts+1);
316       r = region.get(g);
317       checkResult(r, c0, T4);
318 
319   // oldest version still exists
320       g.setTimeRange(0L, ts-2);
321       r = region.get(g);
322       checkResult(r, c0, T1);
323 
324       // gets see only available versions
325       // even before compactions
326       g = new Get(T1);
327       g.setMaxVersions();
328       r = region.get(g); // this'll use ScanWildcardColumnTracker
329       checkResult(r, c0, T4,T3);
330 
331       g = new Get(T1);
332       g.setMaxVersions();
333       g.addColumn(c0, c0);
334       r = region.get(g);  // this'll use ExplicitColumnTracker
335       checkResult(r, c0, T4,T3);
336 
337       // now flush
338       region.flushcache();
339 
340       // with HBASE-4241 a flush will eliminate the expired rows
341       g = new Get(T1);
342       g.setTimeRange(0L, ts-2);
343       r = region.get(g);
344       assertTrue(r.isEmpty());
345 
346       // major compaction
347       region.compactStores(true);
348 
349       // after compaction the 4th version is still available
350       g = new Get(T1);
351       g.setTimeRange(0L, ts+1);
352       r = region.get(g);
353       checkResult(r, c0, T4);
354 
355       // so is the 3rd
356       g.setTimeRange(0L, ts);
357       r = region.get(g);
358       checkResult(r, c0, T3);
359 
360       // but the 2nd and earlier versions are gone
361       g.setTimeRange(0L, ts-1);
362       r = region.get(g);
363       assertTrue(r.isEmpty());
364     } finally {
365       HRegion.closeHRegion(region);
366     }
367   }
368 
369   /**
370    * Verify that basic filters still behave correctly with
371    * minimum versions enabled.
372    */
373   public void testFilters() throws Exception {
374     HTableDescriptor htd = createTableDescriptor(getName(), 2, 1000, 1, false);
375     HRegion region = createNewHRegion(htd, null, null);
376     final byte [] c1 = COLUMNS[1];
377 
378     // 2s in the past
379     long ts = EnvironmentEdgeManager.currentTimeMillis() - 2000;
380     try {
381 
382       Put p = new Put(T1, ts-3);
383       p.add(c0, c0, T0);
384       p.add(c1, c1, T0);
385       region.put(p);
386 
387       p = new Put(T1, ts-2);
388       p.add(c0, c0, T1);
389       p.add(c1, c1, T1);
390       region.put(p);
391 
392       p = new Put(T1, ts-1);
393       p.add(c0, c0, T2);
394       p.add(c1, c1, T2);
395       region.put(p);
396 
397       p = new Put(T1, ts);
398       p.add(c0, c0, T3);
399       p.add(c1, c1, T3);
400       region.put(p);
401 
402       List<Long> tss = new ArrayList<Long>();
403       tss.add(ts-1);
404       tss.add(ts-2);
405 
406       Get g = new Get(T1);
407       g.addColumn(c1,c1);
408       g.setFilter(new TimestampsFilter(tss));
409       g.setMaxVersions();
410       Result r = region.get(g);
411       checkResult(r, c1, T2,T1);
412 
413       g = new Get(T1);
414       g.addColumn(c0,c0);
415       g.setFilter(new TimestampsFilter(tss));
416       g.setMaxVersions();
417       r = region.get(g);
418       checkResult(r, c0, T2,T1);
419 
420       // now flush/compact
421       region.flushcache();
422       region.compactStores(true);
423 
424       g = new Get(T1);
425       g.addColumn(c1,c1);
426       g.setFilter(new TimestampsFilter(tss));
427       g.setMaxVersions();
428       r = region.get(g);
429       checkResult(r, c1, T2);
430 
431       g = new Get(T1);
432       g.addColumn(c0,c0);
433       g.setFilter(new TimestampsFilter(tss));
434       g.setMaxVersions();
435       r = region.get(g);
436       checkResult(r, c0, T2);
437     } finally {
438       HRegion.closeHRegion(region);
439     }
440   }
441 
442   private void checkResult(Result r, byte[] col, byte[] ... vals) {
443     assertEquals(r.size(), vals.length);
444     List<KeyValue> kvs = r.getColumn(col, col);
445     assertEquals(kvs.size(), vals.length);
446     for (int i=0;i<vals.length;i++) {
447       assertEquals(kvs.get(i).getValue(), vals[i]);
448     }
449   }
450 
451 }
452