1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase;
20  
21  import java.io.IOException;
22  
23  import org.apache.hadoop.hbase.client.Delete;
24  import org.apache.hadoop.hbase.client.Get;
25  import org.apache.hadoop.hbase.client.Put;
26  import org.apache.hadoop.hbase.client.Result;
27  import org.apache.hadoop.hbase.util.Bytes;
28  
29  /**
30   * Tests user specifiable time stamps putting, getting and scanning.  Also
31   * tests same in presence of deletes.  Test cores are written so can be
32   * run against an HRegion and against an HTable: i.e. both local and remote.
33   */
34  public class TimestampTestBase extends HBaseTestCase {
35    private static final long T0 = 10L;
36    private static final long T1 = 100L;
37    private static final long T2 = 200L;
38  
39    private static final byte [] FAMILY_NAME = Bytes.toBytes("colfamily1");
40    private static final byte [] QUALIFIER_NAME = Bytes.toBytes("contents");
41  
42    private static final byte [] ROW = Bytes.toBytes("row");
43  
44      /*
45     * Run test that delete works according to description in <a
46     * href="https://issues.apache.org/jira/browse/HADOOP-1784">hadoop-1784</a>.
47     * @param incommon
48     * @param flusher
49     * @throws IOException
50     */
51    public static void doTestDelete(final Incommon incommon, FlushCache flusher)
52    throws IOException {
53      // Add values at various timestamps (Values are timestampes as bytes).
54      put(incommon, T0);
55      put(incommon, T1);
56      put(incommon, T2);
57      put(incommon);
58      // Verify that returned versions match passed timestamps.
59      assertVersions(incommon, new long [] {HConstants.LATEST_TIMESTAMP, T2, T1});
60  
61      // If I delete w/o specifying a timestamp, this means I'm deleting the
62      // latest.
63      delete(incommon);
64      // Verify that I get back T2 through T1 -- that the latest version has
65      // been deleted.
66      assertVersions(incommon, new long [] {T2, T1, T0});
67  
68      // Flush everything out to disk and then retry
69      flusher.flushcache();
70      assertVersions(incommon, new long [] {T2, T1, T0});
71  
72      // Now add, back a latest so I can test remove other than the latest.
73      put(incommon);
74      assertVersions(incommon, new long [] {HConstants.LATEST_TIMESTAMP, T2, T1});
75      delete(incommon, T2);
76      assertVersions(incommon, new long [] {HConstants.LATEST_TIMESTAMP, T1, T0});
77      // Flush everything out to disk and then retry
78      flusher.flushcache();
79      assertVersions(incommon, new long [] {HConstants.LATEST_TIMESTAMP, T1, T0});
80  
81      // Now try deleting all from T2 back inclusive (We first need to add T2
82      // back into the mix and to make things a little interesting, delete and
83      // then readd T1.
84      put(incommon, T2);
85      delete(incommon, T1);
86      put(incommon, T1);
87  
88      Delete delete = new Delete(ROW);
89      delete.deleteColumns(FAMILY_NAME, QUALIFIER_NAME, T2);
90      incommon.delete(delete, null, true);
91  
92      // Should only be current value in set.  Assert this is so
93      assertOnlyLatest(incommon, HConstants.LATEST_TIMESTAMP);
94  
95      // Flush everything out to disk and then redo above tests
96      flusher.flushcache();
97      assertOnlyLatest(incommon, HConstants.LATEST_TIMESTAMP);
98    }
99  
100   private static void assertOnlyLatest(final Incommon incommon,
101     final long currentTime)
102   throws IOException {
103     Get get = null;
104     get = new Get(ROW);
105     get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
106     get.setMaxVersions(3);
107     Result result = incommon.get(get);
108     assertEquals(1, result.size());
109     long time = Bytes.toLong(result.sorted()[0].getValue());
110     assertEquals(time, currentTime);
111   }
112 
113   /*
114    * Assert that returned versions match passed in timestamps and that results
115    * are returned in the right order.  Assert that values when converted to
116    * longs match the corresponding passed timestamp.
117    * @param r
118    * @param tss
119    * @throws IOException
120    */
121   public static void assertVersions(final Incommon incommon, final long [] tss)
122   throws IOException {
123     // Assert that 'latest' is what we expect.
124     Get get = null;
125     get = new Get(ROW);
126     get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
127     Result r = incommon.get(get);
128     byte [] bytes = r.getValue(FAMILY_NAME, QUALIFIER_NAME);
129     long t = Bytes.toLong(bytes);
130     assertEquals(tss[0], t);
131 
132     // Now assert that if we ask for multiple versions, that they come out in
133     // order.
134     get = new Get(ROW);
135     get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
136     get.setMaxVersions(tss.length);
137     Result result = incommon.get(get);
138     KeyValue [] kvs = result.sorted();
139     assertEquals(kvs.length, tss.length);
140     for(int i=0;i<kvs.length;i++) {
141       t = Bytes.toLong(kvs[i].getValue());
142       assertEquals(tss[i], t);
143     }
144 
145     // Determine highest stamp to set as next max stamp
146     long maxStamp = kvs[0].getTimestamp();
147 
148     // Specify a timestamp get multiple versions.
149     get = new Get(ROW);
150     get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
151     get.setTimeRange(0, maxStamp);
152     get.setMaxVersions(kvs.length - 1);
153     result = incommon.get(get);
154     kvs = result.sorted();
155     assertEquals(kvs.length, tss.length - 1);
156     for(int i=1;i<kvs.length;i++) {
157       t = Bytes.toLong(kvs[i-1].getValue());
158       assertEquals(tss[i], t);
159     }
160 
161     // Test scanner returns expected version
162     assertScanContentTimestamp(incommon, tss[0]);
163   }
164 
165   /*
166    * Run test scanning different timestamps.
167    * @param incommon
168    * @param flusher
169    * @throws IOException
170    */
171   public static void doTestTimestampScanning(final Incommon incommon,
172     final FlushCache flusher)
173   throws IOException {
174     // Add a couple of values for three different timestamps.
175     put(incommon, T0);
176     put(incommon, T1);
177     put(incommon, HConstants.LATEST_TIMESTAMP);
178     // Get count of latest items.
179     int count = assertScanContentTimestamp(incommon,
180       HConstants.LATEST_TIMESTAMP);
181     // Assert I get same count when I scan at each timestamp.
182     assertEquals(count, assertScanContentTimestamp(incommon, T0));
183     assertEquals(count, assertScanContentTimestamp(incommon, T1));
184     // Flush everything out to disk and then retry
185     flusher.flushcache();
186     assertEquals(count, assertScanContentTimestamp(incommon, T0));
187     assertEquals(count, assertScanContentTimestamp(incommon, T1));
188   }
189 
190   /*
191    * Assert that the scan returns only values < timestamp.
192    * @param r
193    * @param ts
194    * @return Count of items scanned.
195    * @throws IOException
196    */
197   public static int assertScanContentTimestamp(final Incommon in, final long ts)
198   throws IOException {
199     ScannerIncommon scanner =
200       in.getScanner(COLUMNS[0], null, HConstants.EMPTY_START_ROW, ts);
201     int count = 0;
202     try {
203       // TODO FIX
204 //      HStoreKey key = new HStoreKey();
205 //      TreeMap<byte [], Cell>value =
206 //        new TreeMap<byte [], Cell>(Bytes.BYTES_COMPARATOR);
207 //      while (scanner.next(key, value)) {
208 //        assertTrue(key.getTimestamp() <= ts);
209 //        // Content matches the key or HConstants.LATEST_TIMESTAMP.
210 //        // (Key does not match content if we 'put' with LATEST_TIMESTAMP).
211 //        long l = Bytes.toLong(value.get(COLUMN).getValue());
212 //        assertTrue(key.getTimestamp() == l ||
213 //          HConstants.LATEST_TIMESTAMP == l);
214 //        count++;
215 //        value.clear();
216 //      }
217     } finally {
218       scanner.close();
219     }
220     return count;
221   }
222 
223   public static void put(final Incommon loader, final long ts)
224   throws IOException {
225     put(loader, Bytes.toBytes(ts), ts);
226   }
227 
228   public static void put(final Incommon loader)
229   throws IOException {
230     long ts = HConstants.LATEST_TIMESTAMP;
231     put(loader, Bytes.toBytes(ts), ts);
232   }
233 
234   /*
235    * Put values.
236    * @param loader
237    * @param bytes
238    * @param ts
239    * @throws IOException
240    */
241   public static void put(final Incommon loader, final byte [] bytes,
242     final long ts)
243   throws IOException {
244     Put put = new Put(ROW, ts, null);
245     put.add(FAMILY_NAME, QUALIFIER_NAME, bytes);
246     loader.put(put);
247   }
248 
249   public static void delete(final Incommon loader) throws IOException {
250     delete(loader, null);
251   }
252 
253   public static void delete(final Incommon loader, final byte [] column)
254   throws IOException {
255     delete(loader, column, HConstants.LATEST_TIMESTAMP);
256   }
257 
258   public static void delete(final Incommon loader, final long ts)
259   throws IOException {
260     delete(loader, null, ts);
261   }
262 
263   public static void delete(final Incommon loader, final byte [] column,
264       final long ts)
265   throws IOException {
266     Delete delete = ts == HConstants.LATEST_TIMESTAMP?
267       new Delete(ROW): new Delete(ROW, ts, null);
268     delete.deleteColumn(FAMILY_NAME, QUALIFIER_NAME, ts);
269     loader.delete(delete, null, true);
270   }
271 
272   public static Result get(final Incommon loader) throws IOException {
273     return loader.get(new Get(ROW));
274   }
275 }