1   /**
2    * Copyright 2009 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.regionserver;
21  
22  import java.io.IOException;
23  import java.util.ArrayList;
24  import java.util.List;
25  
26  import org.apache.commons.logging.Log;
27  import org.apache.commons.logging.LogFactory;
28  import org.apache.hadoop.fs.FileSystem;
29  import org.apache.hadoop.fs.Path;
30  import org.apache.hadoop.hbase.HBaseTestCase;
31  import org.apache.hadoop.hbase.HConstants;
32  import org.apache.hadoop.hbase.KeyValue;
33  import org.apache.hadoop.hbase.HRegionInfo;
34  import org.apache.hadoop.hbase.HTableDescriptor;
35  import org.apache.hadoop.hbase.client.Delete;
36  import org.apache.hadoop.hbase.client.Put;
37  import org.apache.hadoop.hbase.client.Result;
38  import org.apache.hadoop.hbase.client.Scan;
39  import org.apache.hadoop.hbase.util.Bytes;
40  import org.apache.hadoop.hbase.util.Writables;
41  import org.apache.hadoop.hdfs.MiniDFSCluster;
42  
43  /**
44   * {@link TestGet} is a medley of tests of get all done up as a single test.
45   * This class
46   */
47  public class TestGetClosestAtOrBefore extends HBaseTestCase {
48    private static final Log LOG = LogFactory.getLog(TestGetClosestAtOrBefore.class);
49    private MiniDFSCluster miniHdfs;
50  
51    private static final byte[] T00 = Bytes.toBytes("000");
52    private static final byte[] T10 = Bytes.toBytes("010");
53    private static final byte[] T11 = Bytes.toBytes("011");
54    private static final byte[] T12 = Bytes.toBytes("012");
55    private static final byte[] T20 = Bytes.toBytes("020");
56    private static final byte[] T30 = Bytes.toBytes("030");
57    private static final byte[] T31 = Bytes.toBytes("031");
58    private static final byte[] T35 = Bytes.toBytes("035");
59    private static final byte[] T40 = Bytes.toBytes("040");
60  
61    @Override
62    protected void setUp() throws Exception {
63      super.setUp();
64      this.miniHdfs = new MiniDFSCluster(this.conf, 1, true, null);
65      // Set the hbase.rootdir to be the home directory in mini dfs.
66      this.conf.set(HConstants.HBASE_DIR,
67        this.miniHdfs.getFileSystem().getHomeDirectory().toString());
68    }
69  
70    public void testUsingMetaAndBinary() throws IOException {
71      FileSystem filesystem = FileSystem.get(conf);
72      Path rootdir = filesystem.makeQualified(new Path(conf.get(HConstants.HBASE_DIR)));
73      filesystem.mkdirs(rootdir);
74      // Up flush size else we bind up when we use default catalog flush of 16k.
75      HRegionInfo.FIRST_META_REGIONINFO.getTableDesc().
76        setMemStoreFlushSize(64 * 1024 * 1024);
77      HRegion mr = HRegion.createHRegion(HRegionInfo.FIRST_META_REGIONINFO,
78        rootdir, this.conf);
79      // Write rows for three tables 'A', 'B', and 'C'.
80      for (char c = 'A'; c < 'D'; c++) {
81        HTableDescriptor htd = new HTableDescriptor("" + c);
82        final int last = 128;
83        final int interval = 2;
84        for (int i = 0; i <= last; i += interval) {
85          HRegionInfo hri = new HRegionInfo(htd,
86            i == 0? HConstants.EMPTY_BYTE_ARRAY: Bytes.toBytes((byte)i),
87            i == last? HConstants.EMPTY_BYTE_ARRAY: Bytes.toBytes((byte)i + interval));
88          Put put = new Put(hri.getRegionName());
89          put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
90                  Writables.getBytes(hri));
91          mr.put(put, false);
92        }
93      }
94      InternalScanner s = mr.getScanner(new Scan());
95      try {
96        List<KeyValue> keys = new ArrayList<KeyValue>();
97        while(s.next(keys)) {
98          LOG.info(keys);
99          keys.clear();
100       }
101     } finally {
102       s.close();
103     }
104     findRow(mr, 'C', 44, 44);
105     findRow(mr, 'C', 45, 44);
106     findRow(mr, 'C', 46, 46);
107     findRow(mr, 'C', 43, 42);
108     mr.flushcache();
109     findRow(mr, 'C', 44, 44);
110     findRow(mr, 'C', 45, 44);
111     findRow(mr, 'C', 46, 46);
112     findRow(mr, 'C', 43, 42);
113     // Now delete 'C' and make sure I don't get entries from 'B'.
114     byte [] firstRowInC = HRegionInfo.createRegionName(Bytes.toBytes("" + 'C'),
115       HConstants.EMPTY_BYTE_ARRAY, HConstants.ZEROES, false);
116     Scan scan = new Scan(firstRowInC);
117     s = mr.getScanner(scan);
118     try {
119       List<KeyValue> keys = new ArrayList<KeyValue>();
120       while (s.next(keys)) {
121         mr.delete(new Delete(keys.get(0).getRow()), null, false);
122         keys.clear();
123       }
124     } finally {
125       s.close();
126     }
127     // Assert we get null back (pass -1).
128     findRow(mr, 'C', 44, -1);
129     findRow(mr, 'C', 45, -1);
130     findRow(mr, 'C', 46, -1);
131     findRow(mr, 'C', 43, -1);
132     mr.flushcache();
133     findRow(mr, 'C', 44, -1);
134     findRow(mr, 'C', 45, -1);
135     findRow(mr, 'C', 46, -1);
136     findRow(mr, 'C', 43, -1);
137   }
138 
139   /*
140    * @param mr
141    * @param table
142    * @param rowToFind
143    * @param answer Pass -1 if we're not to find anything.
144    * @return Row found.
145    * @throws IOException
146    */
147   private byte [] findRow(final HRegion mr, final char table,
148     final int rowToFind, final int answer)
149   throws IOException {
150     byte [] tableb = Bytes.toBytes("" + table);
151     // Find the row.
152     byte [] tofindBytes = Bytes.toBytes((short)rowToFind);
153     byte [] metaKey = HRegionInfo.createRegionName(tableb, tofindBytes,
154       HConstants.NINES, false);
155     LOG.info("find=" + new String(metaKey));
156     Result r = mr.getClosestRowBefore(metaKey);
157     if (answer == -1) {
158       assertNull(r);
159       return null;
160     }
161     assertTrue(Bytes.compareTo(Bytes.toBytes((short)answer),
162       extractRowFromMetaRow(r.getRow())) == 0);
163     return r.getRow();
164   }
165 
166   private byte [] extractRowFromMetaRow(final byte [] b) {
167     int firstDelimiter = KeyValue.getDelimiter(b, 0, b.length,
168       HRegionInfo.DELIMITER);
169     int lastDelimiter = KeyValue.getDelimiterInReverse(b, 0, b.length,
170       HRegionInfo.DELIMITER);
171     int length = lastDelimiter - firstDelimiter - 1;
172     byte [] row = new byte[length];
173     System.arraycopy(b, firstDelimiter + 1, row, 0, length);
174     return row;
175   }
176 
177   /**
178    * Test file of multiple deletes and with deletes as final key.
179    * @see <a href="https://issues.apache.org/jira/browse/HBASE-751">HBASE-751</a>
180    */
181   public void testGetClosestRowBefore3() throws IOException{
182     HRegion region = null;
183     byte [] c0 = COLUMNS[0];
184     byte [] c1 = COLUMNS[1];
185     try {
186       HTableDescriptor htd = createTableDescriptor(getName());
187       region = createNewHRegion(htd, null, null);
188 
189       Put p = new Put(T00);
190       p.add(c0, c0, T00);
191       region.put(p);
192 
193       p = new Put(T10);
194       p.add(c0, c0, T10);
195       region.put(p);
196 
197       p = new Put(T20);
198       p.add(c0, c0, T20);
199       region.put(p);
200 
201       Result r = region.getClosestRowBefore(T20, c0);
202       assertTrue(Bytes.equals(T20, r.getRow()));
203 
204       Delete d = new Delete(T20);
205       d.deleteColumn(c0, c0);
206       region.delete(d, null, false);
207 
208       r = region.getClosestRowBefore(T20, c0);
209       assertTrue(Bytes.equals(T10, r.getRow()));
210 
211       p = new Put(T30);
212       p.add(c0, c0, T30);
213       region.put(p);
214 
215       r = region.getClosestRowBefore(T30, c0);
216       assertTrue(Bytes.equals(T30, r.getRow()));
217 
218       d = new Delete(T30);
219       d.deleteColumn(c0, c0);
220       region.delete(d, null, false);
221 
222       r = region.getClosestRowBefore(T30, c0);
223       assertTrue(Bytes.equals(T10, r.getRow()));
224       r = region.getClosestRowBefore(T31, c0);
225       assertTrue(Bytes.equals(T10, r.getRow()));
226 
227       region.flushcache();
228 
229       // try finding "010" after flush
230       r = region.getClosestRowBefore(T30, c0);
231       assertTrue(Bytes.equals(T10, r.getRow()));
232       r = region.getClosestRowBefore(T31, c0);
233       assertTrue(Bytes.equals(T10, r.getRow()));
234 
235       // Put into a different column family.  Should make it so I still get t10
236       p = new Put(T20);
237       p.add(c1, c1, T20);
238       region.put(p);
239 
240       r = region.getClosestRowBefore(T30, c0);
241       assertTrue(Bytes.equals(T10, r.getRow()));
242       r = region.getClosestRowBefore(T31, c0);
243       assertTrue(Bytes.equals(T10, r.getRow()));
244 
245       region.flushcache();
246 
247       r = region.getClosestRowBefore(T30, c0);
248       assertTrue(Bytes.equals(T10, r.getRow()));
249       r = region.getClosestRowBefore(T31, c0);
250       assertTrue(Bytes.equals(T10, r.getRow()));
251 
252       // Now try combo of memcache and mapfiles.  Delete the t20 COLUMS[1]
253       // in memory; make sure we get back t10 again.
254       d = new Delete(T20);
255       d.deleteColumn(c1, c1);
256       region.delete(d, null, false);
257       r = region.getClosestRowBefore(T30, c0);
258       assertTrue(Bytes.equals(T10, r.getRow()));
259 
260       // Ask for a value off the end of the file.  Should return t10.
261       r = region.getClosestRowBefore(T31, c0);
262       assertTrue(Bytes.equals(T10, r.getRow()));
263       region.flushcache();
264       r = region.getClosestRowBefore(T31, c0);
265       assertTrue(Bytes.equals(T10, r.getRow()));
266 
267       // Ok.  Let the candidate come out of hfile but have delete of
268       // the candidate be in memory.
269       p = new Put(T11);
270       p.add(c0, c0, T11);
271       region.put(p);
272       d = new Delete(T10);
273       d.deleteColumn(c1, c1);
274       r = region.getClosestRowBefore(T12, c0);
275       assertTrue(Bytes.equals(T11, r.getRow()));
276     } finally {
277       if (region != null) {
278         try {
279           region.close();
280         } catch (Exception e) {
281           e.printStackTrace();
282         }
283         region.getLog().closeAndDelete();
284       }
285     }
286   }
287 
288   /** For HBASE-694 */
289   public void testGetClosestRowBefore2() throws IOException{
290     HRegion region = null;
291     byte [] c0 = COLUMNS[0];
292     try {
293       HTableDescriptor htd = createTableDescriptor(getName());
294       region = createNewHRegion(htd, null, null);
295 
296       Put p = new Put(T10);
297       p.add(c0, c0, T10);
298       region.put(p);
299 
300       p = new Put(T30);
301       p.add(c0, c0, T30);
302       region.put(p);
303 
304       p = new Put(T40);
305       p.add(c0, c0, T40);
306       region.put(p);
307 
308       // try finding "035"
309       Result r = region.getClosestRowBefore(T35, c0);
310       assertTrue(Bytes.equals(T30, r.getRow()));
311 
312       region.flushcache();
313 
314       // try finding "035"
315       r = region.getClosestRowBefore(T35, c0);
316       assertTrue(Bytes.equals(T30, r.getRow()));
317 
318       p = new Put(T20);
319       p.add(c0, c0, T20);
320       region.put(p);
321 
322       // try finding "035"
323       r = region.getClosestRowBefore(T35, c0);
324       assertTrue(Bytes.equals(T30, r.getRow()));
325 
326       region.flushcache();
327 
328       // try finding "035"
329       r = region.getClosestRowBefore(T35, c0);
330       assertTrue(Bytes.equals(T30, r.getRow()));
331     } finally {
332       if (region != null) {
333         try {
334           region.close();
335         } catch (Exception e) {
336           e.printStackTrace();
337         }
338         region.getLog().closeAndDelete();
339       }
340     }
341   }
342 
343   @Override
344   protected void tearDown() throws Exception {
345     if (this.miniHdfs != null) {
346       this.miniHdfs.shutdown();
347     }
348     super.tearDown();
349   }
350 }