View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.regionserver;
21  
22  import java.io.IOException;
23  import java.util.ArrayList;
24  import java.util.Iterator;
25  import java.util.List;
26  
27  import org.apache.hadoop.hbase.*;
28  import org.apache.hadoop.hbase.testclassification.SmallTests;
29  import org.apache.hadoop.hbase.util.Bytes;
30  import org.apache.hadoop.hbase.util.CollectionBackedScanner;
31  import org.junit.experimental.categories.Category;
32  
33  @Category(SmallTests.class)
34  public class TestKeyValueHeap extends HBaseTestCase {
35    private static final boolean PRINT = false;
36  
37    List<KeyValueScanner> scanners = new ArrayList<KeyValueScanner>();
38  
39    private byte[] row1;
40    private byte[] fam1;
41    private byte[] col1;
42    private byte[] data;
43  
44    private byte[] row2;
45    private byte[] fam2;
46    private byte[] col2;
47  
48    private byte[] col3;
49    private byte[] col4;
50    private byte[] col5;
51  
52    public void setUp() throws Exception {
53      super.setUp();
54      data = Bytes.toBytes("data");
55      row1 = Bytes.toBytes("row1");
56      fam1 = Bytes.toBytes("fam1");
57      col1 = Bytes.toBytes("col1");
58      row2 = Bytes.toBytes("row2");
59      fam2 = Bytes.toBytes("fam2");
60      col2 = Bytes.toBytes("col2");
61      col3 = Bytes.toBytes("col3");
62      col4 = Bytes.toBytes("col4");
63      col5 = Bytes.toBytes("col5");
64    }
65  
66    public void testSorted() throws IOException{
67      //Cases that need to be checked are:
68      //1. The "smallest" KeyValue is in the same scanners as current
69      //2. Current scanner gets empty
70  
71      List<KeyValue> l1 = new ArrayList<KeyValue>();
72      l1.add(new KeyValue(row1, fam1, col5, data));
73      l1.add(new KeyValue(row2, fam1, col1, data));
74      l1.add(new KeyValue(row2, fam1, col2, data));
75      scanners.add(new Scanner(l1));
76  
77      List<KeyValue> l2 = new ArrayList<KeyValue>();
78      l2.add(new KeyValue(row1, fam1, col1, data));
79      l2.add(new KeyValue(row1, fam1, col2, data));
80      scanners.add(new Scanner(l2));
81  
82      List<KeyValue> l3 = new ArrayList<KeyValue>();
83      l3.add(new KeyValue(row1, fam1, col3, data));
84      l3.add(new KeyValue(row1, fam1, col4, data));
85      l3.add(new KeyValue(row1, fam2, col1, data));
86      l3.add(new KeyValue(row1, fam2, col2, data));
87      l3.add(new KeyValue(row2, fam1, col3, data));
88      scanners.add(new Scanner(l3));
89  
90      List<KeyValue> expected = new ArrayList<KeyValue>();
91      expected.add(new KeyValue(row1, fam1, col1, data));
92      expected.add(new KeyValue(row1, fam1, col2, data));
93      expected.add(new KeyValue(row1, fam1, col3, data));
94      expected.add(new KeyValue(row1, fam1, col4, data));
95      expected.add(new KeyValue(row1, fam1, col5, data));
96      expected.add(new KeyValue(row1, fam2, col1, data));
97      expected.add(new KeyValue(row1, fam2, col2, data));
98      expected.add(new KeyValue(row2, fam1, col1, data));
99      expected.add(new KeyValue(row2, fam1, col2, data));
100     expected.add(new KeyValue(row2, fam1, col3, data));
101 
102     //Creating KeyValueHeap
103     KeyValueHeap kvh =
104       new KeyValueHeap(scanners, KeyValue.COMPARATOR);
105 
106     List<KeyValue> actual = new ArrayList<KeyValue>();
107     while(kvh.peek() != null){
108       actual.add(kvh.next());
109     }
110 
111     assertEquals(expected.size(), actual.size());
112     for(int i=0; i<expected.size(); i++){
113       assertEquals(expected.get(i), actual.get(i));
114       if(PRINT){
115         System.out.println("expected " +expected.get(i)+
116             "\nactual   " +actual.get(i) +"\n");
117       }
118     }
119 
120     //Check if result is sorted according to Comparator
121     for(int i=0; i<actual.size()-1; i++){
122       int ret = KeyValue.COMPARATOR.compare(actual.get(i), actual.get(i+1));
123       assertTrue(ret < 0);
124     }
125 
126   }
127 
128   public void testSeek() throws IOException {
129     //Cases:
130     //1. Seek KeyValue that is not in scanner
131     //2. Check that smallest that is returned from a seek is correct
132 
133     List<KeyValue> l1 = new ArrayList<KeyValue>();
134     l1.add(new KeyValue(row1, fam1, col5, data));
135     l1.add(new KeyValue(row2, fam1, col1, data));
136     l1.add(new KeyValue(row2, fam1, col2, data));
137     scanners.add(new Scanner(l1));
138 
139     List<KeyValue> l2 = new ArrayList<KeyValue>();
140     l2.add(new KeyValue(row1, fam1, col1, data));
141     l2.add(new KeyValue(row1, fam1, col2, data));
142     scanners.add(new Scanner(l2));
143 
144     List<KeyValue> l3 = new ArrayList<KeyValue>();
145     l3.add(new KeyValue(row1, fam1, col3, data));
146     l3.add(new KeyValue(row1, fam1, col4, data));
147     l3.add(new KeyValue(row1, fam2, col1, data));
148     l3.add(new KeyValue(row1, fam2, col2, data));
149     l3.add(new KeyValue(row2, fam1, col3, data));
150     scanners.add(new Scanner(l3));
151 
152     List<KeyValue> expected = new ArrayList<KeyValue>();
153     expected.add(new KeyValue(row2, fam1, col1, data));
154 
155     //Creating KeyValueHeap
156     KeyValueHeap kvh =
157       new KeyValueHeap(scanners, KeyValue.COMPARATOR);
158 
159     KeyValue seekKv = new KeyValue(row2, fam1, null, null);
160     kvh.seek(seekKv);
161 
162     List<KeyValue> actual = new ArrayList<KeyValue>();
163     actual.add(kvh.peek());
164 
165     assertEquals(expected.size(), actual.size());
166     for(int i=0; i<expected.size(); i++){
167       assertEquals(expected.get(i), actual.get(i));
168       if(PRINT){
169         System.out.println("expected " +expected.get(i)+
170             "\nactual   " +actual.get(i) +"\n");
171       }
172     }
173 
174   }
175 
176   public void testScannerLeak() throws IOException {
177     // Test for unclosed scanners (HBASE-1927)
178 
179     List<KeyValue> l1 = new ArrayList<KeyValue>();
180     l1.add(new KeyValue(row1, fam1, col5, data));
181     l1.add(new KeyValue(row2, fam1, col1, data));
182     l1.add(new KeyValue(row2, fam1, col2, data));
183     scanners.add(new Scanner(l1));
184 
185     List<KeyValue> l2 = new ArrayList<KeyValue>();
186     l2.add(new KeyValue(row1, fam1, col1, data));
187     l2.add(new KeyValue(row1, fam1, col2, data));
188     scanners.add(new Scanner(l2));
189 
190     List<KeyValue> l3 = new ArrayList<KeyValue>();
191     l3.add(new KeyValue(row1, fam1, col3, data));
192     l3.add(new KeyValue(row1, fam1, col4, data));
193     l3.add(new KeyValue(row1, fam2, col1, data));
194     l3.add(new KeyValue(row1, fam2, col2, data));
195     l3.add(new KeyValue(row2, fam1, col3, data));
196     scanners.add(new Scanner(l3));
197 
198     List<KeyValue> l4 = new ArrayList<KeyValue>();
199     scanners.add(new Scanner(l4));
200 
201     //Creating KeyValueHeap
202     KeyValueHeap kvh = new KeyValueHeap(scanners, KeyValue.COMPARATOR);
203 
204     while(kvh.next() != null);
205 
206     for(KeyValueScanner scanner : scanners) {
207       assertTrue(((Scanner)scanner).isClosed());
208     }
209   }
210 
211   public void testScannerException() throws IOException {
212     // Test for NPE issue when exception happens in scanners (HBASE-13835)
213 
214     List<KeyValue> l1 = new ArrayList<KeyValue>();
215     l1.add(new KeyValue(row1, fam1, col5, data));
216     l1.add(new KeyValue(row2, fam1, col1, data));
217     l1.add(new KeyValue(row2, fam1, col2, data));
218     SeekScanner s1 = new SeekScanner(l1);
219     scanners.add(s1);
220 
221     List<KeyValue> l2 = new ArrayList<KeyValue>();
222     l2.add(new KeyValue(row1, fam1, col1, data));
223     l2.add(new KeyValue(row1, fam1, col2, data));
224     SeekScanner s2 = new SeekScanner(l2);
225     scanners.add(s2);
226 
227     List<KeyValue> l3 = new ArrayList<KeyValue>();
228     l3.add(new KeyValue(row1, fam1, col3, data));
229     l3.add(new KeyValue(row1, fam1, col4, data));
230     l3.add(new KeyValue(row1, fam2, col1, data));
231     l3.add(new KeyValue(row1, fam2, col2, data));
232     l3.add(new KeyValue(row2, fam1, col3, data));
233     SeekScanner s3 = new SeekScanner(l3);
234     scanners.add(s3);
235 
236     List<KeyValue> l4 = new ArrayList<KeyValue>();
237     SeekScanner s4 = new SeekScanner(l4);
238     scanners.add(s4);
239 
240     // Creating KeyValueHeap
241     KeyValueHeap kvh = new KeyValueHeap(scanners, KeyValue.COMPARATOR);
242 
243     try {
244       for (KeyValueScanner scanner : scanners) {
245         ((SeekScanner) scanner).setRealSeekDone(false);
246       }
247       while (kvh.next() != null);
248       // The pollRealKV should throw IOE.
249       assertTrue(false);
250     } catch (IOException ioe) {
251       kvh.close();
252     }
253 
254     // It implies there is no NPE thrown from kvh.close() if getting here
255     for (KeyValueScanner scanner : scanners) {
256       // Verify that close is called and only called once for each scanner
257       assertTrue(((SeekScanner) scanner).isClosed());
258       assertEquals(((SeekScanner) scanner).getClosedNum(), 1);
259     }
260   }
261 
262   private static class Scanner extends CollectionBackedScanner {
263     private Iterator<KeyValue> iter;
264     private KeyValue current;
265     private boolean closed = false;
266 
267     public Scanner(List<KeyValue> list) {
268       super(list);
269     }
270 
271     @Override
272     public void close(){
273       closed = true;
274     }
275 
276     public boolean isClosed() {
277       return closed;
278     }
279   }
280 
281   private static class SeekScanner extends Scanner {
282     private int closedNum = 0;
283     private boolean realSeekDone = true;
284 
285     public SeekScanner(List<KeyValue> list) {
286       super(list);
287     }
288 
289     @Override
290     public void close() {
291       super.close();
292       closedNum++;
293     }
294 
295     public int getClosedNum() {
296       return closedNum;
297     }
298 
299     @Override
300     public boolean realSeekDone() {
301       return realSeekDone;
302     }
303 
304     public void setRealSeekDone(boolean done) {
305       realSeekDone = done;
306     }
307 
308     @Override
309     public void enforceSeek() throws IOException {
310       throw new IOException("enforceSeek must not be called on a " + "non-lazy scanner");
311     }
312   }
313 }