1   /**
2    * Copyright 2007 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.regionserver;
21  
22  import java.io.IOException;
23  import java.util.ArrayList;
24  import java.util.List;
25  
26  import org.apache.commons.logging.Log;
27  import org.apache.commons.logging.LogFactory;
28  import org.apache.hadoop.fs.Path;
29  import org.apache.hadoop.hbase.HBaseTestCase;
30  import org.apache.hadoop.hbase.HConstants;
31  import org.apache.hadoop.hbase.HTableDescriptor;
32  import org.apache.hadoop.hbase.KeyValue;
33  import org.apache.hadoop.hbase.regionserver.wal.HLog;
34  import org.apache.hadoop.hbase.client.Delete;
35  import org.apache.hadoop.hbase.client.Get;
36  import org.apache.hadoop.hbase.client.Result;
37  import org.apache.hadoop.hbase.client.Scan;
38  import org.apache.hadoop.hbase.io.hfile.HFileScanner;
39  import org.apache.hadoop.hbase.util.Bytes;
40  import org.apache.hadoop.hdfs.MiniDFSCluster;
41  
42  
43  /**
44   * Test compactions
45   */
46  public class TestCompaction extends HBaseTestCase {
47    static final Log LOG = LogFactory.getLog(TestCompaction.class.getName());
48    private HRegion r = null;
49    private Path compactionDir = null;
50    private Path regionCompactionDir = null;
51    private static final byte [] COLUMN_FAMILY = fam1;
52    private final byte [] STARTROW = Bytes.toBytes(START_KEY);
53    private static final byte [] COLUMN_FAMILY_TEXT = COLUMN_FAMILY;
54    private static final int COMPACTION_THRESHOLD = MAXVERSIONS;
55  
56    private MiniDFSCluster cluster;
57  
58    /** constructor */
59    public TestCompaction() {
60      super();
61  
62      // Set cache flush size to 1MB
63      conf.setInt("hbase.hregion.memstore.flush.size", 1024*1024);
64      conf.setInt("hbase.hregion.memstore.block.multiplier", 10);
65      this.cluster = null;
66    }
67  
68    @Override
69    public void setUp() throws Exception {
70      this.cluster = new MiniDFSCluster(conf, 2, true, (String[])null);
71      // Make the hbase rootdir match the minidfs we just span up
72      this.conf.set(HConstants.HBASE_DIR,
73        this.cluster.getFileSystem().getHomeDirectory().toString());
74      super.setUp();
75      HTableDescriptor htd = createTableDescriptor(getName());
76      this.r = createNewHRegion(htd, null, null);
77    }
78  
79    @Override
80    public void tearDown() throws Exception {
81      HLog hlog = r.getLog();
82      this.r.close();
83      hlog.closeAndDelete();
84      if (this.cluster != null) {
85        shutdownDfs(cluster);
86      }
87      super.tearDown();
88    }
89  
90    /**
91     * Test that on a major compaction, if all cells are expired or deleted, then
92     * we'll end up with no product.  Make sure scanner over region returns
93     * right answer in this case - and that it just basically works.
94     * @throws IOException
95     */
96    public void testMajorCompactingToNoOutput() throws IOException {
97      createStoreFile(r);
98      for (int i = 0; i < COMPACTION_THRESHOLD; i++) {
99        createStoreFile(r);
100     }
101     // Now delete everything.
102     InternalScanner s = r.getScanner(new Scan());
103     do {
104       List<KeyValue> results = new ArrayList<KeyValue>();
105       boolean result = s.next(results);
106       r.delete(new Delete(results.get(0).getRow()), null, false);
107       if (!result) break;
108     } while(true);
109     // Flush
110     r.flushcache();
111     // Major compact.
112     r.compactStores(true);
113     s = r.getScanner(new Scan());
114     int counter = 0;
115     do {
116       List<KeyValue> results = new ArrayList<KeyValue>();
117       boolean result = s.next(results);
118       if (!result) break;
119       counter++;
120     } while(true);
121     assertEquals(0, counter);
122   }
123 
124   /**
125    * Run compaction and flushing memstore
126    * Assert deletes get cleaned up.
127    * @throws Exception
128    */
129   public void testCompaction() throws Exception {
130     createStoreFile(r);
131     for (int i = 0; i < COMPACTION_THRESHOLD; i++) {
132       createStoreFile(r);
133     }
134     // Add more content.  Now there are about 5 versions of each column.
135     // Default is that there only 3 (MAXVERSIONS) versions allowed per column.
136     // Assert == 3 when we ask for versions.
137     addContent(new HRegionIncommon(r), Bytes.toString(COLUMN_FAMILY));
138 
139 
140     // FIX!!
141 //    Cell[] cellValues =
142 //      Cell.createSingleCellArray(r.get(STARTROW, COLUMN_FAMILY_TEXT, -1, 100 /*Too many*/));
143     Result result = r.get(new Get(STARTROW).addFamily(COLUMN_FAMILY_TEXT).setMaxVersions(100), null);
144 
145     // Assert that I can get 3 versions since it is the max I should get
146     assertEquals(COMPACTION_THRESHOLD, result.size());
147 //    assertEquals(cellValues.length, 3);
148     r.flushcache();
149     r.compactStores();
150     // Always 3 versions if that is what max versions is.
151     byte [] secondRowBytes = START_KEY.getBytes(HConstants.UTF8_ENCODING);
152     // Increment the least significant character so we get to next row.
153     secondRowBytes[START_KEY_BYTES.length - 1]++;
154     // FIX
155     result = r.get(new Get(secondRowBytes).addFamily(COLUMN_FAMILY_TEXT).setMaxVersions(100), null);
156 
157     // Assert that I can get 3 versions since it is the max I should get
158     assertEquals(3, result.size());
159 //
160 //    cellValues = Cell.createSingleCellArray(r.get(secondRowBytes, COLUMN_FAMILY_TEXT, -1, 100/*Too many*/));
161 //    LOG.info("Count of " + Bytes.toString(secondRowBytes) + ": " +
162 //      cellValues.length);
163 //    assertTrue(cellValues.length == 3);
164 
165     // Now add deletes to memstore and then flush it.  That will put us over
166     // the compaction threshold of 3 store files.  Compacting these store files
167     // should result in a compacted store file that has no references to the
168     // deleted row.
169     Delete delete = new Delete(secondRowBytes, System.currentTimeMillis(), null);
170     byte [][] famAndQf = {COLUMN_FAMILY, null};
171     delete.deleteFamily(famAndQf[0]);
172     r.delete(delete, null, true);
173 
174     // Assert deleted.
175 
176     result = r.get(new Get(secondRowBytes).addFamily(COLUMN_FAMILY_TEXT).setMaxVersions(100), null );
177     assertTrue(result.isEmpty());
178 
179 
180     r.flushcache();
181     result = r.get(new Get(secondRowBytes).addFamily(COLUMN_FAMILY_TEXT).setMaxVersions(100), null );
182     assertTrue(result.isEmpty());
183 
184     // Add a bit of data and flush.  Start adding at 'bbb'.
185     createSmallerStoreFile(this.r);
186     r.flushcache();
187     // Assert that the second row is still deleted.
188     result = r.get(new Get(secondRowBytes).addFamily(COLUMN_FAMILY_TEXT).setMaxVersions(100), null );
189     assertTrue(result.isEmpty());
190 
191     // Force major compaction.
192     r.compactStores(true);
193     assertEquals(r.getStore(COLUMN_FAMILY_TEXT).getStorefiles().size(), 1);
194 
195     result = r.get(new Get(secondRowBytes).addFamily(COLUMN_FAMILY_TEXT).setMaxVersions(100), null );
196     assertTrue(result.isEmpty());
197 
198     // Make sure the store files do have some 'aaa' keys in them -- exactly 3.
199     // Also, that compacted store files do not have any secondRowBytes because
200     // they were deleted.
201     int count = 0;
202     boolean containsStartRow = false;
203     for (StoreFile f: this.r.stores.get(COLUMN_FAMILY_TEXT).getStorefiles()) {
204       HFileScanner scanner = f.getReader().getScanner(false, false);
205       scanner.seekTo();
206       do {
207         byte [] row = scanner.getKeyValue().getRow();
208         if (Bytes.equals(row, STARTROW)) {
209           containsStartRow = true;
210           count++;
211         } else {
212           // After major compaction, should be none of these rows in compacted
213           // file.
214           assertFalse(Bytes.equals(row, secondRowBytes));
215         }
216       } while(scanner.next());
217     }
218     assertTrue(containsStartRow);
219     assertTrue(count == 3);
220     // Do a simple TTL test.
221     final int ttlInSeconds = 1;
222     for (Store store: this.r.stores.values()) {
223       store.ttl = ttlInSeconds * 1000;
224     }
225     Thread.sleep(ttlInSeconds * 1000);
226     r.compactStores(true);
227     count = count();
228     assertTrue(count == 0);
229   }
230 
231   private int count() throws IOException {
232     int count = 0;
233     for (StoreFile f: this.r.stores.
234         get(COLUMN_FAMILY_TEXT).getStorefiles()) {
235       HFileScanner scanner = f.getReader().getScanner(false, false);
236       if (!scanner.seekTo()) {
237         continue;
238       }
239       do {
240         count++;
241       } while(scanner.next());
242     }
243     return count;
244   }
245 
246   private void createStoreFile(final HRegion region) throws IOException {
247     HRegionIncommon loader = new HRegionIncommon(region);
248     addContent(loader, Bytes.toString(COLUMN_FAMILY));
249     loader.flushcache();
250   }
251 
252   private void createSmallerStoreFile(final HRegion region) throws IOException {
253     HRegionIncommon loader = new HRegionIncommon(region);
254     addContent(loader, Bytes.toString(COLUMN_FAMILY), ("" +
255     		"bbb").getBytes(), null);
256     loader.flushcache();
257   }
258 }