1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver;
21
22 import java.io.IOException;
23 import java.util.ArrayList;
24 import java.util.List;
25
26 import org.apache.commons.logging.Log;
27 import org.apache.commons.logging.LogFactory;
28 import org.apache.hadoop.fs.Path;
29 import org.apache.hadoop.hbase.HBaseTestCase;
30 import org.apache.hadoop.hbase.HConstants;
31 import org.apache.hadoop.hbase.HTableDescriptor;
32 import org.apache.hadoop.hbase.KeyValue;
33 import org.apache.hadoop.hbase.regionserver.wal.HLog;
34 import org.apache.hadoop.hbase.client.Delete;
35 import org.apache.hadoop.hbase.client.Get;
36 import org.apache.hadoop.hbase.client.Result;
37 import org.apache.hadoop.hbase.client.Scan;
38 import org.apache.hadoop.hbase.io.hfile.HFileScanner;
39 import org.apache.hadoop.hbase.util.Bytes;
40 import org.apache.hadoop.hdfs.MiniDFSCluster;
41
42
43
44
45
46 public class TestCompaction extends HBaseTestCase {
47 static final Log LOG = LogFactory.getLog(TestCompaction.class.getName());
48 private HRegion r = null;
49 private Path compactionDir = null;
50 private Path regionCompactionDir = null;
51 private static final byte [] COLUMN_FAMILY = fam1;
52 private final byte [] STARTROW = Bytes.toBytes(START_KEY);
53 private static final byte [] COLUMN_FAMILY_TEXT = COLUMN_FAMILY;
54 private static final int COMPACTION_THRESHOLD = MAXVERSIONS;
55
56 private MiniDFSCluster cluster;
57
58
59 public TestCompaction() {
60 super();
61
62
63 conf.setInt("hbase.hregion.memstore.flush.size", 1024*1024);
64 conf.setInt("hbase.hregion.memstore.block.multiplier", 10);
65 this.cluster = null;
66 }
67
68 @Override
69 public void setUp() throws Exception {
70 this.cluster = new MiniDFSCluster(conf, 2, true, (String[])null);
71
72 this.conf.set(HConstants.HBASE_DIR,
73 this.cluster.getFileSystem().getHomeDirectory().toString());
74 super.setUp();
75 HTableDescriptor htd = createTableDescriptor(getName());
76 this.r = createNewHRegion(htd, null, null);
77 }
78
79 @Override
80 public void tearDown() throws Exception {
81 HLog hlog = r.getLog();
82 this.r.close();
83 hlog.closeAndDelete();
84 if (this.cluster != null) {
85 shutdownDfs(cluster);
86 }
87 super.tearDown();
88 }
89
90
91
92
93
94
95
96 public void testMajorCompactingToNoOutput() throws IOException {
97 createStoreFile(r);
98 for (int i = 0; i < COMPACTION_THRESHOLD; i++) {
99 createStoreFile(r);
100 }
101
102 InternalScanner s = r.getScanner(new Scan());
103 do {
104 List<KeyValue> results = new ArrayList<KeyValue>();
105 boolean result = s.next(results);
106 r.delete(new Delete(results.get(0).getRow()), null, false);
107 if (!result) break;
108 } while(true);
109
110 r.flushcache();
111
112 r.compactStores(true);
113 s = r.getScanner(new Scan());
114 int counter = 0;
115 do {
116 List<KeyValue> results = new ArrayList<KeyValue>();
117 boolean result = s.next(results);
118 if (!result) break;
119 counter++;
120 } while(true);
121 assertEquals(0, counter);
122 }
123
124
125
126
127
128
129 public void testCompaction() throws Exception {
130 createStoreFile(r);
131 for (int i = 0; i < COMPACTION_THRESHOLD; i++) {
132 createStoreFile(r);
133 }
134
135
136
137 addContent(new HRegionIncommon(r), Bytes.toString(COLUMN_FAMILY));
138
139
140
141
142
143 Result result = r.get(new Get(STARTROW).addFamily(COLUMN_FAMILY_TEXT).setMaxVersions(100), null);
144
145
146 assertEquals(COMPACTION_THRESHOLD, result.size());
147
148 r.flushcache();
149 r.compactStores();
150
151 byte [] secondRowBytes = START_KEY.getBytes(HConstants.UTF8_ENCODING);
152
153 secondRowBytes[START_KEY_BYTES.length - 1]++;
154
155 result = r.get(new Get(secondRowBytes).addFamily(COLUMN_FAMILY_TEXT).setMaxVersions(100), null);
156
157
158 assertEquals(3, result.size());
159
160
161
162
163
164
165
166
167
168
169 Delete delete = new Delete(secondRowBytes, System.currentTimeMillis(), null);
170 byte [][] famAndQf = {COLUMN_FAMILY, null};
171 delete.deleteFamily(famAndQf[0]);
172 r.delete(delete, null, true);
173
174
175
176 result = r.get(new Get(secondRowBytes).addFamily(COLUMN_FAMILY_TEXT).setMaxVersions(100), null );
177 assertTrue(result.isEmpty());
178
179
180 r.flushcache();
181 result = r.get(new Get(secondRowBytes).addFamily(COLUMN_FAMILY_TEXT).setMaxVersions(100), null );
182 assertTrue(result.isEmpty());
183
184
185 createSmallerStoreFile(this.r);
186 r.flushcache();
187
188 result = r.get(new Get(secondRowBytes).addFamily(COLUMN_FAMILY_TEXT).setMaxVersions(100), null );
189 assertTrue(result.isEmpty());
190
191
192 r.compactStores(true);
193 assertEquals(r.getStore(COLUMN_FAMILY_TEXT).getStorefiles().size(), 1);
194
195 result = r.get(new Get(secondRowBytes).addFamily(COLUMN_FAMILY_TEXT).setMaxVersions(100), null );
196 assertTrue(result.isEmpty());
197
198
199
200
201 int count = 0;
202 boolean containsStartRow = false;
203 for (StoreFile f: this.r.stores.get(COLUMN_FAMILY_TEXT).getStorefiles()) {
204 HFileScanner scanner = f.getReader().getScanner(false, false);
205 scanner.seekTo();
206 do {
207 byte [] row = scanner.getKeyValue().getRow();
208 if (Bytes.equals(row, STARTROW)) {
209 containsStartRow = true;
210 count++;
211 } else {
212
213
214 assertFalse(Bytes.equals(row, secondRowBytes));
215 }
216 } while(scanner.next());
217 }
218 assertTrue(containsStartRow);
219 assertTrue(count == 3);
220
221 final int ttlInSeconds = 1;
222 for (Store store: this.r.stores.values()) {
223 store.ttl = ttlInSeconds * 1000;
224 }
225 Thread.sleep(ttlInSeconds * 1000);
226 r.compactStores(true);
227 count = count();
228 assertTrue(count == 0);
229 }
230
231 private int count() throws IOException {
232 int count = 0;
233 for (StoreFile f: this.r.stores.
234 get(COLUMN_FAMILY_TEXT).getStorefiles()) {
235 HFileScanner scanner = f.getReader().getScanner(false, false);
236 if (!scanner.seekTo()) {
237 continue;
238 }
239 do {
240 count++;
241 } while(scanner.next());
242 }
243 return count;
244 }
245
246 private void createStoreFile(final HRegion region) throws IOException {
247 HRegionIncommon loader = new HRegionIncommon(region);
248 addContent(loader, Bytes.toString(COLUMN_FAMILY));
249 loader.flushcache();
250 }
251
252 private void createSmallerStoreFile(final HRegion region) throws IOException {
253 HRegionIncommon loader = new HRegionIncommon(region);
254 addContent(loader, Bytes.toString(COLUMN_FAMILY), ("" +
255 "bbb").getBytes(), null);
256 loader.flushcache();
257 }
258 }