1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertTrue;
24 import static org.mockito.Matchers.any;
25 import static org.mockito.Matchers.anyInt;
26 import static org.mockito.Matchers.eq;
27 import static org.mockito.Mockito.doNothing;
28 import static org.mockito.Mockito.doThrow;
29 import static org.mockito.Mockito.spy;
30 import static org.mockito.Mockito.when;
31
32 import java.io.IOException;
33 import java.util.ArrayList;
34 import java.util.List;
35
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FileSystem;
38 import org.apache.hadoop.fs.Path;
39 import org.apache.hadoop.hbase.Cell;
40 import org.apache.hadoop.hbase.HBaseTestingUtility;
41 import org.apache.hadoop.hbase.HColumnDescriptor;
42 import org.apache.hadoop.hbase.HConstants;
43 import org.apache.hadoop.hbase.HRegionInfo;
44 import org.apache.hadoop.hbase.HTableDescriptor;
45 import org.apache.hadoop.hbase.Server;
46 import org.apache.hadoop.hbase.testclassification.SmallTests;
47 import org.apache.hadoop.hbase.TableName;
48 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
49 import org.apache.hadoop.hbase.client.Scan;
50 import org.apache.hadoop.hbase.io.hfile.LruBlockCache;
51 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
52 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
53 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
54 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
55 import org.apache.hadoop.hbase.wal.WALFactory;
56 import org.apache.hadoop.hbase.util.Bytes;
57 import org.apache.hadoop.hbase.util.FSUtils;
58 import org.apache.hadoop.hbase.util.PairOfSameType;
59 import org.apache.zookeeper.KeeperException;
60 import org.junit.After;
61 import org.junit.Before;
62 import org.junit.Test;
63 import org.junit.experimental.categories.Category;
64 import org.mockito.Mockito;
65
66 import com.google.common.collect.ImmutableList;
67
68
69
70
71
72 @Category(SmallTests.class)
73 public class TestSplitTransaction {
74 private final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
75 private final Path testdir =
76 TEST_UTIL.getDataTestDir(this.getClass().getName());
77 private HRegion parent;
78 private WALFactory wals;
79 private FileSystem fs;
80 private static final byte [] STARTROW = new byte [] {'a', 'a', 'a'};
81
82 private static final byte [] ENDROW = new byte [] {'{', '{', '{'};
83 private static final byte [] GOOD_SPLIT_ROW = new byte [] {'d', 'd', 'd'};
84 private static final byte [] CF = HConstants.CATALOG_FAMILY;
85
86 private static boolean preRollBackCalled = false;
87 private static boolean postRollBackCalled = false;
88
89 @Before public void setup() throws IOException {
90 this.fs = FileSystem.get(TEST_UTIL.getConfiguration());
91 TEST_UTIL.getConfiguration().set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, CustomObserver.class.getName());
92 this.fs.delete(this.testdir, true);
93 final Configuration walConf = new Configuration(TEST_UTIL.getConfiguration());
94 FSUtils.setRootDir(walConf, this.testdir);
95 this.wals = new WALFactory(walConf, null, this.getClass().getName());
96
97 this.parent = createRegion(this.testdir, this.wals);
98 RegionCoprocessorHost host = new RegionCoprocessorHost(this.parent, null, TEST_UTIL.getConfiguration());
99 this.parent.setCoprocessorHost(host);
100 TEST_UTIL.getConfiguration().setBoolean("hbase.testing.nocluster", true);
101 }
102
103 @After public void teardown() throws IOException {
104 if (this.parent != null && !this.parent.isClosed()) this.parent.close();
105 Path regionDir = this.parent.getRegionFileSystem().getRegionDir();
106 if (this.fs.exists(regionDir) && !this.fs.delete(regionDir, true)) {
107 throw new IOException("Failed delete of " + regionDir);
108 }
109 if (this.wals != null) {
110 this.wals.close();
111 }
112 this.fs.delete(this.testdir, true);
113 }
114
115 @Test public void testFailAfterPONR() throws IOException, KeeperException {
116 final int rowcount = TEST_UTIL.loadRegion(this.parent, CF);
117 assertTrue(rowcount > 0);
118 int parentRowCount = countRows(this.parent);
119 assertEquals(rowcount, parentRowCount);
120
121
122 SplitTransaction st = prepareGOOD_SPLIT_ROW();
123 SplitTransaction spiedUponSt = spy(st);
124 Mockito
125 .doThrow(new MockedFailedDaughterOpen())
126 .when(spiedUponSt)
127 .openDaughterRegion((Server) Mockito.anyObject(),
128 (HRegion) Mockito.anyObject());
129
130
131 boolean expectedException = false;
132 Server mockServer = Mockito.mock(Server.class);
133 when(mockServer.getConfiguration()).thenReturn(TEST_UTIL.getConfiguration());
134 try {
135 spiedUponSt.execute(mockServer, null);
136 } catch (IOException e) {
137 if (e.getCause() != null &&
138 e.getCause() instanceof MockedFailedDaughterOpen) {
139 expectedException = true;
140 }
141 }
142 assertTrue(expectedException);
143
144 assertFalse(spiedUponSt.rollback(null, null));
145
146
147
148 Path tableDir = this.parent.getRegionFileSystem().getTableDir();
149 Path daughterADir = new Path(tableDir, spiedUponSt.getFirstDaughter().getEncodedName());
150 Path daughterBDir = new Path(tableDir, spiedUponSt.getSecondDaughter().getEncodedName());
151 assertTrue(TEST_UTIL.getTestFileSystem().exists(daughterADir));
152 assertTrue(TEST_UTIL.getTestFileSystem().exists(daughterBDir));
153 }
154
155
156
157
158
159 @Test public void testPrepare() throws IOException {
160 prepareGOOD_SPLIT_ROW();
161 }
162
163 private SplitTransaction prepareGOOD_SPLIT_ROW() {
164 return prepareGOOD_SPLIT_ROW(this.parent);
165 }
166
167 private SplitTransaction prepareGOOD_SPLIT_ROW(final HRegion parentRegion) {
168 SplitTransaction st = new SplitTransaction(parentRegion, GOOD_SPLIT_ROW);
169 assertTrue(st.prepare());
170 return st;
171 }
172
173
174
175
176 @Test public void testPrepareWithRegionsWithReference() throws IOException {
177 HStore storeMock = Mockito.mock(HStore.class);
178 when(storeMock.hasReferences()).thenReturn(true);
179 when(storeMock.getFamily()).thenReturn(new HColumnDescriptor("cf"));
180 when(storeMock.close()).thenReturn(ImmutableList.<StoreFile>of());
181 this.parent.stores.put(Bytes.toBytes(""), storeMock);
182
183 SplitTransaction st = new SplitTransaction(this.parent, GOOD_SPLIT_ROW);
184
185 assertFalse("a region should not be splittable if it has instances of store file references",
186 st.prepare());
187 }
188
189
190
191
192 @Test public void testPrepareWithBadSplitRow() throws IOException {
193
194 SplitTransaction st = new SplitTransaction(this.parent, STARTROW);
195 assertFalse(st.prepare());
196 st = new SplitTransaction(this.parent, HConstants.EMPTY_BYTE_ARRAY);
197 assertFalse(st.prepare());
198 st = new SplitTransaction(this.parent, new byte [] {'A', 'A', 'A'});
199 assertFalse(st.prepare());
200 st = new SplitTransaction(this.parent, ENDROW);
201 assertFalse(st.prepare());
202 }
203
204 @Test public void testPrepareWithClosedRegion() throws IOException {
205 this.parent.close();
206 SplitTransaction st = new SplitTransaction(this.parent, GOOD_SPLIT_ROW);
207 assertFalse(st.prepare());
208 }
209
210 @Test public void testWholesomeSplit() throws IOException {
211 final int rowcount = TEST_UTIL.loadRegion(this.parent, CF, true);
212 assertTrue(rowcount > 0);
213 int parentRowCount = countRows(this.parent);
214 assertEquals(rowcount, parentRowCount);
215
216
217
218 CacheConfig cacheConf = new CacheConfig(TEST_UTIL.getConfiguration());
219 ((LruBlockCache) cacheConf.getBlockCache()).clearCache();
220
221
222 SplitTransaction st = prepareGOOD_SPLIT_ROW();
223
224
225 Server mockServer = Mockito.mock(Server.class);
226 when(mockServer.getConfiguration()).thenReturn(TEST_UTIL.getConfiguration());
227 PairOfSameType<HRegion> daughters = st.execute(mockServer, null);
228
229 assertTrue(this.fs.exists(this.parent.getRegionFileSystem().getSplitsDir()));
230
231 assertTrue(this.parent.isClosed());
232
233
234
235 assertEquals(0, this.fs.listStatus(this.parent.getRegionFileSystem().getSplitsDir()).length);
236
237 assertTrue(Bytes.equals(this.parent.getStartKey(), daughters.getFirst().getStartKey()));
238 assertTrue(Bytes.equals(GOOD_SPLIT_ROW, daughters.getFirst().getEndKey()));
239 assertTrue(Bytes.equals(daughters.getSecond().getStartKey(), GOOD_SPLIT_ROW));
240 assertTrue(Bytes.equals(this.parent.getEndKey(), daughters.getSecond().getEndKey()));
241
242 int daughtersRowCount = 0;
243 for (HRegion openRegion: daughters) {
244 try {
245 int count = countRows(openRegion);
246 assertTrue(count > 0 && count != rowcount);
247 daughtersRowCount += count;
248 } finally {
249 HRegion.closeHRegion(openRegion);
250 }
251 }
252 assertEquals(rowcount, daughtersRowCount);
253
254 assertTrue(!this.parent.lock.writeLock().isHeldByCurrentThread());
255 }
256
257 @Test
258 public void testCountReferencesFailsSplit() throws IOException {
259 final int rowcount = TEST_UTIL.loadRegion(this.parent, CF);
260 assertTrue(rowcount > 0);
261 int parentRowCount = countRows(this.parent);
262 assertEquals(rowcount, parentRowCount);
263
264
265 HRegion spiedRegion = spy(this.parent);
266 SplitTransaction st = prepareGOOD_SPLIT_ROW(spiedRegion);
267 SplitTransaction spiedUponSt = spy(st);
268 doThrow(new IOException("Failing split. Expected reference file count isn't equal."))
269 .when(spiedUponSt).assertReferenceFileCount(anyInt(),
270 eq(new Path(this.parent.getRegionFileSystem().getTableDir(),
271 st.getSecondDaughter().getEncodedName())));
272
273
274 boolean expectedException = false;
275 Server mockServer = Mockito.mock(Server.class);
276 when(mockServer.getConfiguration()).thenReturn(TEST_UTIL.getConfiguration());
277 try {
278 spiedUponSt.execute(mockServer, null);
279 } catch (IOException e) {
280 expectedException = true;
281 }
282 assertTrue(expectedException);
283 }
284
285
286 @Test public void testRollback() throws IOException {
287 final int rowcount = TEST_UTIL.loadRegion(this.parent, CF);
288 assertTrue(rowcount > 0);
289 int parentRowCount = countRows(this.parent);
290 assertEquals(rowcount, parentRowCount);
291
292
293 HRegion spiedRegion = spy(this.parent);
294 SplitTransaction st = prepareGOOD_SPLIT_ROW(spiedRegion);
295 SplitTransaction spiedUponSt = spy(st);
296 doNothing().when(spiedUponSt).assertReferenceFileCount(anyInt(),
297 eq(parent.getRegionFileSystem().getSplitsDir(st.getFirstDaughter())));
298 when(spiedRegion.createDaughterRegionFromSplits(spiedUponSt.getSecondDaughter())).
299 thenThrow(new MockedFailedDaughterCreation());
300
301 boolean expectedException = false;
302 Server mockServer = Mockito.mock(Server.class);
303 when(mockServer.getConfiguration()).thenReturn(TEST_UTIL.getConfiguration());
304 try {
305 spiedUponSt.execute(mockServer, null);
306 } catch (MockedFailedDaughterCreation e) {
307 expectedException = true;
308 }
309 assertTrue(expectedException);
310
311 assertTrue(spiedUponSt.rollback(null, null));
312
313
314 int parentRowCount2 = countRows(this.parent);
315 assertEquals(parentRowCount, parentRowCount2);
316
317
318 assertTrue(!this.fs.exists(HRegion.getRegionDir(this.testdir, st.getFirstDaughter())));
319 assertTrue(!this.fs.exists(HRegion.getRegionDir(this.testdir, st.getSecondDaughter())));
320 assertTrue(!this.parent.lock.writeLock().isHeldByCurrentThread());
321
322
323 assertTrue(st.prepare());
324 PairOfSameType<HRegion> daughters = st.execute(mockServer, null);
325
326 int daughtersRowCount = 0;
327 for (HRegion openRegion: daughters) {
328 try {
329 int count = countRows(openRegion);
330 assertTrue(count > 0 && count != rowcount);
331 daughtersRowCount += count;
332 } finally {
333 HRegion.closeHRegion(openRegion);
334 }
335 }
336 assertEquals(rowcount, daughtersRowCount);
337
338 assertTrue(!this.parent.lock.writeLock().isHeldByCurrentThread());
339 assertTrue("Rollback hooks should be called.", wasRollBackHookCalled());
340 }
341
342 private boolean wasRollBackHookCalled(){
343 return (preRollBackCalled && postRollBackCalled);
344 }
345
346
347
348
349 @SuppressWarnings("serial")
350 private class MockedFailedDaughterCreation extends IOException {}
351 private class MockedFailedDaughterOpen extends IOException {}
352
353 private int countRows(final HRegion r) throws IOException {
354 int rowcount = 0;
355 InternalScanner scanner = r.getScanner(new Scan());
356 try {
357 List<Cell> kvs = new ArrayList<Cell>();
358 boolean hasNext = true;
359 while (hasNext) {
360 hasNext = scanner.next(kvs);
361 if (!kvs.isEmpty()) rowcount++;
362 }
363 } finally {
364 scanner.close();
365 }
366 return rowcount;
367 }
368
369 HRegion createRegion(final Path testdir, final WALFactory wals)
370 throws IOException {
371
372
373 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("table"));
374 HColumnDescriptor hcd = new HColumnDescriptor(CF);
375 htd.addFamily(hcd);
376 HRegionInfo hri = new HRegionInfo(htd.getTableName(), STARTROW, ENDROW);
377 HRegion r = HRegion.createHRegion(hri, testdir, TEST_UTIL.getConfiguration(), htd);
378 HRegion.closeHRegion(r);
379 return HRegion.openHRegion(testdir, hri, htd, wals.getWAL(hri.getEncodedNameAsBytes()),
380 TEST_UTIL.getConfiguration());
381 }
382
383 public static class CustomObserver extends BaseRegionObserver{
384 @Override
385 public void preRollBackSplit(
386 ObserverContext<RegionCoprocessorEnvironment> ctx) throws IOException {
387 preRollBackCalled = true;
388 }
389
390 @Override
391 public void postRollBackSplit(
392 ObserverContext<RegionCoprocessorEnvironment> ctx) throws IOException {
393 postRollBackCalled = true;
394 }
395 }
396
397 }
398