1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertTrue;
23 import static org.mockito.Mockito.mock;
24 import static org.mockito.Mockito.when;
25
26 import java.io.IOException;
27 import java.util.ArrayList;
28 import java.util.Collection;
29 import java.util.List;
30
31 import org.apache.hadoop.conf.Configuration;
32 import org.apache.hadoop.fs.FileSystem;
33 import org.apache.hadoop.fs.Path;
34 import org.apache.hadoop.hbase.Cell;
35 import org.apache.hadoop.hbase.CellUtil;
36 import org.apache.hadoop.hbase.HBaseTestingUtility;
37 import org.apache.hadoop.hbase.HColumnDescriptor;
38 import org.apache.hadoop.hbase.HConstants;
39 import org.apache.hadoop.hbase.HRegionInfo;
40 import org.apache.hadoop.hbase.HTableDescriptor;
41 import org.apache.hadoop.hbase.testclassification.SmallTests;
42 import org.apache.hadoop.hbase.Stoppable;
43 import org.apache.hadoop.hbase.TableName;
44 import org.apache.hadoop.hbase.client.Durability;
45 import org.apache.hadoop.hbase.client.Get;
46 import org.apache.hadoop.hbase.client.Put;
47 import org.apache.hadoop.hbase.client.Result;
48 import org.apache.hadoop.hbase.wal.WALFactory;
49 import org.apache.hadoop.hbase.util.Bytes;
50 import org.apache.hadoop.hbase.util.FSUtils;
51 import org.apache.hadoop.hbase.util.StoppableImplementation;
52 import org.junit.Assert;
53 import org.junit.Before;
54 import org.junit.Test;
55 import org.junit.experimental.categories.Category;
56
57 @Category(SmallTests.class)
58 public class TestStoreFileRefresherChore {
59
60 private HBaseTestingUtility TEST_UTIL;
61 private Path testDir;
62
63 @Before
64 public void setUp() throws IOException {
65 TEST_UTIL = new HBaseTestingUtility();
66 testDir = TEST_UTIL.getDataTestDir("TestStoreFileRefresherChore");
67 FSUtils.setRootDir(TEST_UTIL.getConfiguration(), testDir);
68 }
69
70 private HTableDescriptor getTableDesc(TableName tableName, byte[]... families) {
71 HTableDescriptor htd = new HTableDescriptor(tableName);
72 for (byte[] family : families) {
73 HColumnDescriptor hcd = new HColumnDescriptor(family);
74
75 hcd.setMaxVersions(Integer.MAX_VALUE);
76 htd.addFamily(hcd);
77 }
78 return htd;
79 }
80
81 static class FailingHRegionFileSystem extends HRegionFileSystem {
82 boolean fail = false;
83 FailingHRegionFileSystem(Configuration conf, FileSystem fs, Path tableDir, HRegionInfo regionInfo) {
84 super(conf, fs, tableDir, regionInfo);
85 }
86
87 @Override
88 public Collection<StoreFileInfo> getStoreFiles(String familyName) throws IOException {
89 if (fail) {
90 throw new IOException("simulating FS failure");
91 }
92 return super.getStoreFiles(familyName);
93 }
94 }
95
96 private HRegion initHRegion(HTableDescriptor htd, byte[] startKey, byte[] stopKey, int replicaId) throws IOException {
97 Configuration conf = TEST_UTIL.getConfiguration();
98 Path tableDir = FSUtils.getTableDir(testDir, htd.getTableName());
99
100 HRegionInfo info = new HRegionInfo(htd.getTableName(), startKey, stopKey, false, 0, replicaId);
101
102 HRegionFileSystem fs = new FailingHRegionFileSystem(conf, tableDir.getFileSystem(conf), tableDir, info);
103 final Configuration walConf = new Configuration(conf);
104 FSUtils.setRootDir(walConf, tableDir);
105 final WALFactory wals = new WALFactory(walConf, null, "log_" + replicaId);
106 HRegion region = new HRegion(fs, wals.getWAL(info.getEncodedNameAsBytes()), conf, htd, null);
107
108 region.initialize();
109
110 return region;
111 }
112
113 private void putData(HRegion region, int startRow, int numRows, byte[] qf, byte[]... families) throws IOException {
114 for (int i = startRow; i < startRow + numRows; i++) {
115 Put put = new Put(Bytes.toBytes("" + i));
116 put.setDurability(Durability.SKIP_WAL);
117 for (byte[] family : families) {
118 put.add(family, qf, null);
119 }
120 region.put(put);
121 }
122 }
123
124 private void verifyData(HRegion newReg, int startRow, int numRows, byte[] qf, byte[]... families)
125 throws IOException {
126 for (int i = startRow; i < startRow + numRows; i++) {
127 byte[] row = Bytes.toBytes("" + i);
128 Get get = new Get(row);
129 for (byte[] family : families) {
130 get.addColumn(family, qf);
131 }
132 Result result = newReg.get(get);
133 Cell[] raw = result.rawCells();
134 assertEquals(families.length, result.size());
135 for (int j = 0; j < families.length; j++) {
136 assertTrue(CellUtil.matchingRow(raw[j], row));
137 assertTrue(CellUtil.matchingFamily(raw[j], families[j]));
138 assertTrue(CellUtil.matchingQualifier(raw[j], qf));
139 }
140 }
141 }
142
143 static class StaleStorefileRefresherChore extends StorefileRefresherChore {
144 boolean isStale = false;
145 public StaleStorefileRefresherChore(int period, HRegionServer regionServer,
146 Stoppable stoppable) {
147 super(period, regionServer, stoppable);
148 }
149 @Override
150 protected boolean isRegionStale(String encodedName, long time) {
151 return isStale;
152 }
153 }
154
155 @Test (timeout = 60000)
156 public void testIsStale() throws IOException {
157 int period = 0;
158 byte[][] families = new byte[][] {Bytes.toBytes("cf")};
159 byte[] qf = Bytes.toBytes("cq");
160
161 HRegionServer regionServer = mock(HRegionServer.class);
162 List<HRegion> regions = new ArrayList<HRegion>();
163 when(regionServer.getOnlineRegionsLocalContext()).thenReturn(regions);
164 when(regionServer.getConfiguration()).thenReturn(TEST_UTIL.getConfiguration());
165
166 HTableDescriptor htd = getTableDesc(TableName.valueOf("testIsStale"), families);
167 HRegion primary = initHRegion(htd, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, 0);
168 HRegion replica1 = initHRegion(htd, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, 1);
169 regions.add(primary);
170 regions.add(replica1);
171
172 StaleStorefileRefresherChore chore = new StaleStorefileRefresherChore(period, regionServer, new StoppableImplementation());
173
174
175 putData(primary, 0, 100, qf, families);
176 primary.flushcache();
177 verifyData(primary, 0, 100, qf, families);
178
179 try {
180 verifyData(replica1, 0, 100, qf, families);
181 Assert.fail("should have failed");
182 } catch(AssertionError ex) {
183
184 }
185 chore.chore();
186 verifyData(replica1, 0, 100, qf, families);
187
188
189 ((FailingHRegionFileSystem)replica1.getRegionFileSystem()).fail = true;
190
191
192 putData(primary, 100, 100, qf, families);
193 primary.flushcache();
194 verifyData(primary, 0, 200, qf, families);
195
196 chore.chore();
197
198 verifyData(replica1, 0, 100, qf, families);
199 try {
200 verifyData(replica1, 100, 100, qf, families);
201 Assert.fail("should have failed");
202 } catch(AssertionError ex) {
203
204 }
205
206 chore.isStale = true;
207 chore.chore();
208 try {
209 verifyData(replica1, 0, 100, qf, families);
210 Assert.fail("should have failed with IOException");
211 } catch(IOException ex) {
212
213 }
214 }
215 }