1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.regionserver.wal;
19
20 import java.io.FileNotFoundException;
21 import java.io.IOException;
22 import java.util.concurrent.atomic.AtomicLong;
23
24 import org.apache.hadoop.hbase.client.Table;
25 import org.junit.Assert;
26 import static org.junit.Assert.assertTrue;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.conf.Configuration;
31 import org.apache.hadoop.fs.FileSystem;
32 import org.apache.hadoop.fs.Path;
33 import org.apache.hadoop.hbase.HBaseTestingUtility;
34 import org.apache.hadoop.hbase.HColumnDescriptor;
35 import org.apache.hadoop.hbase.HConstants;
36 import org.apache.hadoop.hbase.HRegionInfo;
37 import org.apache.hadoop.hbase.HTableDescriptor;
38 import org.apache.hadoop.hbase.KeyValue;
39 import org.apache.hadoop.hbase.testclassification.MediumTests;
40 import org.apache.hadoop.hbase.MiniHBaseCluster;
41 import org.apache.hadoop.hbase.TableName;
42 import org.apache.hadoop.hbase.client.Admin;
43 import org.apache.hadoop.hbase.client.HTable;
44 import org.apache.hadoop.hbase.client.Put;
45 import org.apache.hadoop.hbase.regionserver.HRegionServer;
46 import org.apache.hadoop.hbase.util.Bytes;
47 import org.apache.hadoop.hbase.util.FSUtils;
48 import org.apache.hadoop.hbase.wal.DefaultWALProvider;
49 import org.apache.hadoop.hbase.wal.WAL;
50 import org.apache.hadoop.hbase.wal.WALFactory;
51 import org.apache.hadoop.hbase.wal.WALKey;
52 import org.apache.hadoop.hbase.wal.WALSplitter;
53 import org.apache.hadoop.hdfs.MiniDFSCluster;
54 import org.junit.After;
55 import org.junit.Before;
56 import org.junit.BeforeClass;
57 import org.junit.Test;
58 import org.junit.experimental.categories.Category;
59
60
61
62
63
64 @Category(MediumTests.class)
65 public class TestLogRollAbort {
66 private static final Log LOG = LogFactory.getLog(TestLogRolling.class);
67 private static MiniDFSCluster dfsCluster;
68 private static Admin admin;
69 private static MiniHBaseCluster cluster;
70 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
71
72
73 private static final Path HBASEDIR = new Path("/hbase");
74 private static final Path OLDLOGDIR = new Path(HBASEDIR, HConstants.HREGION_OLDLOGDIR_NAME);
75
76
77
78 @BeforeClass
79 public static void setUpBeforeClass() throws Exception {
80
81 TEST_UTIL.getConfiguration().setInt(
82 "hbase.regionserver.logroll.errors.tolerated", 2);
83 TEST_UTIL.getConfiguration().setInt("hbase.rpc.timeout", 10 * 1000);
84
85
86 TEST_UTIL.getConfiguration().setLong("hbase.client.pause", 5 * 1000);
87
88
89 TEST_UTIL.getConfiguration().setBoolean("dfs.support.append", true);
90
91
92 TEST_UTIL.getConfiguration().setInt("dfs.namenode.heartbeat.recheck-interval", 5000);
93 TEST_UTIL.getConfiguration().setInt("dfs.heartbeat.interval", 1);
94
95
96 TEST_UTIL.getConfiguration().setInt("dfs.client.block.write.retries", 10);
97 }
98
99 private Configuration conf;
100 private FileSystem fs;
101
102 @Before
103 public void setUp() throws Exception {
104 TEST_UTIL.startMiniCluster(2);
105
106 cluster = TEST_UTIL.getHBaseCluster();
107 dfsCluster = TEST_UTIL.getDFSCluster();
108 admin = TEST_UTIL.getHBaseAdmin();
109 conf = TEST_UTIL.getConfiguration();
110 fs = TEST_UTIL.getDFSCluster().getFileSystem();
111
112
113 cluster.getMaster().balanceSwitch(false);
114 FSUtils.setRootDir(conf, HBASEDIR);
115 }
116
117 @After
118 public void tearDown() throws Exception {
119 TEST_UTIL.shutdownMiniCluster();
120 }
121
122
123
124
125
126 @Test
127 public void testRSAbortWithUnflushedEdits() throws Exception {
128 LOG.info("Starting testRSAbortWithUnflushedEdits()");
129
130
131 new HTable(TEST_UTIL.getConfiguration(),
132 TableName.META_TABLE_NAME).close();
133
134
135 TableName tableName = TableName.valueOf(this.getClass().getSimpleName());
136 HTableDescriptor desc = new HTableDescriptor(tableName);
137 desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
138
139 admin.createTable(desc);
140 Table table = new HTable(TEST_UTIL.getConfiguration(), desc.getTableName());
141 try {
142
143 HRegionServer server = TEST_UTIL.getRSForFirstRegionInTable(tableName);
144 WAL log = server.getWAL(null);
145
146
147 assertTrue("Need append support for this test",
148 FSUtils.isAppendSupported(TEST_UTIL.getConfiguration()));
149
150 Put p = new Put(Bytes.toBytes("row2001"));
151 p.add(HConstants.CATALOG_FAMILY, Bytes.toBytes("col"), Bytes.toBytes(2001));
152 table.put(p);
153
154 log.sync();
155
156 p = new Put(Bytes.toBytes("row2002"));
157 p.add(HConstants.CATALOG_FAMILY, Bytes.toBytes("col"), Bytes.toBytes(2002));
158 table.put(p);
159
160 dfsCluster.restartDataNodes();
161 LOG.info("Restarted datanodes");
162
163 try {
164 log.rollWriter(true);
165 } catch (FailedLogCloseException flce) {
166
167
168
169 } catch (Throwable t) {
170 LOG.fatal("FAILED TEST: Got wrong exception", t);
171 }
172 } finally {
173 table.close();
174 }
175 }
176
177
178
179
180
181
182 @Test (timeout=300000)
183 public void testLogRollAfterSplitStart() throws IOException {
184 LOG.info("Verify wal roll after split starts will fail.");
185 String logName = "testLogRollAfterSplitStart";
186 Path thisTestsDir = new Path(HBASEDIR, DefaultWALProvider.getWALDirectoryName(logName));
187 final WALFactory wals = new WALFactory(conf, null, logName);
188
189 try {
190
191 TableName tableName =
192 TableName.valueOf(this.getClass().getName());
193 HRegionInfo regioninfo = new HRegionInfo(tableName,
194 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
195 final WAL log = wals.getWAL(regioninfo.getEncodedNameAsBytes());
196
197 final AtomicLong sequenceId = new AtomicLong(1);
198
199 final int total = 20;
200 for (int i = 0; i < total; i++) {
201 WALEdit kvs = new WALEdit();
202 kvs.add(new KeyValue(Bytes.toBytes(i), tableName.getName(), tableName.getName()));
203 HTableDescriptor htd = new HTableDescriptor(tableName);
204 htd.addFamily(new HColumnDescriptor("column"));
205 log.append(htd, regioninfo, new WALKey(regioninfo.getEncodedNameAsBytes(), tableName,
206 System.currentTimeMillis()), kvs, sequenceId, true, null);
207 }
208
209 log.sync();
210 ((FSHLog) log).replaceWriter(((FSHLog)log).getOldPath(), null, null, null);
211
212
213
214
215
216 Path rsSplitDir = thisTestsDir.suffix(DefaultWALProvider.SPLITTING_EXT);
217 if (!fs.rename(thisTestsDir, rsSplitDir)) {
218 throw new IOException("Failed fs.rename for log split: " + thisTestsDir);
219 }
220 LOG.debug("Renamed region directory: " + rsSplitDir);
221
222 LOG.debug("Processing the old log files.");
223 WALSplitter.split(HBASEDIR, rsSplitDir, OLDLOGDIR, fs, conf, wals);
224
225 LOG.debug("Trying to roll the WAL.");
226 try {
227 log.rollWriter();
228 Assert.fail("rollWriter() did not throw any exception.");
229 } catch (IOException ioe) {
230 if (ioe.getCause() instanceof FileNotFoundException) {
231 LOG.info("Got the expected exception: ", ioe.getCause());
232 } else {
233 Assert.fail("Unexpected exception: " + ioe);
234 }
235 }
236 } finally {
237 wals.close();
238 if (fs.exists(thisTestsDir)) {
239 fs.delete(thisTestsDir, true);
240 }
241 }
242 }
243 }