1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.regionserver.wal;
19
20 import static org.junit.Assert.assertFalse;
21 import static org.junit.Assert.assertTrue;
22
23 import java.io.IOException;
24 import java.util.concurrent.atomic.AtomicLong;
25
26 import org.apache.commons.io.IOUtils;
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.commons.logging.impl.Log4JLogger;
30 import org.apache.hadoop.conf.Configuration;
31 import org.apache.hadoop.fs.FSDataInputStream;
32 import org.apache.hadoop.fs.FileStatus;
33 import org.apache.hadoop.fs.FileSystem;
34 import org.apache.hadoop.fs.Path;
35 import org.apache.hadoop.hbase.HBaseTestingUtility;
36 import org.apache.hadoop.hbase.HColumnDescriptor;
37 import org.apache.hadoop.hbase.HConstants;
38 import org.apache.hadoop.hbase.HRegionInfo;
39 import org.apache.hadoop.hbase.HTableDescriptor;
40 import org.apache.hadoop.hbase.KeyValue;
41 import org.apache.hadoop.hbase.MediumTests;
42 import org.apache.hadoop.hbase.TableName;
43 import org.apache.hadoop.hbase.io.crypto.KeyProviderForTesting;
44 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
45 import org.apache.hadoop.hbase.regionserver.wal.TestCustomWALCellCodec.CustomWALCellCodec;
46 import org.apache.hadoop.hbase.util.Bytes;
47 import org.apache.hadoop.hbase.util.FSUtils;
48 import org.apache.hadoop.hbase.zookeeper.ZKSplitLog;
49 import org.apache.log4j.Level;
50 import org.junit.BeforeClass;
51 import org.junit.Test;
52 import org.junit.experimental.categories.Category;
53
54
55
56
57 @Category(MediumTests.class)
58 public class TestHLogReaderOnSecureHLog {
59 static final Log LOG = LogFactory.getLog(TestHLogReaderOnSecureHLog.class);
60 static {
61 ((Log4JLogger)LogFactory.getLog("org.apache.hadoop.hbase.regionserver.wal"))
62 .getLogger().setLevel(Level.ALL);
63 };
64 static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
65 final byte[] value = Bytes.toBytes("Test value");
66
67 @BeforeClass
68 public static void setUpBeforeClass() throws Exception {
69 Configuration conf = TEST_UTIL.getConfiguration();
70 conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName());
71 conf.set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase");
72 conf.setBoolean("hbase.hlog.split.skip.errors", true);
73 conf.setBoolean(HConstants.ENABLE_WAL_ENCRYPTION, true);
74 }
75
76 private Path writeWAL(String tblName) throws IOException {
77 Configuration conf = TEST_UTIL.getConfiguration();
78 String clsName = conf.get(WALCellCodec.WAL_CELL_CODEC_CLASS_KEY, WALCellCodec.class.getName());
79 conf.setClass(WALCellCodec.WAL_CELL_CODEC_CLASS_KEY, CustomWALCellCodec.class,
80 WALCellCodec.class);
81 TableName tableName = TableName.valueOf(tblName);
82 HTableDescriptor htd = new HTableDescriptor(tableName);
83 htd.addFamily(new HColumnDescriptor(tableName.getName()));
84 HRegionInfo regioninfo = new HRegionInfo(tableName,
85 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, false);
86 final int total = 10;
87 final byte[] row = Bytes.toBytes("row");
88 final byte[] family = Bytes.toBytes("family");
89 FileSystem fs = TEST_UTIL.getTestFileSystem();
90 Path logDir = TEST_UTIL.getDataTestDir(tblName);
91 final AtomicLong sequenceId = new AtomicLong(1);
92
93
94 FSHLog wal = new FSHLog(fs, TEST_UTIL.getDataTestDir(), logDir.toString(), conf);
95 for (int i = 0; i < total; i++) {
96 WALEdit kvs = new WALEdit();
97 kvs.add(new KeyValue(row, family, Bytes.toBytes(i), value));
98 wal.append(regioninfo, tableName, kvs, System.currentTimeMillis(), htd, sequenceId);
99 }
100 final Path walPath = ((FSHLog) wal).computeFilename();
101 wal.close();
102
103 conf.set(WALCellCodec.WAL_CELL_CODEC_CLASS_KEY, clsName);
104
105 return walPath;
106 }
107
108 @Test()
109 public void testHLogReaderOnSecureHLog() throws Exception {
110 Configuration conf = TEST_UTIL.getConfiguration();
111 HLogFactory.resetLogReaderClass();
112 HLogFactory.resetLogWriterClass();
113 conf.setClass("hbase.regionserver.hlog.reader.impl", ProtobufLogReader.class,
114 HLog.Reader.class);
115 conf.setClass("hbase.regionserver.hlog.writer.impl", SecureProtobufLogWriter.class,
116 HLog.Writer.class);
117 FileSystem fs = TEST_UTIL.getTestFileSystem();
118 Path walPath = writeWAL("testHLogReaderOnSecureHLog");
119
120
121 long length = fs.getFileStatus(walPath).getLen();
122 FSDataInputStream in = fs.open(walPath);
123 byte[] fileData = new byte[(int)length];
124 IOUtils.readFully(in, fileData);
125 in.close();
126 assertFalse("Cells appear to be plaintext", Bytes.contains(fileData, value));
127
128
129 try {
130 HLog.Reader reader = HLogFactory.createReader(TEST_UTIL.getTestFileSystem(), walPath, conf);
131 assertFalse(true);
132 } catch (IOException ioe) {
133
134 }
135
136 FileStatus[] listStatus = fs.listStatus(walPath.getParent());
137 RecoveryMode mode = (conf.getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false) ?
138 RecoveryMode.LOG_REPLAY : RecoveryMode.LOG_SPLITTING);
139 Path rootdir = FSUtils.getRootDir(conf);
140 try {
141 HLogSplitter s = new HLogSplitter(conf, rootdir, fs, null, null, mode);
142 s.splitLogFile(listStatus[0], null);
143 Path file = new Path(ZKSplitLog.getSplitLogDir(rootdir, listStatus[0].getPath().getName()),
144 "corrupt");
145 assertTrue(fs.exists(file));
146
147 } catch (IOException ioe) {
148 assertTrue("WAL should have been sidelined", false);
149 }
150 }
151
152 @Test()
153 public void testSecureHLogReaderOnHLog() throws Exception {
154 Configuration conf = TEST_UTIL.getConfiguration();
155 HLogFactory.resetLogReaderClass();
156 HLogFactory.resetLogWriterClass();
157 conf.setClass("hbase.regionserver.hlog.reader.impl", SecureProtobufLogReader.class,
158 HLog.Reader.class);
159 conf.setClass("hbase.regionserver.hlog.writer.impl", ProtobufLogWriter.class,
160 HLog.Writer.class);
161 FileSystem fs = TEST_UTIL.getTestFileSystem();
162 Path walPath = writeWAL("testSecureHLogReaderOnHLog");
163
164
165 long length = fs.getFileStatus(walPath).getLen();
166 FSDataInputStream in = fs.open(walPath);
167 byte[] fileData = new byte[(int)length];
168 IOUtils.readFully(in, fileData);
169 in.close();
170 assertTrue("Cells should be plaintext", Bytes.contains(fileData, value));
171
172
173 try {
174 HLog.Reader reader = HLogFactory.createReader(TEST_UTIL.getTestFileSystem(), walPath, conf);
175 } catch (IOException ioe) {
176 assertFalse(true);
177 }
178
179 FileStatus[] listStatus = fs.listStatus(walPath.getParent());
180 RecoveryMode mode = (conf.getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false) ?
181 RecoveryMode.LOG_REPLAY : RecoveryMode.LOG_SPLITTING);
182 Path rootdir = FSUtils.getRootDir(conf);
183 try {
184 HLogSplitter s = new HLogSplitter(conf, rootdir, fs, null, null, mode);
185 s.splitLogFile(listStatus[0], null);
186 Path file = new Path(ZKSplitLog.getSplitLogDir(rootdir, listStatus[0].getPath().getName()),
187 "corrupt");
188 assertTrue(!fs.exists(file));
189 } catch (IOException ioe) {
190 assertTrue("WAL should have been processed", false);
191 }
192 }
193 }