1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.replication.regionserver;
20
21 import org.apache.hadoop.conf.Configuration;
22 import org.apache.hadoop.fs.FileSystem;
23 import org.apache.hadoop.fs.Path;
24 import org.apache.hadoop.hbase.HBaseTestingUtility;
25 import org.apache.hadoop.hbase.HConstants;
26 import org.apache.hadoop.hbase.HRegionInfo;
27 import org.apache.hadoop.hbase.HTableDescriptor;
28 import org.apache.hadoop.hbase.KeyValue;
29 import org.apache.hadoop.hbase.LargeTests;
30 import org.apache.hadoop.hbase.TableName;
31 import org.apache.hadoop.hbase.regionserver.wal.HLog;
32 import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
33 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
34 import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
35 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
36 import org.apache.hadoop.hbase.util.Bytes;
37 import org.apache.hadoop.hdfs.MiniDFSCluster;
38 import org.junit.After;
39 import org.junit.AfterClass;
40 import org.junit.Before;
41 import org.junit.BeforeClass;
42 import org.junit.Test;
43 import org.junit.experimental.categories.Category;
44 import org.junit.runner.RunWith;
45 import org.junit.runners.Parameterized;
46 import org.junit.runners.Parameterized.Parameters;
47
48 import static org.junit.Assert.*;
49
50 import java.io.EOFException;
51 import java.io.IOException;
52 import java.util.ArrayList;
53 import java.util.Collection;
54 import java.util.List;
55
56 @Category(LargeTests.class)
57 @RunWith(Parameterized.class)
58 public class TestReplicationHLogReaderManager {
59
60 private static HBaseTestingUtility TEST_UTIL;
61 private static Configuration conf;
62 private static Path hbaseDir;
63 private static FileSystem fs;
64 private static MiniDFSCluster cluster;
65 private static final TableName tableName = TableName.valueOf("tablename");
66 private static final byte [] family = Bytes.toBytes("column");
67 private static final byte [] qualifier = Bytes.toBytes("qualifier");
68 private static final HRegionInfo info = new HRegionInfo(tableName,
69 HConstants.EMPTY_START_ROW, HConstants.LAST_ROW, false);
70 private static final HTableDescriptor htd = new HTableDescriptor(tableName);
71
72 private HLog log;
73 private ReplicationHLogReaderManager logManager;
74 private PathWatcher pathWatcher;
75 private int nbRows;
76 private int walEditKVs;
77
78 @Parameters
79 public static Collection<Object[]> parameters() {
80
81 int[] NB_ROWS = { 1500, 60000 };
82 int[] NB_KVS = { 1, 100 };
83
84 Boolean[] BOOL_VALS = { false, true };
85 List<Object[]> parameters = new ArrayList<Object[]>();
86 for (int nbRows : NB_ROWS) {
87 for (int walEditKVs : NB_KVS) {
88 for (boolean b : BOOL_VALS) {
89 Object[] arr = new Object[3];
90 arr[0] = nbRows;
91 arr[1] = walEditKVs;
92 arr[2] = b;
93 parameters.add(arr);
94 }
95 }
96 }
97 return parameters;
98 }
99
100 public TestReplicationHLogReaderManager(int nbRows, int walEditKVs, boolean enableCompression) {
101 this.nbRows = nbRows;
102 this.walEditKVs = walEditKVs;
103 TEST_UTIL.getConfiguration().setBoolean(HConstants.ENABLE_WAL_COMPRESSION,
104 enableCompression);
105 }
106
107 @BeforeClass
108 public static void setUpBeforeClass() throws Exception {
109 TEST_UTIL = new HBaseTestingUtility();
110 conf = TEST_UTIL.getConfiguration();
111 TEST_UTIL.startMiniDFSCluster(3);
112
113 hbaseDir = TEST_UTIL.createRootDir();
114 cluster = TEST_UTIL.getDFSCluster();
115 fs = cluster.getFileSystem();
116 }
117
118 @AfterClass
119 public static void tearDownAfterClass() throws Exception {
120 TEST_UTIL.shutdownMiniCluster();
121 }
122
123 @Before
124 public void setUp() throws Exception {
125 logManager = new ReplicationHLogReaderManager(fs, conf);
126 List<WALActionsListener> listeners = new ArrayList<WALActionsListener>();
127 pathWatcher = new PathWatcher();
128 listeners.add(pathWatcher);
129 log = HLogFactory.createHLog(fs, hbaseDir, "test", conf, listeners, "some server");
130 }
131
132 @After
133 public void tearDown() throws Exception {
134 log.closeAndDelete();
135 }
136
137 @Test
138 public void test() throws Exception {
139
140 Path path = pathWatcher.currentPath;
141
142
143 try {
144 logManager.openReader(path);
145 fail("Shouldn't be able to open an empty file");
146 } catch (EOFException ex) {}
147
148 assertEquals(0, logManager.getPosition());
149
150 appendToLog();
151
152
153 assertNotNull(logManager.openReader(path));
154 logManager.seek();
155 HLog.Entry[] entriesArray = new HLog.Entry[1];
156 HLog.Entry entry = logManager.readNextAndSetPosition(entriesArray, 0);
157 assertNotNull(entry);
158 entry = logManager.readNextAndSetPosition(entriesArray, 0);
159 assertNull(entry);
160 logManager.closeReader();
161 long oldPos = logManager.getPosition();
162
163 appendToLog();
164
165
166 assertNotNull(logManager.openReader(path));
167 logManager.seek();
168 entry = logManager.readNextAndSetPosition(entriesArray, 0);
169 assertNotEquals(oldPos, logManager.getPosition());
170 assertNotNull(entry);
171 logManager.closeReader();
172 oldPos = logManager.getPosition();
173
174 log.rollWriter();
175
176
177 assertNotNull(logManager.openReader(path));
178 logManager.seek();
179 entry = logManager.readNextAndSetPosition(entriesArray, 0);
180 assertEquals(oldPos, logManager.getPosition());
181 assertNull(entry);
182 logManager.finishCurrentFile();
183
184 path = pathWatcher.currentPath;
185
186
187 try {
188 logManager.openReader(path);
189 fail();
190 } catch (EOFException ex) {}
191
192 for (int i = 0; i < nbRows; i++) { appendToLogPlus(walEditKVs); }
193 log.rollWriter();
194 logManager.openReader(path);
195 logManager.seek();
196 for (int i = 0; i < nbRows; i++) {
197 HLog.Entry e = logManager.readNextAndSetPosition(entriesArray, 0);
198 if (e == null) {
199 fail("Should have enough entries");
200 }
201 }
202 }
203
204 private void appendToLog() throws IOException {
205 appendToLogPlus(1);
206 }
207
208 private void appendToLogPlus(int count) throws IOException {
209 log.append(info, tableName, getWALEdits(count), System.currentTimeMillis(), htd);
210 }
211
212 private WALEdit getWALEdits(int count) {
213 WALEdit edit = new WALEdit();
214 for (int i = 0; i < count; i++) {
215 edit.add(new KeyValue(Bytes.toBytes(System.currentTimeMillis()), family, qualifier,
216 System.currentTimeMillis(), qualifier));
217 }
218 return edit;
219 }
220
221 class PathWatcher implements WALActionsListener {
222
223 Path currentPath;
224
225 @Override
226 public void preLogRoll(Path oldPath, Path newPath) throws IOException {
227 currentPath = newPath;
228 }
229
230 @Override
231 public void postLogRoll(Path oldPath, Path newPath) throws IOException {}
232
233 @Override
234 public void preLogArchive(Path oldPath, Path newPath) throws IOException {}
235
236 @Override
237 public void postLogArchive(Path oldPath, Path newPath) throws IOException {}
238
239 @Override
240 public void logRollRequested() {}
241
242 @Override
243 public void logCloseRequested() {}
244
245 @Override
246 public void visitLogEntryBeforeWrite(HRegionInfo info, HLogKey logKey, WALEdit logEdit) {}
247
248 @Override
249 public void visitLogEntryBeforeWrite(HTableDescriptor htd, HLogKey logKey, WALEdit logEdit) {}
250 }
251 }