1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.replication.regionserver;
20
21 import org.apache.hadoop.conf.Configuration;
22 import org.apache.hadoop.fs.FileSystem;
23 import org.apache.hadoop.fs.Path;
24 import org.apache.hadoop.hbase.HBaseTestingUtility;
25 import org.apache.hadoop.hbase.HConstants;
26 import org.apache.hadoop.hbase.HRegionInfo;
27 import org.apache.hadoop.hbase.HTableDescriptor;
28 import org.apache.hadoop.hbase.KeyValue;
29 import org.apache.hadoop.hbase.LargeTests;
30 import org.apache.hadoop.hbase.TableName;
31 import org.apache.hadoop.hbase.regionserver.wal.HLog;
32 import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
33 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
34 import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
35 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
36 import org.apache.hadoop.hbase.util.Bytes;
37 import org.apache.hadoop.hdfs.MiniDFSCluster;
38 import org.junit.After;
39 import org.junit.AfterClass;
40 import org.junit.Before;
41 import org.junit.BeforeClass;
42 import org.junit.Test;
43 import org.junit.experimental.categories.Category;
44 import org.junit.runner.RunWith;
45 import org.junit.runners.Parameterized;
46 import org.junit.runners.Parameterized.Parameters;
47
48 import static org.junit.Assert.*;
49
50 import java.io.EOFException;
51 import java.io.IOException;
52 import java.util.ArrayList;
53 import java.util.Collection;
54 import java.util.List;
55
56 @Category(LargeTests.class)
57 @RunWith(Parameterized.class)
58 public class TestReplicationHLogReaderManager {
59
60 private static HBaseTestingUtility TEST_UTIL;
61 private static Configuration conf;
62 private static Path hbaseDir;
63 private static FileSystem fs;
64 private static MiniDFSCluster cluster;
65 private static final TableName tableName = TableName.valueOf("tablename");
66 private static final byte [] family = Bytes.toBytes("column");
67 private static final byte [] qualifier = Bytes.toBytes("qualifier");
68 private static final HRegionInfo info = new HRegionInfo(tableName,
69 HConstants.EMPTY_START_ROW, HConstants.LAST_ROW, false);
70 private static final HTableDescriptor htd = new HTableDescriptor(tableName);
71
72 private HLog log;
73 private ReplicationHLogReaderManager logManager;
74 private PathWatcher pathWatcher;
75 private int nbRows;
76 private int walEditKVs;
77
78 @Parameters
79 public static Collection<Object[]> parameters() {
80
81 int[] NB_ROWS = { 1500, 60000 };
82 int[] NB_KVS = { 1, 100 };
83
84 Boolean[] BOOL_VALS = { false, true };
85 List<Object[]> parameters = new ArrayList<Object[]>();
86 for (int nbRows : NB_ROWS) {
87 for (int walEditKVs : NB_KVS) {
88 for (boolean b : BOOL_VALS) {
89 Object[] arr = new Object[3];
90 arr[0] = nbRows;
91 arr[1] = walEditKVs;
92 arr[2] = b;
93 parameters.add(arr);
94 }
95 }
96 }
97 return parameters;
98 }
99
100 public TestReplicationHLogReaderManager(int nbRows, int walEditKVs, boolean enableCompression) {
101 this.nbRows = nbRows;
102 this.walEditKVs = walEditKVs;
103 TEST_UTIL.getConfiguration().setBoolean(HConstants.ENABLE_WAL_COMPRESSION,
104 enableCompression);
105 }
106
107 @BeforeClass
108 public static void setUpBeforeClass() throws Exception {
109 TEST_UTIL = new HBaseTestingUtility();
110 conf = TEST_UTIL.getConfiguration();
111 TEST_UTIL.startMiniDFSCluster(3);
112
113 hbaseDir = TEST_UTIL.createRootDir();
114 cluster = TEST_UTIL.getDFSCluster();
115 fs = cluster.getFileSystem();
116 }
117
118 @AfterClass
119 public static void tearDownAfterClass() throws Exception {
120 TEST_UTIL.shutdownMiniCluster();
121 }
122
123 @Before
124 public void setUp() throws Exception {
125 logManager = new ReplicationHLogReaderManager(fs, conf);
126 List<WALActionsListener> listeners = new ArrayList<WALActionsListener>();
127 pathWatcher = new PathWatcher();
128 listeners.add(pathWatcher);
129 log = HLogFactory.createHLog(fs, hbaseDir, "test", conf, listeners, "some server");
130 }
131
132 @After
133 public void tearDown() throws Exception {
134 log.closeAndDelete();
135 }
136
137 @Test
138 public void test() throws Exception {
139
140 Path path = pathWatcher.currentPath;
141
142 assertEquals(0, logManager.getPosition());
143
144 appendToLog();
145
146
147 assertNotNull(logManager.openReader(path));
148 logManager.seek();
149 HLog.Entry entry = logManager.readNextAndSetPosition();
150 assertNotNull(entry);
151 entry = logManager.readNextAndSetPosition();
152 assertNull(entry);
153 logManager.closeReader();
154 long oldPos = logManager.getPosition();
155
156 appendToLog();
157
158
159 assertNotNull(logManager.openReader(path));
160 logManager.seek();
161 entry = logManager.readNextAndSetPosition();
162 assertNotEquals(oldPos, logManager.getPosition());
163 assertNotNull(entry);
164 logManager.closeReader();
165 oldPos = logManager.getPosition();
166
167 log.rollWriter();
168
169
170 assertNotNull(logManager.openReader(path));
171 logManager.seek();
172 entry = logManager.readNextAndSetPosition();
173 assertEquals(oldPos, logManager.getPosition());
174 assertNull(entry);
175 logManager.finishCurrentFile();
176
177 path = pathWatcher.currentPath;
178
179 for (int i = 0; i < nbRows; i++) { appendToLogPlus(walEditKVs); }
180 log.rollWriter();
181 logManager.openReader(path);
182 logManager.seek();
183 for (int i = 0; i < nbRows; i++) {
184 HLog.Entry e = logManager.readNextAndSetPosition();
185 if (e == null) {
186 fail("Should have enough entries");
187 }
188 }
189 }
190
191 private void appendToLog() throws IOException {
192 appendToLogPlus(1);
193 }
194
195 private void appendToLogPlus(int count) throws IOException {
196 log.append(info, tableName, getWALEdits(count), System.currentTimeMillis(), htd);
197 }
198
199 private WALEdit getWALEdits(int count) {
200 WALEdit edit = new WALEdit();
201 for (int i = 0; i < count; i++) {
202 edit.add(new KeyValue(Bytes.toBytes(System.currentTimeMillis()), family, qualifier,
203 System.currentTimeMillis(), qualifier));
204 }
205 return edit;
206 }
207
208 class PathWatcher implements WALActionsListener {
209
210 Path currentPath;
211
212 @Override
213 public void preLogRoll(Path oldPath, Path newPath) throws IOException {
214 currentPath = newPath;
215 }
216
217 @Override
218 public void postLogRoll(Path oldPath, Path newPath) throws IOException {}
219
220 @Override
221 public void preLogArchive(Path oldPath, Path newPath) throws IOException {}
222
223 @Override
224 public void postLogArchive(Path oldPath, Path newPath) throws IOException {}
225
226 @Override
227 public void logRollRequested() {}
228
229 @Override
230 public void logCloseRequested() {}
231
232 @Override
233 public void visitLogEntryBeforeWrite(HRegionInfo info, HLogKey logKey, WALEdit logEdit) {}
234
235 @Override
236 public void visitLogEntryBeforeWrite(HTableDescriptor htd, HLogKey logKey, WALEdit logEdit) {}
237 }
238 }