1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.mapreduce;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertTrue;
22 import static org.junit.Assert.fail;
23 import static org.mockito.Matchers.any;
24 import static org.mockito.Mockito.doAnswer;
25 import static org.mockito.Mockito.mock;
26 import static org.mockito.Mockito.when;
27
28 import java.io.ByteArrayOutputStream;
29 import java.io.PrintStream;
30 import java.util.ArrayList;
31
32 import org.apache.hadoop.conf.Configuration;
33 import org.apache.hadoop.fs.Path;
34 import org.apache.hadoop.hbase.Cell;
35 import org.apache.hadoop.hbase.CellUtil;
36 import org.apache.hadoop.hbase.HBaseTestingUtility;
37 import org.apache.hadoop.hbase.HConstants;
38 import org.apache.hadoop.hbase.KeyValue;
39 import org.apache.hadoop.hbase.testclassification.LargeTests;
40 import org.apache.hadoop.hbase.MiniHBaseCluster;
41 import org.apache.hadoop.hbase.TableName;
42 import org.apache.hadoop.hbase.client.Delete;
43 import org.apache.hadoop.hbase.client.Get;
44 import org.apache.hadoop.hbase.client.Put;
45 import org.apache.hadoop.hbase.client.Result;
46 import org.apache.hadoop.hbase.client.Table;
47 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
48 import org.apache.hadoop.hbase.mapreduce.WALPlayer.WALKeyValueMapper;
49 import org.apache.hadoop.hbase.wal.WAL;
50 import org.apache.hadoop.hbase.wal.WALKey;
51 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
52 import org.apache.hadoop.hbase.util.Bytes;
53 import org.apache.hadoop.hbase.util.LauncherSecurityManager;
54 import org.apache.hadoop.mapreduce.Mapper;
55 import org.apache.hadoop.mapreduce.Mapper.Context;
56 import org.junit.AfterClass;
57 import org.junit.BeforeClass;
58 import org.junit.Test;
59 import org.junit.experimental.categories.Category;
60 import org.mockito.invocation.InvocationOnMock;
61 import org.mockito.stubbing.Answer;
62
63
64
65
66 @Category(LargeTests.class)
67 public class TestWALPlayer {
68 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
69 private static MiniHBaseCluster cluster;
70
71 @BeforeClass
72 public static void beforeClass() throws Exception {
73 cluster = TEST_UTIL.startMiniCluster();
74 TEST_UTIL.startMiniMapReduceCluster();
75 }
76
77 @AfterClass
78 public static void afterClass() throws Exception {
79 TEST_UTIL.shutdownMiniMapReduceCluster();
80 TEST_UTIL.shutdownMiniCluster();
81 }
82
83
84
85
86
87 @Test
88 public void testWALPlayer() throws Exception {
89 final TableName TABLENAME1 = TableName.valueOf("testWALPlayer1");
90 final TableName TABLENAME2 = TableName.valueOf("testWALPlayer2");
91 final byte[] FAMILY = Bytes.toBytes("family");
92 final byte[] COLUMN1 = Bytes.toBytes("c1");
93 final byte[] COLUMN2 = Bytes.toBytes("c2");
94 final byte[] ROW = Bytes.toBytes("row");
95 Table t1 = TEST_UTIL.createTable(TABLENAME1, FAMILY);
96 Table t2 = TEST_UTIL.createTable(TABLENAME2, FAMILY);
97
98
99 Put p = new Put(ROW);
100 p.add(FAMILY, COLUMN1, COLUMN1);
101 p.add(FAMILY, COLUMN2, COLUMN2);
102 t1.put(p);
103
104 Delete d = new Delete(ROW);
105 d.deleteColumns(FAMILY, COLUMN1);
106 t1.delete(d);
107
108
109 WAL log = cluster.getRegionServer(0).getWAL(null);
110 log.rollWriter();
111 String walInputDir = new Path(cluster.getMaster().getMasterFileSystem()
112 .getRootDir(), HConstants.HREGION_LOGDIR_NAME).toString();
113
114 Configuration configuration= TEST_UTIL.getConfiguration();
115 WALPlayer player = new WALPlayer(configuration);
116 String optionName="_test_.name";
117 configuration.set(optionName, "1000");
118 player.setupTime(configuration, optionName);
119 assertEquals(1000,configuration.getLong(optionName,0));
120 assertEquals(0, player.run(new String[] {walInputDir, TABLENAME1.getNameAsString(),
121 TABLENAME2.getNameAsString() }));
122
123
124
125 Get g = new Get(ROW);
126 Result r = t2.get(g);
127 assertEquals(1, r.size());
128 assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN2));
129 }
130
131
132
133
134 @Test
135 public void testWALKeyValueMapper() throws Exception {
136 testWALKeyValueMapper(WALPlayer.TABLES_KEY);
137 }
138
139 @Test
140 public void testWALKeyValueMapperWithDeprecatedConfig() throws Exception {
141 testWALKeyValueMapper("hlog.input.tables");
142 }
143
144 private void testWALKeyValueMapper(final String tableConfigKey) throws Exception {
145 Configuration configuration = new Configuration();
146 configuration.set(tableConfigKey, "table");
147 WALKeyValueMapper mapper = new WALKeyValueMapper();
148 WALKey key = mock(WALKey.class);
149 when(key.getTablename()).thenReturn(TableName.valueOf("table"));
150 @SuppressWarnings("unchecked")
151 Mapper<WALKey, WALEdit, ImmutableBytesWritable, KeyValue>.Context context =
152 mock(Context.class);
153 when(context.getConfiguration()).thenReturn(configuration);
154
155 WALEdit value = mock(WALEdit.class);
156 ArrayList<Cell> values = new ArrayList<Cell>();
157 KeyValue kv1 = mock(KeyValue.class);
158 when(kv1.getFamily()).thenReturn(Bytes.toBytes("family"));
159 when(kv1.getRow()).thenReturn(Bytes.toBytes("row"));
160 values.add(kv1);
161 when(value.getCells()).thenReturn(values);
162 mapper.setup(context);
163
164 doAnswer(new Answer<Void>() {
165
166 @Override
167 public Void answer(InvocationOnMock invocation) throws Throwable {
168 ImmutableBytesWritable writer = (ImmutableBytesWritable) invocation.getArguments()[0];
169 KeyValue key = (KeyValue) invocation.getArguments()[1];
170 assertEquals("row", Bytes.toString(writer.get()));
171 assertEquals("row", Bytes.toString(key.getRow()));
172 return null;
173 }
174 }).when(context).write(any(ImmutableBytesWritable.class), any(KeyValue.class));
175
176 mapper.map(key, value, context);
177
178 }
179
180
181
182
183 @Test
184 public void testMainMethod() throws Exception {
185
186 PrintStream oldPrintStream = System.err;
187 SecurityManager SECURITY_MANAGER = System.getSecurityManager();
188 LauncherSecurityManager newSecurityManager= new LauncherSecurityManager();
189 System.setSecurityManager(newSecurityManager);
190 ByteArrayOutputStream data = new ByteArrayOutputStream();
191 String[] args = {};
192 System.setErr(new PrintStream(data));
193 try {
194 System.setErr(new PrintStream(data));
195 try {
196 WALPlayer.main(args);
197 fail("should be SecurityException");
198 } catch (SecurityException e) {
199 assertEquals(-1, newSecurityManager.getExitCode());
200 assertTrue(data.toString().contains("ERROR: Wrong number of arguments:"));
201 assertTrue(data.toString().contains("Usage: WALPlayer [options] <wal inputdir>" +
202 " <tables> [<tableMappings>]"));
203 assertTrue(data.toString().contains("-Dwal.bulk.output=/path/for/output"));
204 }
205
206 } finally {
207 System.setErr(oldPrintStream);
208 System.setSecurityManager(SECURITY_MANAGER);
209 }
210
211 }
212
213 }