View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.mapreduce;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertTrue;
22  import static org.junit.Assert.fail;
23  import static org.mockito.Matchers.any;
24  import static org.mockito.Mockito.doAnswer;
25  import static org.mockito.Mockito.mock;
26  import static org.mockito.Mockito.when;
27  
28  import java.io.ByteArrayOutputStream;
29  import java.io.PrintStream;
30  import java.util.ArrayList;
31  
32  import org.apache.hadoop.conf.Configuration;
33  import org.apache.hadoop.fs.Path;
34  import org.apache.hadoop.hbase.Cell;
35  import org.apache.hadoop.hbase.CellUtil;
36  import org.apache.hadoop.hbase.HBaseTestingUtility;
37  import org.apache.hadoop.hbase.HConstants;
38  import org.apache.hadoop.hbase.KeyValue;
39  import org.apache.hadoop.hbase.testclassification.LargeTests;
40  import org.apache.hadoop.hbase.MiniHBaseCluster;
41  import org.apache.hadoop.hbase.TableName;
42  import org.apache.hadoop.hbase.client.Delete;
43  import org.apache.hadoop.hbase.client.Get;
44  import org.apache.hadoop.hbase.client.Put;
45  import org.apache.hadoop.hbase.client.Result;
46  import org.apache.hadoop.hbase.client.Table;
47  import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
48  import org.apache.hadoop.hbase.mapreduce.WALPlayer.WALKeyValueMapper;
49  import org.apache.hadoop.hbase.wal.WAL;
50  import org.apache.hadoop.hbase.wal.WALKey;
51  import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
52  import org.apache.hadoop.hbase.util.Bytes;
53  import org.apache.hadoop.hbase.util.LauncherSecurityManager;
54  import org.apache.hadoop.mapreduce.Mapper;
55  import org.apache.hadoop.mapreduce.Mapper.Context;
56  import org.junit.AfterClass;
57  import org.junit.BeforeClass;
58  import org.junit.Test;
59  import org.junit.experimental.categories.Category;
60  import org.mockito.invocation.InvocationOnMock;
61  import org.mockito.stubbing.Answer;
62  
63  /**
64   * Basic test for the WALPlayer M/R tool
65   */
66  @Category(LargeTests.class)
67  public class TestWALPlayer {
68    private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
69    private static MiniHBaseCluster cluster;
70  
71    @BeforeClass
72    public static void beforeClass() throws Exception {
73      cluster = TEST_UTIL.startMiniCluster();
74      TEST_UTIL.startMiniMapReduceCluster();
75    }
76  
77    @AfterClass
78    public static void afterClass() throws Exception {
79      TEST_UTIL.shutdownMiniMapReduceCluster();
80      TEST_UTIL.shutdownMiniCluster();
81    }
82  
83    /**
84     * Simple end-to-end test
85     * @throws Exception
86     */
87    @Test
88    public void testWALPlayer() throws Exception {
89      final TableName TABLENAME1 = TableName.valueOf("testWALPlayer1");
90      final TableName TABLENAME2 = TableName.valueOf("testWALPlayer2");
91      final byte[] FAMILY = Bytes.toBytes("family");
92      final byte[] COLUMN1 = Bytes.toBytes("c1");
93      final byte[] COLUMN2 = Bytes.toBytes("c2");
94      final byte[] ROW = Bytes.toBytes("row");
95      Table t1 = TEST_UTIL.createTable(TABLENAME1, FAMILY);
96      Table t2 = TEST_UTIL.createTable(TABLENAME2, FAMILY);
97  
98      // put a row into the first table
99      Put p = new Put(ROW);
100     p.add(FAMILY, COLUMN1, COLUMN1);
101     p.add(FAMILY, COLUMN2, COLUMN2);
102     t1.put(p);
103     // delete one column
104     Delete d = new Delete(ROW);
105     d.deleteColumns(FAMILY, COLUMN1);
106     t1.delete(d);
107 
108     // replay the WAL, map table 1 to table 2
109     WAL log = cluster.getRegionServer(0).getWAL(null);
110     log.rollWriter();
111     String walInputDir = new Path(cluster.getMaster().getMasterFileSystem()
112         .getRootDir(), HConstants.HREGION_LOGDIR_NAME).toString();
113 
114     Configuration configuration= TEST_UTIL.getConfiguration();
115     WALPlayer player = new WALPlayer(configuration);
116     String optionName="_test_.name";
117     configuration.set(optionName, "1000");
118     player.setupTime(configuration, optionName);
119     assertEquals(1000,configuration.getLong(optionName,0));
120     assertEquals(0, player.run(new String[] {walInputDir, TABLENAME1.getNameAsString(),
121         TABLENAME2.getNameAsString() }));
122 
123     
124     // verify the WAL was player into table 2
125     Get g = new Get(ROW);
126     Result r = t2.get(g);
127     assertEquals(1, r.size());
128     assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN2));
129   }
130 
131   /**
132    * Test WALKeyValueMapper setup and map
133    */
134   @Test
135   public void testWALKeyValueMapper() throws Exception {
136     testWALKeyValueMapper(WALPlayer.TABLES_KEY);
137   }
138 
139   @Test
140   public void testWALKeyValueMapperWithDeprecatedConfig() throws Exception {
141     testWALKeyValueMapper("hlog.input.tables");
142   }
143 
144   private void testWALKeyValueMapper(final String tableConfigKey) throws Exception {
145     Configuration configuration = new Configuration();
146     configuration.set(tableConfigKey, "table");
147     WALKeyValueMapper mapper = new WALKeyValueMapper();
148     WALKey key = mock(WALKey.class);
149     when(key.getTablename()).thenReturn(TableName.valueOf("table"));
150     @SuppressWarnings("unchecked")
151     Mapper<WALKey, WALEdit, ImmutableBytesWritable, KeyValue>.Context context =
152         mock(Context.class);
153     when(context.getConfiguration()).thenReturn(configuration);
154 
155     WALEdit value = mock(WALEdit.class);
156     ArrayList<Cell> values = new ArrayList<Cell>();
157     KeyValue kv1 = mock(KeyValue.class);
158     when(kv1.getFamily()).thenReturn(Bytes.toBytes("family"));
159     when(kv1.getRow()).thenReturn(Bytes.toBytes("row"));
160     values.add(kv1);
161     when(value.getCells()).thenReturn(values);
162     mapper.setup(context);
163 
164     doAnswer(new Answer<Void>() {
165 
166       @Override
167       public Void answer(InvocationOnMock invocation) throws Throwable {
168         ImmutableBytesWritable writer = (ImmutableBytesWritable) invocation.getArguments()[0];
169         KeyValue key = (KeyValue) invocation.getArguments()[1];
170         assertEquals("row", Bytes.toString(writer.get()));
171         assertEquals("row", Bytes.toString(key.getRow()));
172         return null;
173       }
174     }).when(context).write(any(ImmutableBytesWritable.class), any(KeyValue.class));
175 
176     mapper.map(key, value, context);
177 
178   }
179 
180   /**
181    * Test main method
182    */
183   @Test
184   public void testMainMethod() throws Exception {
185 
186     PrintStream oldPrintStream = System.err;
187     SecurityManager SECURITY_MANAGER = System.getSecurityManager();
188     LauncherSecurityManager newSecurityManager= new LauncherSecurityManager();
189     System.setSecurityManager(newSecurityManager);
190     ByteArrayOutputStream data = new ByteArrayOutputStream();
191     String[] args = {};
192     System.setErr(new PrintStream(data));
193     try {
194       System.setErr(new PrintStream(data));
195       try {
196         WALPlayer.main(args);
197         fail("should be SecurityException");
198       } catch (SecurityException e) {
199         assertEquals(-1, newSecurityManager.getExitCode());
200         assertTrue(data.toString().contains("ERROR: Wrong number of arguments:"));
201         assertTrue(data.toString().contains("Usage: WALPlayer [options] <wal inputdir>" +
202             " <tables> [<tableMappings>]"));
203         assertTrue(data.toString().contains("-Dwal.bulk.output=/path/for/output"));
204       }
205 
206     } finally {
207       System.setErr(oldPrintStream);
208       System.setSecurityManager(SECURITY_MANAGER);
209     }
210 
211   }
212 
213 }