View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.mapreduce;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertTrue;
23  import static org.junit.Assert.fail;
24  
25  import java.io.ByteArrayOutputStream;
26  import java.io.IOException;
27  import java.io.PrintStream;
28  import java.util.ArrayList;
29  
30  import org.apache.commons.logging.Log;
31  import org.apache.commons.logging.LogFactory;
32  import org.apache.hadoop.conf.Configuration;
33  import org.apache.hadoop.hbase.HBaseTestingUtility;
34  import org.apache.hadoop.hbase.MediumTests;
35  import org.apache.hadoop.hbase.client.HTable;
36  import org.apache.hadoop.hbase.client.Put;
37  import org.apache.hadoop.hbase.mapreduce.RowCounter.RowCounterMapper;
38  import org.apache.hadoop.hbase.util.Bytes;
39  import org.apache.hadoop.hbase.util.LauncherSecurityManager;
40  import org.apache.hadoop.mapreduce.Counter;
41  import org.apache.hadoop.mapreduce.Job;
42  import org.apache.hadoop.util.GenericOptionsParser;
43  import org.junit.AfterClass;
44  import org.junit.BeforeClass;
45  import org.junit.Test;
46  import org.junit.experimental.categories.Category;
47  
48  /**
49   * Test the rowcounter map reduce job.
50   */
51  @Category(MediumTests.class)
52  public class TestRowCounter {
53    final Log LOG = LogFactory.getLog(getClass());
54    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
55    private final static String TABLE_NAME = "testRowCounter";
56    private final static String COL_FAM = "col_fam";
57    private final static String COL1 = "c1";
58    private final static String COL2 = "c2";
59    private final static int TOTAL_ROWS = 10;
60    private final static int ROWS_WITH_ONE_COL = 2;
61  
62    /**
63     * @throws java.lang.Exception
64     */
65    @BeforeClass
66    public static void setUpBeforeClass() throws Exception {
67      TEST_UTIL.startMiniCluster();
68      TEST_UTIL.startMiniMapReduceCluster();
69      HTable table = TEST_UTIL.createTable(Bytes.toBytes(TABLE_NAME),
70          Bytes.toBytes(COL_FAM));
71      writeRows(table);
72      table.close();
73    }
74  
75    /**
76     * @throws java.lang.Exception
77     */
78    @AfterClass
79    public static void tearDownAfterClass() throws Exception {
80      TEST_UTIL.shutdownMiniCluster();
81      TEST_UTIL.shutdownMiniMapReduceCluster();
82    }
83  
84    /**
85     * Test a case when no column was specified in command line arguments.
86     * 
87     * @throws Exception
88     */
89    @Test
90    public void testRowCounterNoColumn() throws Exception {
91      String[] args = new String[] {
92          TABLE_NAME
93      };
94      runRowCount(args, 10);
95    }
96  
97    /**
98     * Test a case when the column specified in command line arguments is
99     * exclusive for few rows.
100    * 
101    * @throws Exception
102    */
103   @Test
104   public void testRowCounterExclusiveColumn() throws Exception {
105     String[] args = new String[] {
106         TABLE_NAME, COL_FAM + ":" + COL1
107     };
108     runRowCount(args, 8);
109   }
110 
111   /**
112    * Test a case when the column specified in command line arguments is not part
113    * of first KV for a row.
114    * 
115    * @throws Exception
116    */
117   @Test
118   public void testRowCounterHiddenColumn() throws Exception {
119     String[] args = new String[] {
120         TABLE_NAME, COL_FAM + ":" + COL2
121     };
122     runRowCount(args, 10);
123   }
124 
125   /**
126    * Run the RowCounter map reduce job and verify the row count.
127    * 
128    * @param args the command line arguments to be used for rowcounter job.
129    * @param expectedCount the expected row count (result of map reduce job).
130    * @throws Exception
131    */
132   private void runRowCount(String[] args, int expectedCount) throws Exception {
133     GenericOptionsParser opts = new GenericOptionsParser(
134         TEST_UTIL.getConfiguration(), args);
135     Configuration conf = opts.getConfiguration();
136     args = opts.getRemainingArgs();
137     Job job = RowCounter.createSubmittableJob(conf, args);
138     job.waitForCompletion(true);
139     assertTrue(job.isSuccessful());
140     Counter counter = job.getCounters().findCounter(
141         RowCounterMapper.Counters.ROWS);
142     assertEquals(expectedCount, counter.getValue());
143   }
144 
145   /**
146    * Writes TOTAL_ROWS number of distinct rows in to the table. Few rows have
147    * two columns, Few have one.
148    * 
149    * @param table
150    * @throws IOException
151    */
152   private static void writeRows(HTable table) throws IOException {
153     final byte[] family = Bytes.toBytes(COL_FAM);
154     final byte[] value = Bytes.toBytes("abcd");
155     final byte[] col1 = Bytes.toBytes(COL1);
156     final byte[] col2 = Bytes.toBytes(COL2);
157     ArrayList<Put> rowsUpdate = new ArrayList<Put>();
158     // write few rows with two columns
159     int i = 0;
160     for (; i < TOTAL_ROWS - ROWS_WITH_ONE_COL; i++) {
161       byte[] row = Bytes.toBytes("row" + i);
162       Put put = new Put(row);
163       put.add(family, col1, value);
164       put.add(family, col2, value);
165       rowsUpdate.add(put);
166     }
167 
168     // write few rows with only one column
169     for (; i < TOTAL_ROWS; i++) {
170       byte[] row = Bytes.toBytes("row" + i);
171       Put put = new Put(row);
172       put.add(family, col2, value);
173       rowsUpdate.add(put);
174     }
175     table.put(rowsUpdate);
176   }
177 
178   /**
179    * test main method. Import should print help and call System.exit
180    */
181   @Test
182   public void testImportMain() throws Exception {
183     PrintStream oldPrintStream = System.err;
184     SecurityManager SECURITY_MANAGER = System.getSecurityManager();
185     LauncherSecurityManager newSecurityManager= new LauncherSecurityManager();
186     System.setSecurityManager(newSecurityManager);
187     ByteArrayOutputStream data = new ByteArrayOutputStream();
188     String[] args = {};
189     System.setErr(new PrintStream(data));
190     try {
191       System.setErr(new PrintStream(data));
192 
193       try {
194         RowCounter.main(args);
195         fail("should be SecurityException");
196       } catch (SecurityException e) {
197         assertEquals(-1, newSecurityManager.getExitCode());
198         assertTrue(data.toString().contains("Wrong number of parameters:"));
199         assertTrue(data.toString().contains(
200             "Usage: RowCounter [options] <tablename> [--range=[startKey],[endKey]] " +
201             "[<column1> <column2>...]"));
202         assertTrue(data.toString().contains("-Dhbase.client.scanner.caching=100"));
203         assertTrue(data.toString().contains("-Dmapred.map.tasks.speculative.execution=false"));
204       }
205       data.reset();
206       try {
207         args = new String[2];
208         args[0] = "table";
209         args[1] = "--range=1";
210         RowCounter.main(args);
211         fail("should be SecurityException");
212       } catch (SecurityException e) {
213         assertEquals(-1, newSecurityManager.getExitCode());
214         assertTrue(data.toString().contains(
215             "Please specify range in such format as \"--range=a,b\" or, with only one boundary," +
216             " \"--range=,b\" or \"--range=a,\""));
217         assertTrue(data.toString().contains(
218             "Usage: RowCounter [options] <tablename> [--range=[startKey],[endKey]] " +
219             "[<column1> <column2>...]"));
220       }
221 
222     } finally {
223       System.setErr(oldPrintStream);
224       System.setSecurityManager(SECURITY_MANAGER);
225     }
226 
227   }
228 
229 }