1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.mapreduce;
19  
20  import static org.junit.Assert.assertTrue;
21  
22  import java.io.IOException;
23  import java.util.UUID;
24  
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.conf.Configuration;
28  import org.apache.hadoop.fs.FileSystem;
29  import org.apache.hadoop.fs.Path;
30  import org.apache.hadoop.hbase.HBaseTestingUtility;
31  import org.apache.hadoop.hbase.LargeTests;
32  import org.apache.hadoop.hbase.client.HTable;
33  import org.apache.hadoop.hbase.client.Scan;
34  import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
35  import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
36  import org.apache.hadoop.hbase.util.Bytes;
37  import org.apache.hadoop.mapreduce.Job;
38  import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
39  import org.junit.AfterClass;
40  import org.junit.BeforeClass;
41  import org.junit.Test;
42  import org.junit.experimental.categories.Category;
43  
44  /**
45   * <p>
46   * Tests scanning a snapshot. Tests various scan start and stop row scenarios.
47   * This is set in a scan and tested in a MapReduce job to see if that is handed
48   * over and done properly too.
49   * </p>
50   */
51  @Category(LargeTests.class)
52  public class TestTableSnapshotInputFormatScan {
53  
54    static final Log LOG = LogFactory.getLog(TestTableSnapshotInputFormatScan.class);
55    static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
56  
57    static final byte[] TABLE_NAME = Bytes.toBytes("scantest");
58    static final byte[] SNAPSHOT_NAME = Bytes.toBytes("scantest_snaphot");
59    static final byte[] INPUT_FAMILY = Bytes.toBytes("contents");
60    static final String KEY_STARTROW = "startRow";
61    static final String KEY_LASTROW = "stpRow";
62  
63    private static HTable table = null;
64  
65    @BeforeClass
66    public static void setUpBeforeClass() throws Exception {
67      // config snapshot support
68      TEST_UTIL.getConfiguration().setBoolean(
69          SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
70      TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
71      TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
72      TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 6);
73      TEST_UTIL.getConfiguration().setBoolean(
74          "hbase.master.enabletable.roundrobin", true);
75  
76      // switch TIF to log at DEBUG level
77      TEST_UTIL.enableDebug(TableSnapshotInputFormat.class);
78  
79      // start mini hbase cluster
80      TEST_UTIL.startMiniCluster(3);
81  
82      // create and fill table
83      table = TEST_UTIL.createTable(TABLE_NAME, INPUT_FAMILY);
84      TEST_UTIL.createMultiRegions(table, INPUT_FAMILY);
85      TEST_UTIL.loadTable(table, INPUT_FAMILY);
86      TEST_UTIL.getHBaseAdmin().disableTable(TABLE_NAME);
87      TEST_UTIL.getHBaseAdmin().snapshot(SNAPSHOT_NAME, TABLE_NAME);
88      TEST_UTIL.getHBaseAdmin().enableTable(TABLE_NAME);
89  
90      // start MR cluster
91      TEST_UTIL.startMiniMapReduceCluster();
92    }
93  
94    @AfterClass
95    public static void tearDownAfterClass() throws Exception {
96      TEST_UTIL.shutdownMiniMapReduceCluster();
97      TEST_UTIL.shutdownMiniCluster();
98    }
99  
100   /**
101    * Tests a MR scan using specific start and stop rows.
102    * 
103    * @throws IOException
104    * @throws ClassNotFoundException
105    * @throws InterruptedException
106    */
107   @Test
108   public void testScanEmptyToEmpty() throws IOException, InterruptedException,
109       ClassNotFoundException {
110     testScan(null, null, null);
111   }
112 
113   /**
114    * Tests a MR scan using specific start and stop rows.
115    * 
116    * @throws IOException
117    * @throws ClassNotFoundException
118    * @throws InterruptedException
119    */
120   @Test
121   public void testScanEmptyToAPP() throws IOException, InterruptedException,
122       ClassNotFoundException {
123     testScan(null, "app", "apo");
124   }
125 
126   /**
127    * Tests a MR scan using specific start and stop rows.
128    * 
129    * @throws IOException
130    * @throws ClassNotFoundException
131    * @throws InterruptedException
132    */
133   @Test
134   public void testScanEmptyToBBA() throws IOException, InterruptedException,
135       ClassNotFoundException {
136     testScan(null, "bba", "baz");
137   }
138 
139   /**
140    * Tests a MR scan using specific start and stop rows.
141    * 
142    * @throws IOException
143    * @throws ClassNotFoundException
144    * @throws InterruptedException
145    */
146   @Test
147   public void testScanEmptyToBBB() throws IOException, InterruptedException,
148       ClassNotFoundException {
149     testScan(null, "bbb", "bba");
150   }
151 
152   /**
153    * Tests a MR scan using specific start and stop rows.
154    * 
155    * @throws IOException
156    * @throws ClassNotFoundException
157    * @throws InterruptedException
158    */
159   @Test
160   public void testScanEmptyToOPP() throws IOException, InterruptedException,
161       ClassNotFoundException {
162     testScan(null, "opp", "opo");
163   }
164 
165   /**
166    * Tests a MR scan using specific start and stop rows.
167    * 
168    * @throws IOException
169    * @throws ClassNotFoundException
170    * @throws InterruptedException
171    */
172   protected void testScan(String start, String stop, String last)
173       throws IOException, InterruptedException, ClassNotFoundException {
174     String jobName = "Scan" + (start != null ? start.toUpperCase() : "Empty")
175         + "To" + (stop != null ? stop.toUpperCase() : "Empty");
176     LOG.info("Before map/reduce startup - job " + jobName);
177     Configuration c = new Configuration(TEST_UTIL.getConfiguration());
178     Scan scan = new Scan();
179     scan.addFamily(INPUT_FAMILY);
180     if (start != null) {
181       scan.setStartRow(Bytes.toBytes(start));
182     }
183     c.set(KEY_STARTROW, start != null ? start : "");
184     if (stop != null) {
185       scan.setStopRow(Bytes.toBytes(stop));
186     }
187     c.set(KEY_LASTROW, last != null ? last : "");
188     LOG.info("scan before: " + scan);
189     Job job = new Job(c, jobName);
190 
191     FileSystem fs = FileSystem.get(c);
192     Path tmpDir = new Path("/" + UUID.randomUUID());
193     fs.mkdirs(tmpDir);
194     try {
195       TableMapReduceUtil.initTableSnapshotMapperJob(Bytes.toString(SNAPSHOT_NAME),
196           scan, TestTableInputFormatScanBase.ScanMapper.class,
197           ImmutableBytesWritable.class, ImmutableBytesWritable.class, job,
198           false, tmpDir);
199       job.setReducerClass(TestTableInputFormatScanBase.ScanReducer.class);
200       job.setNumReduceTasks(1); // one to get final "first" and "last" key
201       FileOutputFormat.setOutputPath(job, new Path(job.getJobName()));
202       LOG.info("Started " + job.getJobName());
203       assertTrue(job.waitForCompletion(true));
204       LOG.info("After map/reduce completion - job " + jobName);
205     } finally {
206       fs.delete(tmpDir, true);
207     }
208 
209   }
210 
211 }