1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.mapreduce;
19
20 import static org.junit.Assert.assertTrue;
21
22 import java.io.IOException;
23 import java.util.UUID;
24
25 import org.apache.commons.logging.Log;
26 import org.apache.commons.logging.LogFactory;
27 import org.apache.hadoop.conf.Configuration;
28 import org.apache.hadoop.fs.FileSystem;
29 import org.apache.hadoop.fs.Path;
30 import org.apache.hadoop.hbase.HBaseTestingUtility;
31 import org.apache.hadoop.hbase.LargeTests;
32 import org.apache.hadoop.hbase.client.HTable;
33 import org.apache.hadoop.hbase.client.Scan;
34 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
35 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
36 import org.apache.hadoop.hbase.util.Bytes;
37 import org.apache.hadoop.mapreduce.Job;
38 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
39 import org.junit.AfterClass;
40 import org.junit.BeforeClass;
41 import org.junit.Test;
42 import org.junit.experimental.categories.Category;
43
44
45
46
47
48
49
50
51 @Category(LargeTests.class)
52 public class TestTableSnapshotInputFormatScan {
53
54 static final Log LOG = LogFactory.getLog(TestTableSnapshotInputFormatScan.class);
55 static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
56
57 static final byte[] TABLE_NAME = Bytes.toBytes("scantest");
58 static final byte[] SNAPSHOT_NAME = Bytes.toBytes("scantest_snaphot");
59 static final byte[] INPUT_FAMILY = Bytes.toBytes("contents");
60 static final String KEY_STARTROW = "startRow";
61 static final String KEY_LASTROW = "stpRow";
62
63 private static HTable table = null;
64
65 @BeforeClass
66 public static void setUpBeforeClass() throws Exception {
67
68 TEST_UTIL.getConfiguration().setBoolean(
69 SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
70 TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
71 TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
72 TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 6);
73 TEST_UTIL.getConfiguration().setBoolean(
74 "hbase.master.enabletable.roundrobin", true);
75
76
77 TEST_UTIL.enableDebug(TableSnapshotInputFormat.class);
78
79
80 TEST_UTIL.startMiniCluster(3);
81
82
83 table = TEST_UTIL.createTable(TABLE_NAME, INPUT_FAMILY);
84 TEST_UTIL.createMultiRegions(table, INPUT_FAMILY);
85 TEST_UTIL.loadTable(table, INPUT_FAMILY);
86 TEST_UTIL.getHBaseAdmin().disableTable(TABLE_NAME);
87 TEST_UTIL.getHBaseAdmin().snapshot(SNAPSHOT_NAME, TABLE_NAME);
88 TEST_UTIL.getHBaseAdmin().enableTable(TABLE_NAME);
89
90
91 TEST_UTIL.startMiniMapReduceCluster();
92 }
93
94 @AfterClass
95 public static void tearDownAfterClass() throws Exception {
96 TEST_UTIL.shutdownMiniMapReduceCluster();
97 TEST_UTIL.shutdownMiniCluster();
98 }
99
100
101
102
103
104
105
106
107 @Test
108 public void testScanEmptyToEmpty() throws IOException, InterruptedException,
109 ClassNotFoundException {
110 testScan(null, null, null);
111 }
112
113
114
115
116
117
118
119
120 @Test
121 public void testScanEmptyToAPP() throws IOException, InterruptedException,
122 ClassNotFoundException {
123 testScan(null, "app", "apo");
124 }
125
126
127
128
129
130
131
132
133 @Test
134 public void testScanEmptyToBBA() throws IOException, InterruptedException,
135 ClassNotFoundException {
136 testScan(null, "bba", "baz");
137 }
138
139
140
141
142
143
144
145
146 @Test
147 public void testScanEmptyToBBB() throws IOException, InterruptedException,
148 ClassNotFoundException {
149 testScan(null, "bbb", "bba");
150 }
151
152
153
154
155
156
157
158
159 @Test
160 public void testScanEmptyToOPP() throws IOException, InterruptedException,
161 ClassNotFoundException {
162 testScan(null, "opp", "opo");
163 }
164
165
166
167
168
169
170
171
172 protected void testScan(String start, String stop, String last)
173 throws IOException, InterruptedException, ClassNotFoundException {
174 String jobName = "Scan" + (start != null ? start.toUpperCase() : "Empty")
175 + "To" + (stop != null ? stop.toUpperCase() : "Empty");
176 LOG.info("Before map/reduce startup - job " + jobName);
177 Configuration c = new Configuration(TEST_UTIL.getConfiguration());
178 Scan scan = new Scan();
179 scan.addFamily(INPUT_FAMILY);
180 if (start != null) {
181 scan.setStartRow(Bytes.toBytes(start));
182 }
183 c.set(KEY_STARTROW, start != null ? start : "");
184 if (stop != null) {
185 scan.setStopRow(Bytes.toBytes(stop));
186 }
187 c.set(KEY_LASTROW, last != null ? last : "");
188 LOG.info("scan before: " + scan);
189 Job job = new Job(c, jobName);
190
191 FileSystem fs = FileSystem.get(c);
192 Path tmpDir = new Path("/" + UUID.randomUUID());
193 fs.mkdirs(tmpDir);
194 try {
195 TableMapReduceUtil.initTableSnapshotMapperJob(Bytes.toString(SNAPSHOT_NAME),
196 scan, TestTableInputFormatScanBase.ScanMapper.class,
197 ImmutableBytesWritable.class, ImmutableBytesWritable.class, job,
198 false, tmpDir);
199 job.setReducerClass(TestTableInputFormatScanBase.ScanReducer.class);
200 job.setNumReduceTasks(1);
201 FileOutputFormat.setOutputPath(job, new Path(job.getJobName()));
202 LOG.info("Started " + job.getJobName());
203 assertTrue(job.waitForCompletion(true));
204 LOG.info("After map/reduce completion - job " + jobName);
205 } finally {
206 fs.delete(tmpDir, true);
207 }
208
209 }
210
211 }