1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.mapreduce;
19
20 import static org.junit.Assert.*;
21
22 import java.io.IOException;
23
24 import org.apache.hadoop.conf.Configuration;
25 import org.apache.hadoop.fs.FileSystem;
26 import org.apache.hadoop.fs.Path;
27 import org.apache.hadoop.hbase.HBaseTestingUtility;
28 import org.apache.hadoop.hbase.HColumnDescriptor;
29 import org.apache.hadoop.hbase.HTableDescriptor;
30 import org.apache.hadoop.hbase.KeyValue;
31 import org.apache.hadoop.hbase.MediumTests;
32 import org.apache.hadoop.hbase.client.Delete;
33 import org.apache.hadoop.hbase.client.Get;
34 import org.apache.hadoop.hbase.client.HTable;
35 import org.apache.hadoop.hbase.client.Put;
36 import org.apache.hadoop.hbase.client.Result;
37 import org.apache.hadoop.hbase.client.ResultScanner;
38 import org.apache.hadoop.hbase.client.Scan;
39 import org.apache.hadoop.hbase.filter.Filter;
40 import org.apache.hadoop.hbase.filter.PrefixFilter;
41 import org.apache.hadoop.hbase.util.Bytes;
42 import org.apache.hadoop.mapreduce.Job;
43 import org.apache.hadoop.util.GenericOptionsParser;
44 import org.junit.After;
45 import org.junit.AfterClass;
46 import org.junit.Assert;
47 import org.junit.Before;
48 import org.junit.BeforeClass;
49 import org.junit.Test;
50 import org.junit.experimental.categories.Category;
51
52 @Category(MediumTests.class)
53 public class TestImportExport {
54 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
55 private static final byte[] ROW1 = Bytes.toBytes("row1");
56 private static final byte[] ROW2 = Bytes.toBytes("row2");
57 private static final String FAMILYA_STRING = "a";
58 private static final String FAMILYB_STRING = "b";
59 private static final byte[] FAMILYA = Bytes.toBytes(FAMILYA_STRING);
60 private static final byte[] FAMILYB = Bytes.toBytes(FAMILYB_STRING);
61 private static final byte[] QUAL = Bytes.toBytes("q");
62 private static final String OUTPUT_DIR = "outputdir";
63
64 private static long now = System.currentTimeMillis();
65
66 @BeforeClass
67 public static void beforeClass() throws Exception {
68 UTIL.startMiniCluster();
69 UTIL.startMiniMapReduceCluster();
70 UTIL.getConfiguration().set("mapred.job.tracker", "local");
71 }
72
73 @AfterClass
74 public static void afterClass() throws Exception {
75 UTIL.shutdownMiniMapReduceCluster();
76 UTIL.shutdownMiniCluster();
77 }
78
79 @Before
80 @After
81 public void cleanup() throws Exception {
82 FileSystem fs = FileSystem.get(UTIL.getConfiguration());
83 fs.delete(new Path(OUTPUT_DIR), true);
84 }
85
86
87
88
89
90 @Test
91 public void testSimpleCase() throws Exception {
92 String EXPORT_TABLE = "exportSimpleCase";
93 HTable t = UTIL.createTable(Bytes.toBytes(EXPORT_TABLE), FAMILYA);
94 Put p = new Put(ROW1);
95 p.add(FAMILYA, QUAL, now, QUAL);
96 p.add(FAMILYA, QUAL, now+1, QUAL);
97 p.add(FAMILYA, QUAL, now+2, QUAL);
98 t.put(p);
99 p = new Put(ROW2);
100 p.add(FAMILYA, QUAL, now, QUAL);
101 p.add(FAMILYA, QUAL, now+1, QUAL);
102 p.add(FAMILYA, QUAL, now+2, QUAL);
103 t.put(p);
104
105 String[] args = new String[] {
106 EXPORT_TABLE,
107 OUTPUT_DIR,
108 "1000"
109 };
110 GenericOptionsParser opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
111 Configuration conf = opts.getConfiguration();
112 args = opts.getRemainingArgs();
113
114 Job job = Export.createSubmittableJob(conf, args);
115 job.getConfiguration().set("mapreduce.framework.name", "yarn");
116 job.waitForCompletion(false);
117 assertTrue(job.isSuccessful());
118
119
120 String IMPORT_TABLE = "importTableSimpleCase";
121 t = UTIL.createTable(Bytes.toBytes(IMPORT_TABLE), FAMILYB);
122 args = new String[] {
123 "-D" + Import.CF_RENAME_PROP + "="+FAMILYA_STRING+":"+FAMILYB_STRING,
124 IMPORT_TABLE,
125 OUTPUT_DIR
126 };
127
128 opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
129 conf = opts.getConfiguration();
130 args = opts.getRemainingArgs();
131
132 job = Import.createSubmittableJob(conf, args);
133 job.getConfiguration().set("mapreduce.framework.name", "yarn");
134 job.waitForCompletion(false);
135 assertTrue(job.isSuccessful());
136
137 Get g = new Get(ROW1);
138 g.setMaxVersions();
139 Result r = t.get(g);
140 assertEquals(3, r.size());
141 g = new Get(ROW2);
142 g.setMaxVersions();
143 r = t.get(g);
144 assertEquals(3, r.size());
145 }
146
147
148
149
150
151
152 @Test
153 public void testMetaExport() throws Exception {
154 String EXPORT_TABLE = ".META.";
155 String[] args = new String[] { EXPORT_TABLE, OUTPUT_DIR, "1", "0", "0" };
156 GenericOptionsParser opts = new GenericOptionsParser(new Configuration(
157 UTIL.getConfiguration()), args);
158 Configuration conf = opts.getConfiguration();
159 args = opts.getRemainingArgs();
160
161 Job job = Export.createSubmittableJob(conf, args);
162 job.getConfiguration().set("mapreduce.framework.name", "yarn");
163 job.waitForCompletion(false);
164 assertTrue(job.isSuccessful());
165 }
166
167 @Test
168 public void testWithDeletes() throws Exception {
169 String EXPORT_TABLE = "exportWithDeletes";
170 HTableDescriptor desc = new HTableDescriptor(EXPORT_TABLE);
171 desc.addFamily(new HColumnDescriptor(FAMILYA)
172 .setMaxVersions(5)
173 .setKeepDeletedCells(true)
174 );
175 UTIL.getHBaseAdmin().createTable(desc);
176 HTable t = new HTable(UTIL.getConfiguration(), EXPORT_TABLE);
177
178 Put p = new Put(ROW1);
179 p.add(FAMILYA, QUAL, now, QUAL);
180 p.add(FAMILYA, QUAL, now+1, QUAL);
181 p.add(FAMILYA, QUAL, now+2, QUAL);
182 p.add(FAMILYA, QUAL, now+3, QUAL);
183 p.add(FAMILYA, QUAL, now+4, QUAL);
184 t.put(p);
185
186 Delete d = new Delete(ROW1, now+3);
187 t.delete(d);
188 d = new Delete(ROW1);
189 d.deleteColumns(FAMILYA, QUAL, now+2);
190 t.delete(d);
191
192 String[] args = new String[] {
193 "-D" + Export.RAW_SCAN + "=true",
194 EXPORT_TABLE,
195 OUTPUT_DIR,
196 "1000"
197 };
198
199 GenericOptionsParser opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
200 Configuration conf = opts.getConfiguration();
201 args = opts.getRemainingArgs();
202
203 Job job = Export.createSubmittableJob(conf, args);
204 job.getConfiguration().set("mapreduce.framework.name", "yarn");
205 job.waitForCompletion(false);
206 assertTrue(job.isSuccessful());
207
208
209 String IMPORT_TABLE = "importWithDeletes";
210 desc = new HTableDescriptor(IMPORT_TABLE);
211 desc.addFamily(new HColumnDescriptor(FAMILYA)
212 .setMaxVersions(5)
213 .setKeepDeletedCells(true)
214 );
215 UTIL.getHBaseAdmin().createTable(desc);
216 t.close();
217 t = new HTable(UTIL.getConfiguration(), IMPORT_TABLE);
218 args = new String[] {
219 IMPORT_TABLE,
220 OUTPUT_DIR
221 };
222
223 opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
224 conf = opts.getConfiguration();
225 args = opts.getRemainingArgs();
226
227 job = Import.createSubmittableJob(conf, args);
228 job.getConfiguration().set("mapreduce.framework.name", "yarn");
229 job.waitForCompletion(false);
230 assertTrue(job.isSuccessful());
231
232 Scan s = new Scan();
233 s.setMaxVersions();
234 s.setRaw(true);
235 ResultScanner scanner = t.getScanner(s);
236 Result r = scanner.next();
237 KeyValue[] res = r.raw();
238 assertTrue(res[0].isDeleteFamily());
239 assertEquals(now+4, res[1].getTimestamp());
240 assertEquals(now+3, res[2].getTimestamp());
241 assertTrue(res[3].isDelete());
242 assertEquals(now+2, res[4].getTimestamp());
243 assertEquals(now+1, res[5].getTimestamp());
244 assertEquals(now, res[6].getTimestamp());
245 t.close();
246 }
247
248 @Test
249 public void testWithFilter() throws Exception {
250 String EXPORT_TABLE = "exportSimpleCase_ImportWithFilter";
251 HTableDescriptor desc = new HTableDescriptor(EXPORT_TABLE);
252 desc.addFamily(new HColumnDescriptor(FAMILYA).setMaxVersions(5));
253 UTIL.getHBaseAdmin().createTable(desc);
254 HTable exportTable = new HTable(UTIL.getConfiguration(), EXPORT_TABLE);
255
256 Put p = new Put(ROW1);
257 p.add(FAMILYA, QUAL, now, QUAL);
258 p.add(FAMILYA, QUAL, now + 1, QUAL);
259 p.add(FAMILYA, QUAL, now + 2, QUAL);
260 p.add(FAMILYA, QUAL, now + 3, QUAL);
261 p.add(FAMILYA, QUAL, now + 4, QUAL);
262 exportTable.put(p);
263
264 String[] args = new String[] { EXPORT_TABLE, OUTPUT_DIR, "1000" };
265
266 GenericOptionsParser opts = new GenericOptionsParser(new Configuration(
267 UTIL.getConfiguration()), args);
268 Configuration conf = opts.getConfiguration();
269 args = opts.getRemainingArgs();
270
271 Job job = Export.createSubmittableJob(conf, args);
272 job.getConfiguration().set("mapreduce.framework.name", "yarn");
273 job.waitForCompletion(false);
274 assertTrue(job.isSuccessful());
275
276 String IMPORT_TABLE = "importWithFilter";
277 desc = new HTableDescriptor(IMPORT_TABLE);
278 desc.addFamily(new HColumnDescriptor(FAMILYA).setMaxVersions(5));
279 UTIL.getHBaseAdmin().createTable(desc);
280
281 HTable importTable = new HTable(UTIL.getConfiguration(), IMPORT_TABLE);
282 args = new String[] { "-D" + Import.FILTER_CLASS_CONF_KEY + "=" + PrefixFilter.class.getName(),
283 "-D" + Import.FILTER_ARGS_CONF_KEY + "=" + Bytes.toString(ROW1), IMPORT_TABLE, OUTPUT_DIR,
284 "1000" };
285
286 opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
287 conf = opts.getConfiguration();
288 args = opts.getRemainingArgs();
289
290 job = Import.createSubmittableJob(conf, args);
291 job.getConfiguration().set("mapreduce.framework.name", "yarn");
292 job.waitForCompletion(false);
293 assertTrue(job.isSuccessful());
294
295
296 PrefixFilter filter = new PrefixFilter(ROW1);
297 int count = getCount(exportTable, filter);
298
299 Assert.assertEquals("Unexpected row count between export and import tables", count,
300 getCount(importTable, null));
301
302
303
304
305 args = new String[] { "-D" + Import.FILTER_CLASS_CONF_KEY + "=" + Filter.class.getName(),
306 "-D" + Import.FILTER_ARGS_CONF_KEY + "=" + Bytes.toString(ROW1) + "", EXPORT_TABLE,
307 OUTPUT_DIR, "1000" };
308
309 opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
310 conf = opts.getConfiguration();
311 args = opts.getRemainingArgs();
312
313 job = Import.createSubmittableJob(conf, args);
314 job.getConfiguration().set("mapreduce.framework.name", "yarn");
315 job.waitForCompletion(false);
316 assertFalse("Job succeeedd, but it had a non-instantiable filter!", job.isSuccessful());
317
318
319 exportTable.close();
320 importTable.close();
321 }
322
323
324 @Test
325 public void testWithMultipleDeleteFamilyMarkersOfSameRowSameFamily() throws Exception {
326 String EXPORT_TABLE = "exportWithMultipleDeleteFamilyMarkersOfSameRowSameFamily";
327 HTableDescriptor desc = new HTableDescriptor(EXPORT_TABLE);
328 desc.addFamily(new HColumnDescriptor(FAMILYA)
329 .setMaxVersions(5)
330 .setKeepDeletedCells(true)
331 );
332 UTIL.getHBaseAdmin().createTable(desc);
333 HTable exportT = new HTable(UTIL.getConfiguration(), EXPORT_TABLE);
334
335
336 Put p = new Put(ROW1);
337 p.add(FAMILYA, QUAL, now, QUAL);
338 exportT.put(p);
339
340
341 Delete d = new Delete(ROW1, now+3);
342 exportT.delete(d);
343
344
345 p = new Put(ROW1);
346 p.add(FAMILYA, QUAL, now+5, "s".getBytes());
347 exportT.put(p);
348
349
350 d = new Delete(ROW1, now+7);
351 exportT.delete(d);
352
353 String[] args = new String[] {
354 "-D" + Export.RAW_SCAN + "=true",
355 EXPORT_TABLE,
356 OUTPUT_DIR,
357 "1000",
358 };
359
360 GenericOptionsParser opts = new GenericOptionsParser(new Configuration(
361 UTIL.getConfiguration()), args);
362 Configuration conf = opts.getConfiguration();
363 args = opts.getRemainingArgs();
364
365 Job job = Export.createSubmittableJob(conf, args);
366 job.getConfiguration().set("mapreduce.framework.name", "yarn");
367 job.waitForCompletion(false);
368 assertTrue(job.isSuccessful());
369
370 String IMPORT_TABLE = "importWithMultipleDeleteFamilyMarkersOfSameRowSameFamily";
371 desc = new HTableDescriptor(IMPORT_TABLE);
372 desc.addFamily(new HColumnDescriptor(FAMILYA)
373 .setMaxVersions(5)
374 .setKeepDeletedCells(true)
375 );
376 UTIL.getHBaseAdmin().createTable(desc);
377
378 HTable importT = new HTable(UTIL.getConfiguration(), IMPORT_TABLE);
379 args = new String[] {
380 IMPORT_TABLE,
381 OUTPUT_DIR
382 };
383
384 opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
385 conf = opts.getConfiguration();
386 args = opts.getRemainingArgs();
387
388 job = Import.createSubmittableJob(conf, args);
389 job.getConfiguration().set("mapreduce.framework.name", "yarn");
390 job.waitForCompletion(false);
391 assertTrue(job.isSuccessful());
392
393 Scan s = new Scan();
394 s.setMaxVersions();
395 s.setRaw(true);
396
397 ResultScanner importedTScanner = importT.getScanner(s);
398 Result importedTResult = importedTScanner.next();
399
400 ResultScanner exportedTScanner = exportT.getScanner(s);
401 Result exportedTResult = exportedTScanner.next();
402 try
403 {
404 Result.compareResults(exportedTResult, importedTResult);
405 }
406 catch (Exception e) {
407 fail("Original and imported tables data comparision failed with error:"+e.getMessage());
408 }
409 finally
410 {
411 exportT.close();
412 importT.close();
413 }
414 }
415
416
417
418
419
420
421
422
423
424 private int getCount(HTable table, Filter filter) throws IOException {
425 Scan scan = new Scan();
426 scan.setFilter(filter);
427 ResultScanner results = table.getScanner(scan);
428 int count = 0;
429 for (Result res : results) {
430 count += res.size();
431 }
432 results.close();
433 return count;
434 }
435 }