1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.mapreduce;
19  
20  import static org.junit.Assert.*;
21  
22  import java.io.IOException;
23  
24  import org.apache.hadoop.conf.Configuration;
25  import org.apache.hadoop.fs.FileSystem;
26  import org.apache.hadoop.fs.Path;
27  import org.apache.hadoop.hbase.HBaseTestingUtility;
28  import org.apache.hadoop.hbase.HColumnDescriptor;
29  import org.apache.hadoop.hbase.HTableDescriptor;
30  import org.apache.hadoop.hbase.KeyValue;
31  import org.apache.hadoop.hbase.MediumTests;
32  import org.apache.hadoop.hbase.client.Delete;
33  import org.apache.hadoop.hbase.client.Get;
34  import org.apache.hadoop.hbase.client.HTable;
35  import org.apache.hadoop.hbase.client.Put;
36  import org.apache.hadoop.hbase.client.Result;
37  import org.apache.hadoop.hbase.client.ResultScanner;
38  import org.apache.hadoop.hbase.client.Scan;
39  import org.apache.hadoop.hbase.filter.Filter;
40  import org.apache.hadoop.hbase.filter.PrefixFilter;
41  import org.apache.hadoop.hbase.util.Bytes;
42  import org.apache.hadoop.mapreduce.Job;
43  import org.apache.hadoop.util.GenericOptionsParser;
44  import org.junit.After;
45  import org.junit.AfterClass;
46  import org.junit.Assert;
47  import org.junit.Before;
48  import org.junit.BeforeClass;
49  import org.junit.Test;
50  import org.junit.experimental.categories.Category;
51  
52  @Category(MediumTests.class)
53  public class TestImportExport {
54    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
55    private static final byte[] ROW1 = Bytes.toBytes("row1");
56    private static final byte[] ROW2 = Bytes.toBytes("row2");
57    private static final String FAMILYA_STRING = "a";
58    private static final String FAMILYB_STRING = "b";
59    private static final byte[] FAMILYA = Bytes.toBytes(FAMILYA_STRING);
60    private static final byte[] FAMILYB = Bytes.toBytes(FAMILYB_STRING);
61    private static final byte[] QUAL = Bytes.toBytes("q");
62    private static final String OUTPUT_DIR = "outputdir";
63  
64    private static long now = System.currentTimeMillis();
65  
66    @BeforeClass
67    public static void beforeClass() throws Exception {
68      UTIL.startMiniCluster();
69      UTIL.startMiniMapReduceCluster();
70      UTIL.getConfiguration().set("mapred.job.tracker", "local");
71    }
72  
73    @AfterClass
74    public static void afterClass() throws Exception {
75      UTIL.shutdownMiniMapReduceCluster();
76      UTIL.shutdownMiniCluster();
77    }
78  
79    @Before
80    @After
81    public void cleanup() throws Exception {
82      FileSystem fs = FileSystem.get(UTIL.getConfiguration());
83      fs.delete(new Path(OUTPUT_DIR), true);
84    }
85  
86    /**
87     * Test simple replication case with column mapping
88     * @throws Exception
89     */
90    @Test
91    public void testSimpleCase() throws Exception {
92      String EXPORT_TABLE = "exportSimpleCase";
93      HTable t = UTIL.createTable(Bytes.toBytes(EXPORT_TABLE), FAMILYA);
94      Put p = new Put(ROW1);
95      p.add(FAMILYA, QUAL, now, QUAL);
96      p.add(FAMILYA, QUAL, now+1, QUAL);
97      p.add(FAMILYA, QUAL, now+2, QUAL);
98      t.put(p);
99      p = new Put(ROW2);
100     p.add(FAMILYA, QUAL, now, QUAL);
101     p.add(FAMILYA, QUAL, now+1, QUAL);
102     p.add(FAMILYA, QUAL, now+2, QUAL);
103     t.put(p);
104 
105     String[] args = new String[] {
106         EXPORT_TABLE,
107         OUTPUT_DIR,
108         "1000"
109     };
110     GenericOptionsParser opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
111     Configuration conf = opts.getConfiguration();
112     args = opts.getRemainingArgs();
113 
114     Job job = Export.createSubmittableJob(conf, args);
115     job.getConfiguration().set("mapreduce.framework.name", "yarn");
116     job.waitForCompletion(false);
117     assertTrue(job.isSuccessful());
118 
119 
120     String IMPORT_TABLE = "importTableSimpleCase";
121     t = UTIL.createTable(Bytes.toBytes(IMPORT_TABLE), FAMILYB);
122     args = new String[] {
123         "-D" + Import.CF_RENAME_PROP + "="+FAMILYA_STRING+":"+FAMILYB_STRING,
124         IMPORT_TABLE,
125         OUTPUT_DIR
126     };
127 
128     opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
129     conf = opts.getConfiguration();
130     args = opts.getRemainingArgs();
131 
132     job = Import.createSubmittableJob(conf, args);
133     job.getConfiguration().set("mapreduce.framework.name", "yarn");
134     job.waitForCompletion(false);
135     assertTrue(job.isSuccessful());
136 
137     Get g = new Get(ROW1);
138     g.setMaxVersions();
139     Result r = t.get(g);
140     assertEquals(3, r.size());
141     g = new Get(ROW2);
142     g.setMaxVersions();
143     r = t.get(g);
144     assertEquals(3, r.size());
145   }
146 
147   /**
148    * Test export .META. table
149    * 
150    * @throws Exception
151    */
152   @Test
153   public void testMetaExport() throws Exception {
154     String EXPORT_TABLE = ".META.";
155     String[] args = new String[] { EXPORT_TABLE, OUTPUT_DIR, "1", "0", "0" };
156     GenericOptionsParser opts = new GenericOptionsParser(new Configuration(
157         UTIL.getConfiguration()), args);
158     Configuration conf = opts.getConfiguration();
159     args = opts.getRemainingArgs();
160 
161     Job job = Export.createSubmittableJob(conf, args);
162     job.getConfiguration().set("mapreduce.framework.name", "yarn");
163     job.waitForCompletion(false);
164     assertTrue(job.isSuccessful());
165   }
166 
167   @Test
168   public void testWithDeletes() throws Exception {
169     String EXPORT_TABLE = "exportWithDeletes";
170     HTableDescriptor desc = new HTableDescriptor(EXPORT_TABLE);
171     desc.addFamily(new HColumnDescriptor(FAMILYA)
172         .setMaxVersions(5)
173         .setKeepDeletedCells(true)
174     );
175     UTIL.getHBaseAdmin().createTable(desc);
176     HTable t = new HTable(UTIL.getConfiguration(), EXPORT_TABLE);
177 
178     Put p = new Put(ROW1);
179     p.add(FAMILYA, QUAL, now, QUAL);
180     p.add(FAMILYA, QUAL, now+1, QUAL);
181     p.add(FAMILYA, QUAL, now+2, QUAL);
182     p.add(FAMILYA, QUAL, now+3, QUAL);
183     p.add(FAMILYA, QUAL, now+4, QUAL);
184     t.put(p);
185 
186     Delete d = new Delete(ROW1, now+3);
187     t.delete(d);
188     d = new Delete(ROW1);
189     d.deleteColumns(FAMILYA, QUAL, now+2);
190     t.delete(d);
191     
192     String[] args = new String[] {
193         "-D" + Export.RAW_SCAN + "=true",
194         EXPORT_TABLE,
195         OUTPUT_DIR,
196         "1000"
197     };
198 
199     GenericOptionsParser opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
200     Configuration conf = opts.getConfiguration();
201     args = opts.getRemainingArgs();
202 
203     Job job = Export.createSubmittableJob(conf, args);
204     job.getConfiguration().set("mapreduce.framework.name", "yarn");
205     job.waitForCompletion(false);
206     assertTrue(job.isSuccessful());
207 
208 
209     String IMPORT_TABLE = "importWithDeletes";
210     desc = new HTableDescriptor(IMPORT_TABLE);
211     desc.addFamily(new HColumnDescriptor(FAMILYA)
212         .setMaxVersions(5)
213         .setKeepDeletedCells(true)
214     );
215     UTIL.getHBaseAdmin().createTable(desc);
216     t.close();
217     t = new HTable(UTIL.getConfiguration(), IMPORT_TABLE);
218     args = new String[] {
219         IMPORT_TABLE,
220         OUTPUT_DIR
221     };
222 
223     opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
224     conf = opts.getConfiguration();
225     args = opts.getRemainingArgs();
226 
227     job = Import.createSubmittableJob(conf, args);
228     job.getConfiguration().set("mapreduce.framework.name", "yarn");
229     job.waitForCompletion(false);
230     assertTrue(job.isSuccessful());
231 
232     Scan s = new Scan();
233     s.setMaxVersions();
234     s.setRaw(true);
235     ResultScanner scanner = t.getScanner(s);
236     Result r = scanner.next();
237     KeyValue[] res = r.raw();
238     assertTrue(res[0].isDeleteFamily());
239     assertEquals(now+4, res[1].getTimestamp());
240     assertEquals(now+3, res[2].getTimestamp());
241     assertTrue(res[3].isDelete());
242     assertEquals(now+2, res[4].getTimestamp());
243     assertEquals(now+1, res[5].getTimestamp());
244     assertEquals(now, res[6].getTimestamp());
245     t.close();
246   }
247 
248   @Test
249   public void testWithFilter() throws Exception {
250     String EXPORT_TABLE = "exportSimpleCase_ImportWithFilter";
251     HTableDescriptor desc = new HTableDescriptor(EXPORT_TABLE);
252     desc.addFamily(new HColumnDescriptor(FAMILYA).setMaxVersions(5));
253     UTIL.getHBaseAdmin().createTable(desc);
254     HTable exportTable = new HTable(UTIL.getConfiguration(), EXPORT_TABLE);
255 
256     Put p = new Put(ROW1);
257     p.add(FAMILYA, QUAL, now, QUAL);
258     p.add(FAMILYA, QUAL, now + 1, QUAL);
259     p.add(FAMILYA, QUAL, now + 2, QUAL);
260     p.add(FAMILYA, QUAL, now + 3, QUAL);
261     p.add(FAMILYA, QUAL, now + 4, QUAL);
262     exportTable.put(p);
263 
264     String[] args = new String[] { EXPORT_TABLE, OUTPUT_DIR, "1000" };
265 
266     GenericOptionsParser opts = new GenericOptionsParser(new Configuration(
267         UTIL.getConfiguration()), args);
268     Configuration conf = opts.getConfiguration();
269     args = opts.getRemainingArgs();
270 
271     Job job = Export.createSubmittableJob(conf, args);
272     job.getConfiguration().set("mapreduce.framework.name", "yarn");
273     job.waitForCompletion(false);
274     assertTrue(job.isSuccessful());
275 
276     String IMPORT_TABLE = "importWithFilter";
277     desc = new HTableDescriptor(IMPORT_TABLE);
278     desc.addFamily(new HColumnDescriptor(FAMILYA).setMaxVersions(5));
279     UTIL.getHBaseAdmin().createTable(desc);
280 
281     HTable importTable = new HTable(UTIL.getConfiguration(), IMPORT_TABLE);
282     args = new String[] { "-D" + Import.FILTER_CLASS_CONF_KEY + "=" + PrefixFilter.class.getName(),
283         "-D" + Import.FILTER_ARGS_CONF_KEY + "=" + Bytes.toString(ROW1), IMPORT_TABLE, OUTPUT_DIR,
284         "1000" };
285 
286     opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
287     conf = opts.getConfiguration();
288     args = opts.getRemainingArgs();
289 
290     job = Import.createSubmittableJob(conf, args);
291     job.getConfiguration().set("mapreduce.framework.name", "yarn");
292     job.waitForCompletion(false);
293     assertTrue(job.isSuccessful());
294 
295     // get the count of the source table for that time range
296     PrefixFilter filter = new PrefixFilter(ROW1);
297     int count = getCount(exportTable, filter);
298 
299     Assert.assertEquals("Unexpected row count between export and import tables", count,
300       getCount(importTable, null));
301 
302     // and then test that a broken command doesn't bork everything - easier here because we don't
303     // need to re-run the export job
304 
305     args = new String[] { "-D" + Import.FILTER_CLASS_CONF_KEY + "=" + Filter.class.getName(),
306         "-D" + Import.FILTER_ARGS_CONF_KEY + "=" + Bytes.toString(ROW1) + "", EXPORT_TABLE,
307         OUTPUT_DIR, "1000" };
308 
309     opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
310     conf = opts.getConfiguration();
311     args = opts.getRemainingArgs();
312 
313     job = Import.createSubmittableJob(conf, args);
314     job.getConfiguration().set("mapreduce.framework.name", "yarn");
315     job.waitForCompletion(false);
316     assertFalse("Job succeeedd, but it had a non-instantiable filter!", job.isSuccessful());
317 
318     // cleanup
319     exportTable.close();
320     importTable.close();
321   }
322   
323   
324   @Test
325   public void testWithMultipleDeleteFamilyMarkersOfSameRowSameFamily() throws Exception {
326     String EXPORT_TABLE = "exportWithMultipleDeleteFamilyMarkersOfSameRowSameFamily";
327     HTableDescriptor desc = new HTableDescriptor(EXPORT_TABLE);
328     desc.addFamily(new HColumnDescriptor(FAMILYA)
329         .setMaxVersions(5)
330         .setKeepDeletedCells(true)
331     );
332     UTIL.getHBaseAdmin().createTable(desc);
333     HTable exportT = new HTable(UTIL.getConfiguration(), EXPORT_TABLE);
334 
335     //Add first version of QUAL
336     Put p = new Put(ROW1);
337     p.add(FAMILYA, QUAL, now, QUAL);
338     exportT.put(p);
339 
340     //Add Delete family marker
341     Delete d = new Delete(ROW1, now+3);
342     exportT.delete(d);
343 
344     //Add second version of QUAL
345     p = new Put(ROW1);
346     p.add(FAMILYA, QUAL, now+5, "s".getBytes());
347     exportT.put(p);
348 
349     //Add second Delete family marker
350     d = new Delete(ROW1, now+7);
351     exportT.delete(d);
352 
353     String[] args = new String[] {
354         "-D" + Export.RAW_SCAN + "=true",
355         EXPORT_TABLE,
356         OUTPUT_DIR,
357         "1000", // max number of key versions per key to export
358     };
359 
360     GenericOptionsParser opts = new GenericOptionsParser(new Configuration(
361         UTIL.getConfiguration()), args);
362     Configuration conf = opts.getConfiguration();
363     args = opts.getRemainingArgs();
364 
365     Job job = Export.createSubmittableJob(conf, args);
366     job.getConfiguration().set("mapreduce.framework.name", "yarn");
367     job.waitForCompletion(false);
368     assertTrue(job.isSuccessful());
369 
370     String IMPORT_TABLE = "importWithMultipleDeleteFamilyMarkersOfSameRowSameFamily";
371     desc = new HTableDescriptor(IMPORT_TABLE);
372     desc.addFamily(new HColumnDescriptor(FAMILYA)
373         .setMaxVersions(5)
374         .setKeepDeletedCells(true)
375     );
376     UTIL.getHBaseAdmin().createTable(desc);
377     
378     HTable importT = new HTable(UTIL.getConfiguration(), IMPORT_TABLE);
379     args = new String[] {
380         IMPORT_TABLE,
381         OUTPUT_DIR
382     };
383 
384     opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
385     conf = opts.getConfiguration();
386     args = opts.getRemainingArgs();
387 
388     job = Import.createSubmittableJob(conf, args);
389     job.getConfiguration().set("mapreduce.framework.name", "yarn");
390     job.waitForCompletion(false);
391     assertTrue(job.isSuccessful());
392 
393     Scan s = new Scan();
394     s.setMaxVersions();
395     s.setRaw(true);
396     
397     ResultScanner importedTScanner = importT.getScanner(s);
398     Result importedTResult = importedTScanner.next();
399     
400     ResultScanner exportedTScanner = exportT.getScanner(s);
401     Result  exportedTResult =  exportedTScanner.next();
402     try
403     {
404       Result.compareResults(exportedTResult, importedTResult);
405     }
406     catch (Exception e) {
407       fail("Original and imported tables data comparision failed with error:"+e.getMessage());
408     }
409     finally
410     {
411       exportT.close();
412       importT.close();
413     }
414   }
415 
416   /**
417    * Count the number of keyvalues in the specified table for the given timerange
418    * @param start
419    * @param end
420    * @param table
421    * @return
422    * @throws IOException
423    */
424   private int getCount(HTable table, Filter filter) throws IOException {
425     Scan scan = new Scan();
426     scan.setFilter(filter);
427     ResultScanner results = table.getScanner(scan);
428     int count = 0;
429     for (Result res : results) {
430       count += res.size();
431     }
432     results.close();
433     return count;
434   }
435 }