View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.mapreduce;
19  
20  import static org.junit.Assert.assertEquals;
21  
22  import java.io.IOException;
23  import java.util.ArrayList;
24  import java.util.Arrays;
25  import java.util.List;
26  import java.util.UUID;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.conf.Configurable;
31  import org.apache.hadoop.conf.Configuration;
32  import org.apache.hadoop.fs.FSDataOutputStream;
33  import org.apache.hadoop.fs.FileSystem;
34  import org.apache.hadoop.fs.Path;
35  import org.apache.hadoop.hbase.HBaseTestingUtility;
36  import org.apache.hadoop.hbase.TableName;
37  import org.apache.hadoop.hbase.client.Durability;
38  import org.apache.hadoop.hbase.client.Put;
39  import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
40  import org.apache.hadoop.hbase.coprocessor.ObserverContext;
41  import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
42  import org.apache.hadoop.hbase.regionserver.HRegion;
43  import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
44  import org.apache.hadoop.hbase.testclassification.LargeTests;
45  import org.apache.hadoop.hbase.util.Bytes;
46  import org.apache.hadoop.util.Tool;
47  import org.apache.hadoop.util.ToolRunner;
48  
49  import org.junit.AfterClass;
50  import org.junit.BeforeClass;
51  import org.junit.Test;
52  import org.junit.experimental.categories.Category;
53  
54  @Category(LargeTests.class)
55  public class TestImportTSVWithTTLs implements Configurable {
56  
57    protected static final Log LOG = LogFactory.getLog(TestImportTSVWithTTLs.class);
58    protected static final String NAME = TestImportTsv.class.getSimpleName();
59    protected static HBaseTestingUtility util = new HBaseTestingUtility();
60  
61    /**
62     * Delete the tmp directory after running doMROnTableTest. Boolean. Default is
63     * false.
64     */
65    protected static final String DELETE_AFTER_LOAD_CONF = NAME + ".deleteAfterLoad";
66  
67    /**
68     * Force use of combiner in doMROnTableTest. Boolean. Default is true.
69     */
70    protected static final String FORCE_COMBINER_CONF = NAME + ".forceCombiner";
71  
72    private final String FAMILY = "FAM";
73    private static Configuration conf;
74  
75    @Override
76    public Configuration getConf() {
77      return util.getConfiguration();
78    }
79  
80    @Override
81    public void setConf(Configuration conf) {
82      throw new IllegalArgumentException("setConf not supported");
83    }
84  
85    @BeforeClass
86    public static void provisionCluster() throws Exception {
87      conf = util.getConfiguration();
88      // We don't check persistence in HFiles in this test, but if we ever do we will
89      // need this where the default hfile version is not 3 (i.e. 0.98)
90      conf.setInt("hfile.format.version", 3);
91      conf.set("hbase.coprocessor.region.classes", TTLCheckingObserver.class.getName());
92      util.startMiniCluster();
93      util.startMiniMapReduceCluster();
94    }
95  
96    @AfterClass
97    public static void releaseCluster() throws Exception {
98      util.shutdownMiniMapReduceCluster();
99      util.shutdownMiniCluster();
100   }
101 
102   @Test
103   public void testMROnTable() throws Exception {
104     String tableName = "test-" + UUID.randomUUID();
105 
106     // Prepare the arguments required for the test.
107     String[] args = new String[] {
108         "-D" + ImportTsv.MAPPER_CONF_KEY
109             + "=org.apache.hadoop.hbase.mapreduce.TsvImporterMapper",
110         "-D" + ImportTsv.COLUMNS_CONF_KEY + "=HBASE_ROW_KEY,FAM:A,FAM:B,HBASE_CELL_TTL",
111         "-D" + ImportTsv.SEPARATOR_CONF_KEY + "=\u001b", tableName };
112     String data = "KEY\u001bVALUE1\u001bVALUE2\u001b1000000\n";
113     util.createTable(TableName.valueOf(tableName), FAMILY);
114     doMROnTableTest(util, FAMILY, data, args, 1);
115     util.deleteTable(tableName);
116   }
117 
118   protected static Tool doMROnTableTest(HBaseTestingUtility util, String family, String data,
119       String[] args, int valueMultiplier) throws Exception {
120     TableName table = TableName.valueOf(args[args.length - 1]);
121     Configuration conf = new Configuration(util.getConfiguration());
122 
123     // populate input file
124     FileSystem fs = FileSystem.get(conf);
125     Path inputPath = fs.makeQualified(new Path(util
126         .getDataTestDirOnTestFS(table.getNameAsString()), "input.dat"));
127     FSDataOutputStream op = fs.create(inputPath, true);
128     op.write(Bytes.toBytes(data));
129     op.close();
130     LOG.debug(String.format("Wrote test data to file: %s", inputPath));
131 
132     if (conf.getBoolean(FORCE_COMBINER_CONF, true)) {
133       LOG.debug("Forcing combiner.");
134       conf.setInt("mapreduce.map.combine.minspills", 1);
135     }
136 
137     // run the import
138     List<String> argv = new ArrayList<String>(Arrays.asList(args));
139     argv.add(inputPath.toString());
140     Tool tool = new ImportTsv();
141     LOG.debug("Running ImportTsv with arguments: " + argv);
142     try {
143       // Job will fail if observer rejects entries without TTL
144       assertEquals(0, ToolRunner.run(conf, tool, argv.toArray(args)));
145     } finally {
146       // Clean up
147       if (conf.getBoolean(DELETE_AFTER_LOAD_CONF, true)) {
148         LOG.debug("Deleting test subdirectory");
149         util.cleanupDataTestDirOnTestFS(table.getNameAsString());
150       }
151     }
152 
153     return tool;
154   }
155 
156   public static class TTLCheckingObserver extends BaseRegionObserver {
157 
158     @Override
159     public void prePut(ObserverContext<RegionCoprocessorEnvironment> e, Put put, WALEdit edit,
160         Durability durability) throws IOException {
161       HRegion region = e.getEnvironment().getRegion();
162       if (!region.getRegionInfo().isMetaTable()
163           && !region.getRegionInfo().getTable().isSystemTable()) {
164         // The put carries the TTL attribute
165         if (put.getTTL() != Long.MAX_VALUE) {
166           return;
167         }
168         throw new IOException("Operation does not have TTL set");
169       }
170     }
171   }
172 }