View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.regionserver;
19  
20  import java.io.IOException;
21  import java.util.ArrayList;
22  import java.util.List;
23  import java.util.concurrent.atomic.AtomicLong;
24  
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.conf.Configuration;
28  import org.apache.hadoop.fs.FileSystem;
29  import org.apache.hadoop.fs.Path;
30  import org.apache.hadoop.hbase.HBaseConfiguration;
31  import org.apache.hadoop.hbase.HBaseTestingUtility;
32  import org.apache.hadoop.hbase.HColumnDescriptor;
33  import org.apache.hadoop.hbase.HTableDescriptor;
34  import org.apache.hadoop.hbase.KeyValue;
35  import org.apache.hadoop.hbase.testclassification.LargeTests;
36  import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread;
37  import org.apache.hadoop.hbase.MultithreadedTestUtil.TestContext;
38  import org.apache.hadoop.hbase.TableExistsException;
39  import org.apache.hadoop.hbase.TableName;
40  import org.apache.hadoop.hbase.client.HConnection;
41  import org.apache.hadoop.hbase.client.HTable;
42  import org.apache.hadoop.hbase.client.RegionServerCallable;
43  import org.apache.hadoop.hbase.client.Result;
44  import org.apache.hadoop.hbase.client.ResultScanner;
45  import org.apache.hadoop.hbase.client.RpcRetryingCaller;
46  import org.apache.hadoop.hbase.client.RpcRetryingCallerFactory;
47  import org.apache.hadoop.hbase.client.Scan;
48  import org.apache.hadoop.hbase.io.compress.Compression;
49  import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
50  import org.apache.hadoop.hbase.io.hfile.CacheConfig;
51  import org.apache.hadoop.hbase.io.hfile.HFile;
52  import org.apache.hadoop.hbase.io.hfile.HFileContext;
53  import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
54  import org.apache.hadoop.hbase.protobuf.RequestConverter;
55  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
56  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CompactRegionRequest;
57  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileRequest;
58  import org.apache.hadoop.hbase.util.Bytes;
59  import org.apache.hadoop.hbase.util.Pair;
60  import org.junit.Test;
61  import org.junit.experimental.categories.Category;
62  
63  import com.google.common.collect.Lists;
64  
65  /**
66   * Tests bulk loading of HFiles and shows the atomicity or lack of atomicity of
67   * the region server's bullkLoad functionality.
68   */
69  @Category(LargeTests.class)
70  public class TestHRegionServerBulkLoad {
71    final static Log LOG = LogFactory.getLog(TestHRegionServerBulkLoad.class);
72    private static HBaseTestingUtility UTIL = new HBaseTestingUtility();
73    private final static Configuration conf = UTIL.getConfiguration();
74    private final static byte[] QUAL = Bytes.toBytes("qual");
75    private final static int NUM_CFS = 10;
76    public static int BLOCKSIZE = 64 * 1024;
77    public static Algorithm COMPRESSION = Compression.Algorithm.NONE;
78  
79    private final static byte[][] families = new byte[NUM_CFS][];
80    static {
81      for (int i = 0; i < NUM_CFS; i++) {
82        families[i] = Bytes.toBytes(family(i));
83      }
84    }
85  
86    /**
87     * Create a rowkey compatible with
88     * {@link #createHFile(FileSystem, Path, byte[], byte[], byte[], int)}.
89     */
90    public static byte[] rowkey(int i) {
91      return Bytes.toBytes(String.format("row_%08d", i));
92    }
93  
94    static String family(int i) {
95      return String.format("family_%04d", i);
96    }
97  
98    /**
99     * Create an HFile with the given number of rows with a specified value.
100    */
101   public static void createHFile(FileSystem fs, Path path, byte[] family,
102       byte[] qualifier, byte[] value, int numRows) throws IOException {
103     HFileContext context = new HFileContextBuilder().withBlockSize(BLOCKSIZE)
104                             .withCompression(COMPRESSION)
105                             .build();
106     HFile.Writer writer = HFile
107         .getWriterFactory(conf, new CacheConfig(conf))
108         .withPath(fs, path)
109         .withFileContext(context)
110         .create();
111     long now = System.currentTimeMillis();
112     try {
113       // subtract 2 since iterateOnSplits doesn't include boundary keys
114       for (int i = 0; i < numRows; i++) {
115         KeyValue kv = new KeyValue(rowkey(i), family, qualifier, now, value);
116         writer.append(kv);
117       }
118       writer.appendFileInfo(StoreFile.BULKLOAD_TIME_KEY, Bytes.toBytes(now));
119     } finally {
120       writer.close();
121     }
122   }
123 
124   /**
125    * Thread that does full scans of the table looking for any partially
126    * completed rows.
127    *
128    * Each iteration of this loads 10 hdfs files, which occupies 5 file open file
129    * handles. So every 10 iterations (500 file handles) it does a region
130    * compaction to reduce the number of open file handles.
131    */
132   public static class AtomicHFileLoader extends RepeatingTestThread {
133     final AtomicLong numBulkLoads = new AtomicLong();
134     final AtomicLong numCompactions = new AtomicLong();
135     private TableName tableName;
136 
137     public AtomicHFileLoader(TableName tableName, TestContext ctx,
138         byte targetFamilies[][]) throws IOException {
139       super(ctx);
140       this.tableName = tableName;
141     }
142 
143     public void doAnAction() throws Exception {
144       long iteration = numBulkLoads.getAndIncrement();
145       Path dir =  UTIL.getDataTestDirOnTestFS(String.format("bulkLoad_%08d",
146           iteration));
147 
148       // create HFiles for different column families
149       FileSystem fs = UTIL.getTestFileSystem();
150       byte[] val = Bytes.toBytes(String.format("%010d", iteration));
151       final List<Pair<byte[], String>> famPaths = new ArrayList<Pair<byte[], String>>(
152           NUM_CFS);
153       for (int i = 0; i < NUM_CFS; i++) {
154         Path hfile = new Path(dir, family(i));
155         byte[] fam = Bytes.toBytes(family(i));
156         createHFile(fs, hfile, fam, QUAL, val, 1000);
157         famPaths.add(new Pair<byte[], String>(fam, hfile.toString()));
158       }
159 
160       // bulk load HFiles
161       final HConnection conn = UTIL.getHBaseAdmin().getConnection();
162       RegionServerCallable<Void> callable =
163           new RegionServerCallable<Void>(conn, tableName, Bytes.toBytes("aaa")) {
164         @Override
165         public Void call(int callTimeout) throws Exception {
166           LOG.debug("Going to connect to server " + getLocation() + " for row "
167               + Bytes.toStringBinary(getRow()));
168           byte[] regionName = getLocation().getRegionInfo().getRegionName();
169           BulkLoadHFileRequest request =
170             RequestConverter.buildBulkLoadHFileRequest(famPaths, regionName, true);
171           getStub().bulkLoadHFile(null, request);
172           return null;
173         }
174       };
175       RpcRetryingCallerFactory factory = new RpcRetryingCallerFactory(conf);
176       RpcRetryingCaller<Void> caller = factory.<Void> newCaller();
177       caller.callWithRetries(callable, Integer.MAX_VALUE);
178 
179       // Periodically do compaction to reduce the number of open file handles.
180       if (numBulkLoads.get() % 10 == 0) {
181         // 10 * 50 = 500 open file handles!
182         callable = new RegionServerCallable<Void>(conn, tableName, Bytes.toBytes("aaa")) {
183           @Override
184           public Void call(int callTimeout) throws Exception {
185             LOG.debug("compacting " + getLocation() + " for row "
186                 + Bytes.toStringBinary(getRow()));
187             AdminProtos.AdminService.BlockingInterface server =
188               conn.getAdmin(getLocation().getServerName());
189             CompactRegionRequest request =
190               RequestConverter.buildCompactRegionRequest(
191                 getLocation().getRegionInfo().getRegionName(), true, null);
192             server.compactRegion(null, request);
193             numCompactions.incrementAndGet();
194             return null;
195           }
196         };
197         caller.callWithRetries(callable, Integer.MAX_VALUE);
198       }
199     }
200   }
201 
202   /**
203    * Thread that does full scans of the table looking for any partially
204    * completed rows.
205    */
206   public static class AtomicScanReader extends RepeatingTestThread {
207     byte targetFamilies[][];
208     HTable table;
209     AtomicLong numScans = new AtomicLong();
210     AtomicLong numRowsScanned = new AtomicLong();
211     TableName TABLE_NAME;
212 
213     public AtomicScanReader(TableName TABLE_NAME, TestContext ctx,
214         byte targetFamilies[][]) throws IOException {
215       super(ctx);
216       this.TABLE_NAME = TABLE_NAME;
217       this.targetFamilies = targetFamilies;
218       table = new HTable(conf, TABLE_NAME);
219     }
220 
221     public void doAnAction() throws Exception {
222       Scan s = new Scan();
223       for (byte[] family : targetFamilies) {
224         s.addFamily(family);
225       }
226       ResultScanner scanner = table.getScanner(s);
227 
228       for (Result res : scanner) {
229         byte[] lastRow = null, lastFam = null, lastQual = null;
230         byte[] gotValue = null;
231         for (byte[] family : targetFamilies) {
232           byte qualifier[] = QUAL;
233           byte thisValue[] = res.getValue(family, qualifier);
234           if (gotValue != null && thisValue != null
235               && !Bytes.equals(gotValue, thisValue)) {
236 
237             StringBuilder msg = new StringBuilder();
238             msg.append("Failed on scan ").append(numScans)
239                 .append(" after scanning ").append(numRowsScanned)
240                 .append(" rows!\n");
241             msg.append("Current  was " + Bytes.toString(res.getRow()) + "/"
242                 + Bytes.toString(family) + ":" + Bytes.toString(qualifier)
243                 + " = " + Bytes.toString(thisValue) + "\n");
244             msg.append("Previous  was " + Bytes.toString(lastRow) + "/"
245                 + Bytes.toString(lastFam) + ":" + Bytes.toString(lastQual)
246                 + " = " + Bytes.toString(gotValue));
247             throw new RuntimeException(msg.toString());
248           }
249 
250           lastFam = family;
251           lastQual = qualifier;
252           lastRow = res.getRow();
253           gotValue = thisValue;
254         }
255         numRowsScanned.getAndIncrement();
256       }
257       numScans.getAndIncrement();
258     }
259   }
260 
261   /**
262    * Creates a table with given table name and specified number of column
263    * families if the table does not already exist.
264    */
265   private void setupTable(TableName table, int cfs) throws IOException {
266     try {
267       LOG.info("Creating table " + table);
268       HTableDescriptor htd = new HTableDescriptor(table);
269       for (int i = 0; i < 10; i++) {
270         htd.addFamily(new HColumnDescriptor(family(i)));
271       }
272 
273       UTIL.getHBaseAdmin().createTable(htd);
274     } catch (TableExistsException tee) {
275       LOG.info("Table " + table + " already exists");
276     }
277   }
278 
279   /**
280    * Atomic bulk load.
281    */
282   @Test
283   public void testAtomicBulkLoad() throws Exception {
284     TableName TABLE_NAME = TableName.valueOf("atomicBulkLoad");
285 
286     int millisToRun = 30000;
287     int numScanners = 50;
288 
289     UTIL.startMiniCluster(1);
290     try {
291       runAtomicBulkloadTest(TABLE_NAME, millisToRun, numScanners);
292     } finally {
293       UTIL.shutdownMiniCluster();
294     }
295   }
296 
297   void runAtomicBulkloadTest(TableName tableName, int millisToRun, int numScanners)
298       throws Exception {
299     setupTable(tableName, 10);
300 
301     TestContext ctx = new TestContext(UTIL.getConfiguration());
302 
303     AtomicHFileLoader loader = new AtomicHFileLoader(tableName, ctx, null);
304     ctx.addThread(loader);
305 
306     List<AtomicScanReader> scanners = Lists.newArrayList();
307     for (int i = 0; i < numScanners; i++) {
308       AtomicScanReader scanner = new AtomicScanReader(tableName, ctx, families);
309       scanners.add(scanner);
310       ctx.addThread(scanner);
311     }
312 
313     ctx.startThreads();
314     ctx.waitFor(millisToRun);
315     ctx.stop();
316 
317     LOG.info("Loaders:");
318     LOG.info("  loaded " + loader.numBulkLoads.get());
319     LOG.info("  compations " + loader.numCompactions.get());
320 
321     LOG.info("Scanners:");
322     for (AtomicScanReader scanner : scanners) {
323       LOG.info("  scanned " + scanner.numScans.get());
324       LOG.info("  verified " + scanner.numRowsScanned.get() + " rows");
325     }
326   }
327 
328   /**
329    * Run test on an HBase instance for 5 minutes. This assumes that the table
330    * under test only has a single region.
331    */
332   public static void main(String args[]) throws Exception {
333     try {
334       Configuration c = HBaseConfiguration.create();
335       TestHRegionServerBulkLoad test = new TestHRegionServerBulkLoad();
336       test.setConf(c);
337       test.runAtomicBulkloadTest(TableName.valueOf("atomicTableTest"), 5 * 60 * 1000, 50);
338     } finally {
339       System.exit(0); // something hangs (believe it is lru threadpool)
340     }
341   }
342 
343   private void setConf(Configuration c) {
344     UTIL = new HBaseTestingUtility(c);
345   }
346 
347 }
348