1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package org.apache.hadoop.hbase.util;
18
19 import static org.junit.Assert.assertEquals;
20
21 import java.io.IOException;
22 import java.util.ArrayList;
23 import java.util.Collection;
24 import java.util.List;
25
26 import org.apache.commons.logging.Log;
27 import org.apache.commons.logging.LogFactory;
28 import org.apache.hadoop.conf.Configuration;
29 import org.apache.hadoop.hbase.HBaseTestingUtility;
30 import org.apache.hadoop.hbase.HColumnDescriptor;
31 import org.apache.hadoop.hbase.HConstants;
32 import org.apache.hadoop.hbase.LargeTests;
33 import org.apache.hadoop.hbase.TableNotFoundException;
34 import org.apache.hadoop.hbase.client.HBaseAdmin;
35 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
36 import org.apache.hadoop.hbase.io.hfile.Compression;
37 import org.junit.After;
38 import org.junit.Before;
39 import org.junit.Test;
40 import org.junit.experimental.categories.Category;
41 import org.junit.runner.RunWith;
42 import org.junit.runners.Parameterized;
43 import org.junit.runners.Parameterized.Parameters;
44
45
46
47
48
49 @Category(LargeTests.class)
50 @RunWith(Parameterized.class)
51 public class TestMiniClusterLoadSequential {
52
53 private static final Log LOG = LogFactory.getLog(
54 TestMiniClusterLoadSequential.class);
55
56 protected static final byte[] TABLE = Bytes.toBytes("load_test_tbl");
57 protected static final byte[] CF = Bytes.toBytes("load_test_cf");
58 protected static final int NUM_THREADS = 8;
59 protected static final int NUM_RS = 2;
60 protected static final int TIMEOUT_MS = 180000;
61 protected static final HBaseTestingUtility TEST_UTIL =
62 new HBaseTestingUtility();
63
64 protected final Configuration conf = TEST_UTIL.getConfiguration();
65 protected final boolean isMultiPut;
66 protected final DataBlockEncoding dataBlockEncoding;
67
68 protected MultiThreadedWriter writerThreads;
69 protected MultiThreadedReader readerThreads;
70 protected int numKeys;
71
72 protected Compression.Algorithm compression = Compression.Algorithm.NONE;
73
74 public TestMiniClusterLoadSequential(boolean isMultiPut,
75 DataBlockEncoding dataBlockEncoding) {
76 this.isMultiPut = isMultiPut;
77 this.dataBlockEncoding = dataBlockEncoding;
78 conf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024 * 1024);
79 }
80
81 @Parameters
82 public static Collection<Object[]> parameters() {
83 List<Object[]> parameters = new ArrayList<Object[]>();
84 for (boolean multiPut : new boolean[]{false, true}) {
85 for (DataBlockEncoding dataBlockEncoding : new DataBlockEncoding[] {
86 DataBlockEncoding.NONE, DataBlockEncoding.PREFIX }) {
87 parameters.add(new Object[]{multiPut, dataBlockEncoding});
88 }
89 }
90 return parameters;
91 }
92
93 @Before
94 public void setUp() throws Exception {
95 LOG.debug("Test setup: isMultiPut=" + isMultiPut);
96 TEST_UTIL.startMiniCluster(1, NUM_RS);
97 }
98
99 @After
100 public void tearDown() throws Exception {
101 LOG.debug("Test teardown: isMultiPut=" + isMultiPut);
102 TEST_UTIL.shutdownMiniCluster();
103 }
104
105 @Test(timeout=TIMEOUT_MS)
106 public void loadTest() throws Exception {
107 prepareForLoadTest();
108 runLoadTestOnExistingTable();
109 }
110
111 protected void runLoadTestOnExistingTable() throws IOException {
112 writerThreads.start(0, numKeys, NUM_THREADS);
113 writerThreads.waitForFinish();
114 assertEquals(0, writerThreads.getNumWriteFailures());
115
116 readerThreads.start(0, numKeys, NUM_THREADS);
117 readerThreads.waitForFinish();
118 assertEquals(0, readerThreads.getNumReadFailures());
119 assertEquals(0, readerThreads.getNumReadErrors());
120 assertEquals(numKeys, readerThreads.getNumKeysVerified());
121 }
122
123 protected void prepareForLoadTest() throws IOException {
124 LOG.info("Starting load test: dataBlockEncoding=" + dataBlockEncoding +
125 ", isMultiPut=" + isMultiPut);
126 numKeys = numKeys();
127 HBaseAdmin admin = new HBaseAdmin(conf);
128 while (admin.getClusterStatus().getServers().size() < NUM_RS) {
129 LOG.info("Sleeping until " + NUM_RS + " RSs are online");
130 Threads.sleepWithoutInterrupt(1000);
131 }
132 admin.close();
133
134 int numRegions = HBaseTestingUtility.createPreSplitLoadTestTable(conf,
135 TABLE, CF, compression, dataBlockEncoding);
136
137 TEST_UTIL.waitUntilAllRegionsAssigned(numRegions);
138
139 writerThreads = new MultiThreadedWriter(conf, TABLE, CF);
140 writerThreads.setMultiPut(isMultiPut);
141 readerThreads = new MultiThreadedReader(conf, TABLE, CF, 100);
142 }
143
144 protected int numKeys() {
145 return 10000;
146 }
147
148 protected HColumnDescriptor getColumnDesc(HBaseAdmin admin)
149 throws TableNotFoundException, IOException {
150 return admin.getTableDescriptor(TABLE).getFamily(CF);
151 }
152
153 }