1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.rest;
21
22 import java.io.ByteArrayInputStream;
23 import java.io.IOException;
24 import java.io.StringWriter;
25 import java.util.Iterator;
26 import java.util.Random;
27
28 import javax.xml.bind.JAXBContext;
29 import javax.xml.bind.JAXBException;
30 import javax.xml.bind.Marshaller;
31 import javax.xml.bind.Unmarshaller;
32
33 import org.apache.commons.httpclient.Header;
34 import org.apache.hadoop.conf.Configuration;
35 import org.apache.hadoop.hbase.*;
36 import org.apache.hadoop.hbase.client.HBaseAdmin;
37 import org.apache.hadoop.hbase.client.HTable;
38 import org.apache.hadoop.hbase.client.Put;
39 import org.apache.hadoop.hbase.client.Durability;
40 import org.apache.hadoop.hbase.rest.client.Client;
41 import org.apache.hadoop.hbase.rest.client.Cluster;
42 import org.apache.hadoop.hbase.rest.client.Response;
43 import org.apache.hadoop.hbase.rest.model.CellModel;
44 import org.apache.hadoop.hbase.rest.model.CellSetModel;
45 import org.apache.hadoop.hbase.rest.model.RowModel;
46 import org.apache.hadoop.hbase.rest.model.ScannerModel;
47 import org.apache.hadoop.hbase.util.Bytes;
48
49 import static org.junit.Assert.*;
50
51 import org.junit.AfterClass;
52 import org.junit.BeforeClass;
53 import org.junit.Test;
54 import org.junit.experimental.categories.Category;
55
56 @Category(MediumTests.class)
57 public class TestScannerResource {
58 private static final String TABLE = "TestScannerResource";
59 private static final String NONEXISTENT_TABLE = "ThisTableDoesNotExist";
60 private static final String CFA = "a";
61 private static final String CFB = "b";
62 private static final String COLUMN_1 = CFA + ":1";
63 private static final String COLUMN_2 = CFB + ":2";
64
65 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
66 private static final HBaseRESTTestingUtility REST_TEST_UTIL =
67 new HBaseRESTTestingUtility();
68 private static Client client;
69 private static JAXBContext context;
70 private static Marshaller marshaller;
71 private static Unmarshaller unmarshaller;
72 private static int expectedRows1;
73 private static int expectedRows2;
74 private static Configuration conf;
75
76 private static int insertData(String tableName, String column, double prob)
77 throws IOException {
78 Random rng = new Random();
79 int count = 0;
80 HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
81 byte[] k = new byte[3];
82 byte [][] famAndQf = KeyValue.parseColumn(Bytes.toBytes(column));
83 for (byte b1 = 'a'; b1 < 'z'; b1++) {
84 for (byte b2 = 'a'; b2 < 'z'; b2++) {
85 for (byte b3 = 'a'; b3 < 'z'; b3++) {
86 if (rng.nextDouble() < prob) {
87 k[0] = b1;
88 k[1] = b2;
89 k[2] = b3;
90 Put put = new Put(k);
91 put.setDurability(Durability.SKIP_WAL);
92 put.add(famAndQf[0], famAndQf[1], k);
93 table.put(put);
94 count++;
95 }
96 }
97 }
98 }
99 table.flushCommits();
100 return count;
101 }
102
103 private static int countCellSet(CellSetModel model) {
104 int count = 0;
105 Iterator<RowModel> rows = model.getRows().iterator();
106 while (rows.hasNext()) {
107 RowModel row = rows.next();
108 Iterator<CellModel> cells = row.getCells().iterator();
109 while (cells.hasNext()) {
110 cells.next();
111 count++;
112 }
113 }
114 return count;
115 }
116
117 private static int fullTableScan(ScannerModel model) throws IOException {
118 model.setBatch(100);
119 Response response = client.put("/" + TABLE + "/scanner",
120 Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
121 assertEquals(response.getCode(), 201);
122 String scannerURI = response.getLocation();
123 assertNotNull(scannerURI);
124 int count = 0;
125 while (true) {
126 response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
127 assertTrue(response.getCode() == 200 || response.getCode() == 204);
128 if (response.getCode() == 200) {
129 assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
130 CellSetModel cellSet = new CellSetModel();
131 cellSet.getObjectFromMessage(response.getBody());
132 Iterator<RowModel> rows = cellSet.getRows().iterator();
133 while (rows.hasNext()) {
134 RowModel row = rows.next();
135 Iterator<CellModel> cells = row.getCells().iterator();
136 while (cells.hasNext()) {
137 cells.next();
138 count++;
139 }
140 }
141 } else {
142 break;
143 }
144 }
145
146 response = client.delete(scannerURI);
147 assertEquals(response.getCode(), 200);
148 return count;
149 }
150
151 @BeforeClass
152 public static void setUpBeforeClass() throws Exception {
153 conf = TEST_UTIL.getConfiguration();
154 TEST_UTIL.startMiniCluster();
155 REST_TEST_UTIL.startServletContainer(conf);
156 client = new Client(new Cluster().add("localhost",
157 REST_TEST_UTIL.getServletPort()));
158 context = JAXBContext.newInstance(
159 CellModel.class,
160 CellSetModel.class,
161 RowModel.class,
162 ScannerModel.class);
163 marshaller = context.createMarshaller();
164 unmarshaller = context.createUnmarshaller();
165 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
166 if (admin.tableExists(TABLE)) {
167 return;
168 }
169 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE));
170 htd.addFamily(new HColumnDescriptor(CFA));
171 htd.addFamily(new HColumnDescriptor(CFB));
172 admin.createTable(htd);
173 expectedRows1 = insertData(TABLE, COLUMN_1, 1.0);
174 expectedRows2 = insertData(TABLE, COLUMN_2, 0.5);
175 }
176
177 @AfterClass
178 public static void tearDownAfterClass() throws Exception {
179 REST_TEST_UTIL.shutdownServletContainer();
180 TEST_UTIL.shutdownMiniCluster();
181 }
182
183 @Test
184 public void testSimpleScannerXML() throws IOException, JAXBException {
185 final int BATCH_SIZE = 5;
186
187 ScannerModel model = new ScannerModel();
188 model.setBatch(BATCH_SIZE);
189 model.addColumn(Bytes.toBytes(COLUMN_1));
190 StringWriter writer = new StringWriter();
191 marshaller.marshal(model, writer);
192 byte[] body = Bytes.toBytes(writer.toString());
193
194
195 conf.set("hbase.rest.readonly", "true");
196 Response response = client.put("/" + TABLE + "/scanner",
197 Constants.MIMETYPE_XML, body);
198 assertEquals(response.getCode(), 403);
199 String scannerURI = response.getLocation();
200 assertNull(scannerURI);
201
202
203 conf.set("hbase.rest.readonly", "false");
204 response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML,
205 body);
206 assertEquals(response.getCode(), 201);
207 scannerURI = response.getLocation();
208 assertNotNull(scannerURI);
209
210
211 response = client.get(scannerURI, Constants.MIMETYPE_XML);
212 assertEquals(response.getCode(), 200);
213 assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
214 CellSetModel cellSet = (CellSetModel)
215 unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
216
217 assertEquals(countCellSet(cellSet), BATCH_SIZE);
218
219
220 conf.set("hbase.rest.readonly", "true");
221 response = client.delete(scannerURI);
222 assertEquals(response.getCode(), 403);
223
224
225 conf.set("hbase.rest.readonly", "false");
226 response = client.delete(scannerURI);
227 assertEquals(response.getCode(), 200);
228 }
229
230 @Test
231 public void testSimpleScannerPB() throws IOException {
232 final int BATCH_SIZE = 10;
233
234 ScannerModel model = new ScannerModel();
235 model.setBatch(BATCH_SIZE);
236 model.addColumn(Bytes.toBytes(COLUMN_1));
237
238
239 conf.set("hbase.rest.readonly", "true");
240 Response response = client.put("/" + TABLE + "/scanner",
241 Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
242 assertEquals(response.getCode(), 403);
243 String scannerURI = response.getLocation();
244 assertNull(scannerURI);
245
246
247 conf.set("hbase.rest.readonly", "false");
248 response = client.put("/" + TABLE + "/scanner",
249 Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
250 assertEquals(response.getCode(), 201);
251 scannerURI = response.getLocation();
252 assertNotNull(scannerURI);
253
254
255 response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
256 assertEquals(response.getCode(), 200);
257 assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
258 CellSetModel cellSet = new CellSetModel();
259 cellSet.getObjectFromMessage(response.getBody());
260
261 assertEquals(countCellSet(cellSet), BATCH_SIZE);
262
263
264 conf.set("hbase.rest.readonly", "true");
265 response = client.delete(scannerURI);
266 assertEquals(response.getCode(), 403);
267
268
269 conf.set("hbase.rest.readonly", "false");
270 response = client.delete(scannerURI);
271 assertEquals(response.getCode(), 200);
272 }
273
274 @Test
275 public void testSimpleScannerBinary() throws IOException {
276
277 ScannerModel model = new ScannerModel();
278 model.setBatch(1);
279 model.addColumn(Bytes.toBytes(COLUMN_1));
280
281
282 conf.set("hbase.rest.readonly", "true");
283 Response response = client.put("/" + TABLE + "/scanner",
284 Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
285 assertEquals(response.getCode(), 403);
286 String scannerURI = response.getLocation();
287 assertNull(scannerURI);
288
289
290 conf.set("hbase.rest.readonly", "false");
291 response = client.put("/" + TABLE + "/scanner",
292 Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
293 assertEquals(response.getCode(), 201);
294 scannerURI = response.getLocation();
295 assertNotNull(scannerURI);
296
297
298 response = client.get(scannerURI, Constants.MIMETYPE_BINARY);
299 assertEquals(response.getCode(), 200);
300 assertEquals(Constants.MIMETYPE_BINARY, response.getHeader("content-type"));
301
302 assertTrue(response.getBody().length > 0);
303
304 boolean foundRowHeader = false, foundColumnHeader = false,
305 foundTimestampHeader = false;
306 for (Header header: response.getHeaders()) {
307 if (header.getName().equals("X-Row")) {
308 foundRowHeader = true;
309 } else if (header.getName().equals("X-Column")) {
310 foundColumnHeader = true;
311 } else if (header.getName().equals("X-Timestamp")) {
312 foundTimestampHeader = true;
313 }
314 }
315 assertTrue(foundRowHeader);
316 assertTrue(foundColumnHeader);
317 assertTrue(foundTimestampHeader);
318
319
320 conf.set("hbase.rest.readonly", "true");
321 response = client.delete(scannerURI);
322 assertEquals(response.getCode(), 403);
323
324
325 conf.set("hbase.rest.readonly", "false");
326 response = client.delete(scannerURI);
327 assertEquals(response.getCode(), 200);
328 }
329
330 @Test
331 public void testFullTableScan() throws IOException {
332 ScannerModel model = new ScannerModel();
333 model.addColumn(Bytes.toBytes(COLUMN_1));
334 assertEquals(fullTableScan(model), expectedRows1);
335
336 model = new ScannerModel();
337 model.addColumn(Bytes.toBytes(COLUMN_2));
338 assertEquals(fullTableScan(model), expectedRows2);
339 }
340
341 @Test
342 public void testTableDoesNotExist() throws IOException, JAXBException {
343 ScannerModel model = new ScannerModel();
344 StringWriter writer = new StringWriter();
345 marshaller.marshal(model, writer);
346 byte[] body = Bytes.toBytes(writer.toString());
347 Response response = client.put("/" + NONEXISTENT_TABLE +
348 "/scanner", Constants.MIMETYPE_XML, body);
349 assertEquals(response.getCode(), 404);
350 }
351
352 }
353