1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 package org.apache.hadoop.hbase.rest;
22
23 import java.io.ByteArrayInputStream;
24 import java.io.IOException;
25 import java.io.StringWriter;
26 import java.util.Iterator;
27 import java.util.Random;
28
29 import javax.xml.bind.JAXBContext;
30 import javax.xml.bind.JAXBException;
31 import javax.xml.bind.Marshaller;
32 import javax.xml.bind.Unmarshaller;
33
34 import org.apache.commons.httpclient.Header;
35 import org.apache.hadoop.conf.Configuration;
36 import org.apache.hadoop.hbase.*;
37 import org.apache.hadoop.hbase.client.HBaseAdmin;
38 import org.apache.hadoop.hbase.client.HTable;
39 import org.apache.hadoop.hbase.client.Put;
40 import org.apache.hadoop.hbase.rest.client.Client;
41 import org.apache.hadoop.hbase.rest.client.Cluster;
42 import org.apache.hadoop.hbase.rest.client.Response;
43 import org.apache.hadoop.hbase.rest.model.CellModel;
44 import org.apache.hadoop.hbase.rest.model.CellSetModel;
45 import org.apache.hadoop.hbase.rest.model.RowModel;
46 import org.apache.hadoop.hbase.rest.model.ScannerModel;
47 import org.apache.hadoop.hbase.util.Bytes;
48
49 import static org.junit.Assert.*;
50 import org.junit.AfterClass;
51 import org.junit.BeforeClass;
52 import org.junit.Test;
53 import org.junit.experimental.categories.Category;
54
55 @Category(MediumTests.class)
56 public class TestScannerResource {
57 private static final String TABLE = "TestScannerResource";
58 private static final String NONEXISTENT_TABLE = "ThisTableDoesNotExist";
59 private static final String CFA = "a";
60 private static final String CFB = "b";
61 private static final String COLUMN_1 = CFA + ":1";
62 private static final String COLUMN_2 = CFB + ":2";
63
64 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
65 private static final HBaseRESTTestingUtility REST_TEST_UTIL =
66 new HBaseRESTTestingUtility();
67 private static Client client;
68 private static JAXBContext context;
69 private static Marshaller marshaller;
70 private static Unmarshaller unmarshaller;
71 private static int expectedRows1;
72 private static int expectedRows2;
73 private static Configuration conf;
74
75 private static int insertData(String tableName, String column, double prob)
76 throws IOException {
77 Random rng = new Random();
78 int count = 0;
79 HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
80 byte[] k = new byte[3];
81 byte [][] famAndQf = KeyValue.parseColumn(Bytes.toBytes(column));
82 for (byte b1 = 'a'; b1 < 'z'; b1++) {
83 for (byte b2 = 'a'; b2 < 'z'; b2++) {
84 for (byte b3 = 'a'; b3 < 'z'; b3++) {
85 if (rng.nextDouble() < prob) {
86 k[0] = b1;
87 k[1] = b2;
88 k[2] = b3;
89 Put put = new Put(k);
90 put.setWriteToWAL(false);
91 put.add(famAndQf[0], famAndQf[1], k);
92 table.put(put);
93 count++;
94 }
95 }
96 }
97 }
98 table.flushCommits();
99 return count;
100 }
101
102 private static int countCellSet(CellSetModel model) {
103 int count = 0;
104 Iterator<RowModel> rows = model.getRows().iterator();
105 while (rows.hasNext()) {
106 RowModel row = rows.next();
107 Iterator<CellModel> cells = row.getCells().iterator();
108 while (cells.hasNext()) {
109 cells.next();
110 count++;
111 }
112 }
113 return count;
114 }
115
116 private static int fullTableScan(ScannerModel model) throws IOException {
117 model.setBatch(100);
118 Response response = client.put("/" + TABLE + "/scanner",
119 Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
120 assertEquals(response.getCode(), 201);
121 String scannerURI = response.getLocation();
122 assertNotNull(scannerURI);
123 int count = 0;
124 while (true) {
125 response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
126 assertTrue(response.getCode() == 200 || response.getCode() == 204);
127 if (response.getCode() == 200) {
128 assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
129 CellSetModel cellSet = new CellSetModel();
130 cellSet.getObjectFromMessage(response.getBody());
131 Iterator<RowModel> rows = cellSet.getRows().iterator();
132 while (rows.hasNext()) {
133 RowModel row = rows.next();
134 Iterator<CellModel> cells = row.getCells().iterator();
135 while (cells.hasNext()) {
136 cells.next();
137 count++;
138 }
139 }
140 } else {
141 break;
142 }
143 }
144
145 response = client.delete(scannerURI);
146 assertEquals(response.getCode(), 200);
147 return count;
148 }
149
150 @BeforeClass
151 public static void setUpBeforeClass() throws Exception {
152 conf = TEST_UTIL.getConfiguration();
153 TEST_UTIL.startMiniCluster();
154 REST_TEST_UTIL.startServletContainer(conf);
155 client = new Client(new Cluster().add("localhost",
156 REST_TEST_UTIL.getServletPort()));
157 context = JAXBContext.newInstance(
158 CellModel.class,
159 CellSetModel.class,
160 RowModel.class,
161 ScannerModel.class);
162 marshaller = context.createMarshaller();
163 unmarshaller = context.createUnmarshaller();
164 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
165 if (admin.tableExists(TABLE)) {
166 return;
167 }
168 HTableDescriptor htd = new HTableDescriptor(TABLE);
169 htd.addFamily(new HColumnDescriptor(CFA));
170 htd.addFamily(new HColumnDescriptor(CFB));
171 admin.createTable(htd);
172 expectedRows1 = insertData(TABLE, COLUMN_1, 1.0);
173 expectedRows2 = insertData(TABLE, COLUMN_2, 0.5);
174 }
175
176 @AfterClass
177 public static void tearDownAfterClass() throws Exception {
178 REST_TEST_UTIL.shutdownServletContainer();
179 TEST_UTIL.shutdownMiniCluster();
180 }
181
182 @Test
183 public void testSimpleScannerXML() throws IOException, JAXBException {
184 final int BATCH_SIZE = 5;
185
186 ScannerModel model = new ScannerModel();
187 model.setBatch(BATCH_SIZE);
188 model.addColumn(Bytes.toBytes(COLUMN_1));
189 StringWriter writer = new StringWriter();
190 marshaller.marshal(model, writer);
191 byte[] body = Bytes.toBytes(writer.toString());
192
193
194 conf.set("hbase.rest.readonly", "true");
195 Response response = client.put("/" + TABLE + "/scanner",
196 Constants.MIMETYPE_XML, body);
197 assertEquals(response.getCode(), 403);
198 String scannerURI = response.getLocation();
199 assertNull(scannerURI);
200
201
202 conf.set("hbase.rest.readonly", "false");
203 response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML,
204 body);
205 assertEquals(response.getCode(), 201);
206 scannerURI = response.getLocation();
207 assertNotNull(scannerURI);
208
209
210 response = client.get(scannerURI, Constants.MIMETYPE_XML);
211 assertEquals(response.getCode(), 200);
212 assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
213 CellSetModel cellSet = (CellSetModel)
214 unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
215
216 assertEquals(countCellSet(cellSet), BATCH_SIZE);
217
218
219 conf.set("hbase.rest.readonly", "true");
220 response = client.delete(scannerURI);
221 assertEquals(response.getCode(), 403);
222
223
224 conf.set("hbase.rest.readonly", "false");
225 response = client.delete(scannerURI);
226 assertEquals(response.getCode(), 200);
227 }
228
229 @Test
230 public void testSimpleScannerPB() throws IOException {
231 final int BATCH_SIZE = 10;
232
233 ScannerModel model = new ScannerModel();
234 model.setBatch(BATCH_SIZE);
235 model.addColumn(Bytes.toBytes(COLUMN_1));
236
237
238 conf.set("hbase.rest.readonly", "true");
239 Response response = client.put("/" + TABLE + "/scanner",
240 Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
241 assertEquals(response.getCode(), 403);
242 String scannerURI = response.getLocation();
243 assertNull(scannerURI);
244
245
246 conf.set("hbase.rest.readonly", "false");
247 response = client.put("/" + TABLE + "/scanner",
248 Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
249 assertEquals(response.getCode(), 201);
250 scannerURI = response.getLocation();
251 assertNotNull(scannerURI);
252
253
254 response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
255 assertEquals(response.getCode(), 200);
256 assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
257 CellSetModel cellSet = new CellSetModel();
258 cellSet.getObjectFromMessage(response.getBody());
259
260 assertEquals(countCellSet(cellSet), BATCH_SIZE);
261
262
263 conf.set("hbase.rest.readonly", "true");
264 response = client.delete(scannerURI);
265 assertEquals(response.getCode(), 403);
266
267
268 conf.set("hbase.rest.readonly", "false");
269 response = client.delete(scannerURI);
270 assertEquals(response.getCode(), 200);
271 }
272
273 @Test
274 public void testSimpleScannerBinary() throws IOException {
275
276 ScannerModel model = new ScannerModel();
277 model.setBatch(1);
278 model.addColumn(Bytes.toBytes(COLUMN_1));
279
280
281 conf.set("hbase.rest.readonly", "true");
282 Response response = client.put("/" + TABLE + "/scanner",
283 Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
284 assertEquals(response.getCode(), 403);
285 String scannerURI = response.getLocation();
286 assertNull(scannerURI);
287
288
289 conf.set("hbase.rest.readonly", "false");
290 response = client.put("/" + TABLE + "/scanner",
291 Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
292 assertEquals(response.getCode(), 201);
293 scannerURI = response.getLocation();
294 assertNotNull(scannerURI);
295
296
297 response = client.get(scannerURI, Constants.MIMETYPE_BINARY);
298 assertEquals(response.getCode(), 200);
299 assertEquals(Constants.MIMETYPE_BINARY, response.getHeader("content-type"));
300
301 assertTrue(response.getBody().length > 0);
302
303 boolean foundRowHeader = false, foundColumnHeader = false,
304 foundTimestampHeader = false;
305 for (Header header: response.getHeaders()) {
306 if (header.getName().equals("X-Row")) {
307 foundRowHeader = true;
308 } else if (header.getName().equals("X-Column")) {
309 foundColumnHeader = true;
310 } else if (header.getName().equals("X-Timestamp")) {
311 foundTimestampHeader = true;
312 }
313 }
314 assertTrue(foundRowHeader);
315 assertTrue(foundColumnHeader);
316 assertTrue(foundTimestampHeader);
317
318
319 conf.set("hbase.rest.readonly", "true");
320 response = client.delete(scannerURI);
321 assertEquals(response.getCode(), 403);
322
323
324 conf.set("hbase.rest.readonly", "false");
325 response = client.delete(scannerURI);
326 assertEquals(response.getCode(), 200);
327 }
328
329 @Test
330 public void testFullTableScan() throws IOException {
331 ScannerModel model = new ScannerModel();
332 model.addColumn(Bytes.toBytes(COLUMN_1));
333 assertEquals(fullTableScan(model), expectedRows1);
334
335 model = new ScannerModel();
336 model.addColumn(Bytes.toBytes(COLUMN_2));
337 assertEquals(fullTableScan(model), expectedRows2);
338 }
339
340 @Test
341 public void testTableDoesNotExist() throws IOException, JAXBException {
342 ScannerModel model = new ScannerModel();
343 StringWriter writer = new StringWriter();
344 marshaller.marshal(model, writer);
345 byte[] body = Bytes.toBytes(writer.toString());
346 Response response = client.put("/" + NONEXISTENT_TABLE +
347 "/scanner", Constants.MIMETYPE_XML, body);
348 assertEquals(response.getCode(), 404);
349 }
350
351 @org.junit.Rule
352 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
353 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
354 }
355