1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 package org.apache.hadoop.hbase.rest;
22
23 import java.io.ByteArrayInputStream;
24 import java.io.IOException;
25 import java.io.StringWriter;
26 import java.util.Iterator;
27 import java.util.Random;
28
29 import javax.xml.bind.JAXBContext;
30 import javax.xml.bind.JAXBException;
31 import javax.xml.bind.Marshaller;
32 import javax.xml.bind.Unmarshaller;
33
34 import org.apache.commons.httpclient.Header;
35 import org.apache.hadoop.conf.Configuration;
36 import org.apache.hadoop.hbase.HBaseTestingUtility;
37 import org.apache.hadoop.hbase.HColumnDescriptor;
38 import org.apache.hadoop.hbase.HTableDescriptor;
39 import org.apache.hadoop.hbase.KeyValue;
40 import org.apache.hadoop.hbase.client.HBaseAdmin;
41 import org.apache.hadoop.hbase.client.HTable;
42 import org.apache.hadoop.hbase.client.Put;
43 import org.apache.hadoop.hbase.rest.client.Client;
44 import org.apache.hadoop.hbase.rest.client.Cluster;
45 import org.apache.hadoop.hbase.rest.client.Response;
46 import org.apache.hadoop.hbase.rest.model.CellModel;
47 import org.apache.hadoop.hbase.rest.model.CellSetModel;
48 import org.apache.hadoop.hbase.rest.model.RowModel;
49 import org.apache.hadoop.hbase.rest.model.ScannerModel;
50 import org.apache.hadoop.hbase.util.Bytes;
51
52 import static org.junit.Assert.*;
53 import org.junit.AfterClass;
54 import org.junit.BeforeClass;
55 import org.junit.Test;
56
57 public class TestScannerResource {
58 private static final String TABLE = "TestScannerResource";
59 private static final String NONEXISTENT_TABLE = "ThisTableDoesNotExist";
60 private static final String CFA = "a";
61 private static final String CFB = "b";
62 private static final String COLUMN_1 = CFA + ":1";
63 private static final String COLUMN_2 = CFB + ":2";
64
65 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
66 private static final HBaseRESTTestingUtility REST_TEST_UTIL =
67 new HBaseRESTTestingUtility();
68 private static Client client;
69 private static JAXBContext context;
70 private static Marshaller marshaller;
71 private static Unmarshaller unmarshaller;
72 private static int expectedRows1;
73 private static int expectedRows2;
74 private static Configuration conf;
75
76 private static int insertData(String tableName, String column, double prob)
77 throws IOException {
78 Random rng = new Random();
79 int count = 0;
80 HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
81 byte[] k = new byte[3];
82 byte [][] famAndQf = KeyValue.parseColumn(Bytes.toBytes(column));
83 for (byte b1 = 'a'; b1 < 'z'; b1++) {
84 for (byte b2 = 'a'; b2 < 'z'; b2++) {
85 for (byte b3 = 'a'; b3 < 'z'; b3++) {
86 if (rng.nextDouble() < prob) {
87 k[0] = b1;
88 k[1] = b2;
89 k[2] = b3;
90 Put put = new Put(k);
91 put.add(famAndQf[0], famAndQf[1], k);
92 table.put(put);
93 count++;
94 }
95 }
96 }
97 }
98 table.flushCommits();
99 return count;
100 }
101
102 private static int countCellSet(CellSetModel model) {
103 int count = 0;
104 Iterator<RowModel> rows = model.getRows().iterator();
105 while (rows.hasNext()) {
106 RowModel row = rows.next();
107 Iterator<CellModel> cells = row.getCells().iterator();
108 while (cells.hasNext()) {
109 cells.next();
110 count++;
111 }
112 }
113 return count;
114 }
115
116 private static int fullTableScan(ScannerModel model) throws IOException {
117 model.setBatch(100);
118 Response response = client.put("/" + TABLE + "/scanner",
119 Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
120 assertEquals(response.getCode(), 201);
121 String scannerURI = response.getLocation();
122 assertNotNull(scannerURI);
123 int count = 0;
124 while (true) {
125 response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
126 assertTrue(response.getCode() == 200 || response.getCode() == 204);
127 if (response.getCode() == 200) {
128 CellSetModel cellSet = new CellSetModel();
129 cellSet.getObjectFromMessage(response.getBody());
130 Iterator<RowModel> rows = cellSet.getRows().iterator();
131 while (rows.hasNext()) {
132 RowModel row = rows.next();
133 Iterator<CellModel> cells = row.getCells().iterator();
134 while (cells.hasNext()) {
135 cells.next();
136 count++;
137 }
138 }
139 } else {
140 break;
141 }
142 }
143
144 response = client.delete(scannerURI);
145 assertEquals(response.getCode(), 200);
146 return count;
147 }
148
149 @BeforeClass
150 public static void setUpBeforeClass() throws Exception {
151 conf = TEST_UTIL.getConfiguration();
152 TEST_UTIL.startMiniCluster(3);
153 REST_TEST_UTIL.startServletContainer(conf);
154 client = new Client(new Cluster().add("localhost",
155 REST_TEST_UTIL.getServletPort()));
156 context = JAXBContext.newInstance(
157 CellModel.class,
158 CellSetModel.class,
159 RowModel.class,
160 ScannerModel.class);
161 marshaller = context.createMarshaller();
162 unmarshaller = context.createUnmarshaller();
163 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
164 if (admin.tableExists(TABLE)) {
165 return;
166 }
167 HTableDescriptor htd = new HTableDescriptor(TABLE);
168 htd.addFamily(new HColumnDescriptor(CFA));
169 htd.addFamily(new HColumnDescriptor(CFB));
170 admin.createTable(htd);
171 expectedRows1 = insertData(TABLE, COLUMN_1, 1.0);
172 expectedRows2 = insertData(TABLE, COLUMN_2, 0.5);
173 }
174
175 @AfterClass
176 public static void tearDownAfterClass() throws Exception {
177 REST_TEST_UTIL.shutdownServletContainer();
178 TEST_UTIL.shutdownMiniCluster();
179 }
180
181 @Test
182 public void testSimpleScannerXML() throws IOException, JAXBException {
183 final int BATCH_SIZE = 5;
184
185 ScannerModel model = new ScannerModel();
186 model.setBatch(BATCH_SIZE);
187 model.addColumn(Bytes.toBytes(COLUMN_1));
188 StringWriter writer = new StringWriter();
189 marshaller.marshal(model, writer);
190 byte[] body = Bytes.toBytes(writer.toString());
191
192
193 conf.set("hbase.rest.readonly", "true");
194 Response response = client.put("/" + TABLE + "/scanner",
195 Constants.MIMETYPE_XML, body);
196 assertEquals(response.getCode(), 403);
197 String scannerURI = response.getLocation();
198 assertNull(scannerURI);
199
200
201 conf.set("hbase.rest.readonly", "false");
202 response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML,
203 body);
204 assertEquals(response.getCode(), 201);
205 scannerURI = response.getLocation();
206 assertNotNull(scannerURI);
207
208
209 response = client.get(scannerURI, Constants.MIMETYPE_XML);
210 assertEquals(response.getCode(), 200);
211 CellSetModel cellSet = (CellSetModel)
212 unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
213
214 assertEquals(countCellSet(cellSet), BATCH_SIZE);
215
216
217 conf.set("hbase.rest.readonly", "true");
218 response = client.delete(scannerURI);
219 assertEquals(response.getCode(), 403);
220
221
222 conf.set("hbase.rest.readonly", "false");
223 response = client.delete(scannerURI);
224 assertEquals(response.getCode(), 200);
225 }
226
227 @Test
228 public void testSimpleScannerPB() throws IOException {
229 final int BATCH_SIZE = 10;
230
231 ScannerModel model = new ScannerModel();
232 model.setBatch(BATCH_SIZE);
233 model.addColumn(Bytes.toBytes(COLUMN_1));
234
235
236 conf.set("hbase.rest.readonly", "true");
237 Response response = client.put("/" + TABLE + "/scanner",
238 Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
239 assertEquals(response.getCode(), 403);
240 String scannerURI = response.getLocation();
241 assertNull(scannerURI);
242
243
244 conf.set("hbase.rest.readonly", "false");
245 response = client.put("/" + TABLE + "/scanner",
246 Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
247 assertEquals(response.getCode(), 201);
248 scannerURI = response.getLocation();
249 assertNotNull(scannerURI);
250
251
252 response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
253 assertEquals(response.getCode(), 200);
254 CellSetModel cellSet = new CellSetModel();
255 cellSet.getObjectFromMessage(response.getBody());
256
257 assertEquals(countCellSet(cellSet), BATCH_SIZE);
258
259
260 conf.set("hbase.rest.readonly", "true");
261 response = client.delete(scannerURI);
262 assertEquals(response.getCode(), 403);
263
264
265 conf.set("hbase.rest.readonly", "false");
266 response = client.delete(scannerURI);
267 assertEquals(response.getCode(), 200);
268 }
269
270 @Test
271 public void testSimpleScannerBinary() throws IOException {
272
273 ScannerModel model = new ScannerModel();
274 model.setBatch(1);
275 model.addColumn(Bytes.toBytes(COLUMN_1));
276
277
278 conf.set("hbase.rest.readonly", "true");
279 Response response = client.put("/" + TABLE + "/scanner",
280 Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
281 assertEquals(response.getCode(), 403);
282 String scannerURI = response.getLocation();
283 assertNull(scannerURI);
284
285
286 conf.set("hbase.rest.readonly", "false");
287 response = client.put("/" + TABLE + "/scanner",
288 Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
289 assertEquals(response.getCode(), 201);
290 scannerURI = response.getLocation();
291 assertNotNull(scannerURI);
292
293
294 response = client.get(scannerURI, Constants.MIMETYPE_BINARY);
295 assertEquals(response.getCode(), 200);
296
297 assertTrue(response.getBody().length > 0);
298
299 boolean foundRowHeader = false, foundColumnHeader = false,
300 foundTimestampHeader = false;
301 for (Header header: response.getHeaders()) {
302 if (header.getName().equals("X-Row")) {
303 foundRowHeader = true;
304 } else if (header.getName().equals("X-Column")) {
305 foundColumnHeader = true;
306 } else if (header.getName().equals("X-Timestamp")) {
307 foundTimestampHeader = true;
308 }
309 }
310 assertTrue(foundRowHeader);
311 assertTrue(foundColumnHeader);
312 assertTrue(foundTimestampHeader);
313
314
315 conf.set("hbase.rest.readonly", "true");
316 response = client.delete(scannerURI);
317 assertEquals(response.getCode(), 403);
318
319
320 conf.set("hbase.rest.readonly", "false");
321 response = client.delete(scannerURI);
322 assertEquals(response.getCode(), 200);
323 }
324
325 @Test
326 public void testFullTableScan() throws IOException {
327 ScannerModel model = new ScannerModel();
328 model.addColumn(Bytes.toBytes(COLUMN_1));
329 assertEquals(fullTableScan(model), expectedRows1);
330
331 model = new ScannerModel();
332 model.addColumn(Bytes.toBytes(COLUMN_2));
333 assertEquals(fullTableScan(model), expectedRows2);
334 }
335
336 @Test
337 public void testTableDoesNotExist() throws IOException, JAXBException {
338 ScannerModel model = new ScannerModel();
339 StringWriter writer = new StringWriter();
340 marshaller.marshal(model, writer);
341 byte[] body = Bytes.toBytes(writer.toString());
342 Response response = client.put("/" + NONEXISTENT_TABLE +
343 "/scanner", Constants.MIMETYPE_XML, body);
344 assertEquals(response.getCode(), 404);
345 }
346 }