1   /*
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  
21  package org.apache.hadoop.hbase.rest;
22  
23  import java.io.ByteArrayInputStream;
24  import java.io.IOException;
25  import java.io.StringWriter;
26  import java.util.Iterator;
27  import java.util.Random;
28  
29  import javax.xml.bind.JAXBContext;
30  import javax.xml.bind.JAXBException;
31  import javax.xml.bind.Marshaller;
32  import javax.xml.bind.Unmarshaller;
33  
34  import org.apache.commons.httpclient.Header;
35  import org.apache.hadoop.conf.Configuration;
36  import org.apache.hadoop.hbase.HBaseTestingUtility;
37  import org.apache.hadoop.hbase.HColumnDescriptor;
38  import org.apache.hadoop.hbase.HTableDescriptor;
39  import org.apache.hadoop.hbase.KeyValue;
40  import org.apache.hadoop.hbase.client.HBaseAdmin;
41  import org.apache.hadoop.hbase.client.HTable;
42  import org.apache.hadoop.hbase.client.Put;
43  import org.apache.hadoop.hbase.rest.client.Client;
44  import org.apache.hadoop.hbase.rest.client.Cluster;
45  import org.apache.hadoop.hbase.rest.client.Response;
46  import org.apache.hadoop.hbase.rest.model.CellModel;
47  import org.apache.hadoop.hbase.rest.model.CellSetModel;
48  import org.apache.hadoop.hbase.rest.model.RowModel;
49  import org.apache.hadoop.hbase.rest.model.ScannerModel;
50  import org.apache.hadoop.hbase.util.Bytes;
51  
52  import static org.junit.Assert.*;
53  import org.junit.AfterClass;
54  import org.junit.BeforeClass;
55  import org.junit.Test;
56  
57  public class TestScannerResource {
58    private static final String TABLE = "TestScannerResource";
59    private static final String NONEXISTENT_TABLE = "ThisTableDoesNotExist";
60    private static final String CFA = "a";
61    private static final String CFB = "b";
62    private static final String COLUMN_1 = CFA + ":1";
63    private static final String COLUMN_2 = CFB + ":2";
64  
65    private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
66    private static final HBaseRESTTestingUtility REST_TEST_UTIL = 
67      new HBaseRESTTestingUtility();
68    private static Client client;
69    private static JAXBContext context;
70    private static Marshaller marshaller;
71    private static Unmarshaller unmarshaller;
72    private static int expectedRows1;
73    private static int expectedRows2;
74    private static Configuration conf;
75  
76    private static int insertData(String tableName, String column, double prob)
77        throws IOException {
78      Random rng = new Random();
79      int count = 0;
80      HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
81      byte[] k = new byte[3];
82      byte [][] famAndQf = KeyValue.parseColumn(Bytes.toBytes(column));
83      for (byte b1 = 'a'; b1 < 'z'; b1++) {
84        for (byte b2 = 'a'; b2 < 'z'; b2++) {
85          for (byte b3 = 'a'; b3 < 'z'; b3++) {
86            if (rng.nextDouble() < prob) {
87              k[0] = b1;
88              k[1] = b2;
89              k[2] = b3;
90              Put put = new Put(k);
91              put.add(famAndQf[0], famAndQf[1], k);
92              table.put(put);
93              count++;
94            }
95          }
96        }
97      }
98      table.flushCommits();
99      return count;
100   }
101 
102   private static int countCellSet(CellSetModel model) {
103     int count = 0;
104     Iterator<RowModel> rows = model.getRows().iterator();
105     while (rows.hasNext()) {
106       RowModel row = rows.next();
107       Iterator<CellModel> cells = row.getCells().iterator();
108       while (cells.hasNext()) {
109         cells.next();
110         count++;
111       }
112     }
113     return count;
114   }
115 
116   private static int fullTableScan(ScannerModel model) throws IOException {
117     model.setBatch(100);
118     Response response = client.put("/" + TABLE + "/scanner",
119       Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
120     assertEquals(response.getCode(), 201);
121     String scannerURI = response.getLocation();
122     assertNotNull(scannerURI);
123     int count = 0;
124     while (true) {
125       response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
126       assertTrue(response.getCode() == 200 || response.getCode() == 204);
127       if (response.getCode() == 200) {
128         CellSetModel cellSet = new CellSetModel();
129         cellSet.getObjectFromMessage(response.getBody());
130         Iterator<RowModel> rows = cellSet.getRows().iterator();
131         while (rows.hasNext()) {
132           RowModel row = rows.next();
133           Iterator<CellModel> cells = row.getCells().iterator();
134           while (cells.hasNext()) {
135             cells.next();
136             count++;
137           }
138         }
139       } else {
140         break;
141       }
142     }
143     // delete the scanner
144     response = client.delete(scannerURI);
145     assertEquals(response.getCode(), 200);
146     return count;
147   }
148 
149   @BeforeClass
150   public static void setUpBeforeClass() throws Exception {
151     conf = TEST_UTIL.getConfiguration();
152     TEST_UTIL.startMiniCluster(3);
153     REST_TEST_UTIL.startServletContainer(conf);
154     client = new Client(new Cluster().add("localhost",
155       REST_TEST_UTIL.getServletPort()));
156     context = JAXBContext.newInstance(
157       CellModel.class,
158       CellSetModel.class,
159       RowModel.class,
160       ScannerModel.class);
161     marshaller = context.createMarshaller();
162     unmarshaller = context.createUnmarshaller();
163     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
164     if (admin.tableExists(TABLE)) {
165       return;
166     }
167     HTableDescriptor htd = new HTableDescriptor(TABLE);
168     htd.addFamily(new HColumnDescriptor(CFA));
169     htd.addFamily(new HColumnDescriptor(CFB));
170     admin.createTable(htd);
171     expectedRows1 = insertData(TABLE, COLUMN_1, 1.0);
172     expectedRows2 = insertData(TABLE, COLUMN_2, 0.5);
173   }
174 
175   @AfterClass
176   public static void tearDownAfterClass() throws Exception {
177     REST_TEST_UTIL.shutdownServletContainer();
178     TEST_UTIL.shutdownMiniCluster();
179   }
180 
181   @Test
182   public void testSimpleScannerXML() throws IOException, JAXBException {
183     final int BATCH_SIZE = 5;
184     // new scanner
185     ScannerModel model = new ScannerModel();
186     model.setBatch(BATCH_SIZE);
187     model.addColumn(Bytes.toBytes(COLUMN_1));
188     StringWriter writer = new StringWriter();
189     marshaller.marshal(model, writer);
190     byte[] body = Bytes.toBytes(writer.toString());
191 
192     // test put operation is forbidden in read-only mode
193     conf.set("hbase.rest.readonly", "true");
194     Response response = client.put("/" + TABLE + "/scanner",
195       Constants.MIMETYPE_XML, body);
196     assertEquals(response.getCode(), 403);
197     String scannerURI = response.getLocation();
198     assertNull(scannerURI);
199 
200     // recall previous put operation with read-only off
201     conf.set("hbase.rest.readonly", "false");
202     response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML,
203       body);
204     assertEquals(response.getCode(), 201);
205     scannerURI = response.getLocation();
206     assertNotNull(scannerURI);
207 
208     // get a cell set
209     response = client.get(scannerURI, Constants.MIMETYPE_XML);
210     assertEquals(response.getCode(), 200);
211     CellSetModel cellSet = (CellSetModel)
212       unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
213     // confirm batch size conformance
214     assertEquals(countCellSet(cellSet), BATCH_SIZE);
215 
216     // test delete scanner operation is forbidden in read-only mode
217     conf.set("hbase.rest.readonly", "true");
218     response = client.delete(scannerURI);
219     assertEquals(response.getCode(), 403);
220 
221     // recall previous delete scanner operation with read-only off
222     conf.set("hbase.rest.readonly", "false");
223     response = client.delete(scannerURI);
224     assertEquals(response.getCode(), 200);
225   }
226 
227   @Test
228   public void testSimpleScannerPB() throws IOException {
229     final int BATCH_SIZE = 10;
230     // new scanner
231     ScannerModel model = new ScannerModel();
232     model.setBatch(BATCH_SIZE);
233     model.addColumn(Bytes.toBytes(COLUMN_1));
234 
235     // test put operation is forbidden in read-only mode
236     conf.set("hbase.rest.readonly", "true");
237     Response response = client.put("/" + TABLE + "/scanner",
238       Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
239     assertEquals(response.getCode(), 403);
240     String scannerURI = response.getLocation();
241     assertNull(scannerURI);
242 
243     // recall previous put operation with read-only off
244     conf.set("hbase.rest.readonly", "false");
245     response = client.put("/" + TABLE + "/scanner",
246       Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
247     assertEquals(response.getCode(), 201);
248     scannerURI = response.getLocation();
249     assertNotNull(scannerURI);
250 
251     // get a cell set
252     response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
253     assertEquals(response.getCode(), 200);
254     CellSetModel cellSet = new CellSetModel();
255     cellSet.getObjectFromMessage(response.getBody());
256     // confirm batch size conformance
257     assertEquals(countCellSet(cellSet), BATCH_SIZE);
258 
259     // test delete scanner operation is forbidden in read-only mode
260     conf.set("hbase.rest.readonly", "true");
261     response = client.delete(scannerURI);
262     assertEquals(response.getCode(), 403);
263 
264     // recall previous delete scanner operation with read-only off
265     conf.set("hbase.rest.readonly", "false");
266     response = client.delete(scannerURI);
267     assertEquals(response.getCode(), 200);
268   }
269 
270   @Test
271   public void testSimpleScannerBinary() throws IOException {
272     // new scanner
273     ScannerModel model = new ScannerModel();
274     model.setBatch(1);
275     model.addColumn(Bytes.toBytes(COLUMN_1));
276 
277     // test put operation is forbidden in read-only mode
278     conf.set("hbase.rest.readonly", "true");
279     Response response = client.put("/" + TABLE + "/scanner",
280       Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
281     assertEquals(response.getCode(), 403);
282     String scannerURI = response.getLocation();
283     assertNull(scannerURI);
284 
285     // recall previous put operation with read-only off
286     conf.set("hbase.rest.readonly", "false");
287     response = client.put("/" + TABLE + "/scanner",
288       Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
289     assertEquals(response.getCode(), 201);
290     scannerURI = response.getLocation();
291     assertNotNull(scannerURI);
292 
293     // get a cell
294     response = client.get(scannerURI, Constants.MIMETYPE_BINARY);
295     assertEquals(response.getCode(), 200);
296     // verify that data was returned
297     assertTrue(response.getBody().length > 0);
298     // verify that the expected X-headers are present
299     boolean foundRowHeader = false, foundColumnHeader = false,
300       foundTimestampHeader = false;
301     for (Header header: response.getHeaders()) {
302       if (header.getName().equals("X-Row")) {
303         foundRowHeader = true;
304       } else if (header.getName().equals("X-Column")) {
305         foundColumnHeader = true;
306       } else if (header.getName().equals("X-Timestamp")) {
307         foundTimestampHeader = true;
308       }
309     }
310     assertTrue(foundRowHeader);
311     assertTrue(foundColumnHeader);
312     assertTrue(foundTimestampHeader);
313 
314     // test delete scanner operation is forbidden in read-only mode
315     conf.set("hbase.rest.readonly", "true");
316     response = client.delete(scannerURI);
317     assertEquals(response.getCode(), 403);
318 
319     // recall previous delete scanner operation with read-only off
320     conf.set("hbase.rest.readonly", "false");
321     response = client.delete(scannerURI);
322     assertEquals(response.getCode(), 200);
323   }
324 
325   @Test
326   public void testFullTableScan() throws IOException {
327     ScannerModel model = new ScannerModel();
328     model.addColumn(Bytes.toBytes(COLUMN_1));
329     assertEquals(fullTableScan(model), expectedRows1);
330 
331     model = new ScannerModel();
332     model.addColumn(Bytes.toBytes(COLUMN_2));
333     assertEquals(fullTableScan(model), expectedRows2);
334   }
335 
336   @Test
337   public void testTableDoesNotExist() throws IOException, JAXBException {
338     ScannerModel model = new ScannerModel();
339     StringWriter writer = new StringWriter();
340     marshaller.marshal(model, writer);
341     byte[] body = Bytes.toBytes(writer.toString());
342     Response response = client.put("/" + NONEXISTENT_TABLE +
343       "/scanner", Constants.MIMETYPE_XML, body);
344     assertEquals(response.getCode(), 404);
345   }
346 }