Skip to content

Commit

Permalink
HBASE-28518 Allow specifying a filter for the REST multiget endpoint
Browse files Browse the repository at this point in the history
Signed-off-by: Ankit Singhal <ankit@apache.org>
stoty committed Apr 16, 2024
1 parent f5e0ba6 commit 4d3a8b8
Showing 6 changed files with 160 additions and 26 deletions.
Original file line number Diff line number Diff line change
@@ -42,7 +42,7 @@
* of this class and a filter object is constructed. This filter object is then wrapped in a scanner
* object which is then returned
* <p>
* This class addresses the HBASE-4168 JIRA. More documentation on this Filter Language can be found
* This class addresses the HBASE-4176 JIRA. More documentation on this Filter Language can be found
* at: https://issues.apache.org/jira/browse/HBASE-4176
*/
@InterfaceAudience.Public
Original file line number Diff line number Diff line change
@@ -85,7 +85,8 @@ public interface Constants {
String SCAN_BATCH_SIZE = "batchsize";
String SCAN_LIMIT = "limit";
String SCAN_FETCH_SIZE = "hbase.rest.scan.fetchsize";
String SCAN_FILTER = "filter";
String FILTER = "filter";
String FILTER_B64 = "filter_b64";
String SCAN_REVERSED = "reversed";
String SCAN_CACHE_BLOCKS = "cacheblocks";
String CUSTOM_FILTERS = "hbase.rest.custom.filters";
Original file line number Diff line number Diff line change
@@ -18,8 +18,12 @@
package org.apache.hadoop.hbase.rest;

import java.io.IOException;
import java.util.Base64;
import java.util.Base64.Decoder;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.ParseFilter;
import org.apache.hadoop.hbase.rest.model.CellModel;
import org.apache.hadoop.hbase.rest.model.CellSetModel;
import org.apache.hadoop.hbase.rest.model.RowModel;
@@ -28,9 +32,11 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.apache.hbase.thirdparty.javax.ws.rs.Encoded;
import org.apache.hbase.thirdparty.javax.ws.rs.GET;
import org.apache.hbase.thirdparty.javax.ws.rs.HeaderParam;
import org.apache.hbase.thirdparty.javax.ws.rs.Produces;
import org.apache.hbase.thirdparty.javax.ws.rs.QueryParam;
import org.apache.hbase.thirdparty.javax.ws.rs.core.Context;
import org.apache.hbase.thirdparty.javax.ws.rs.core.MultivaluedMap;
import org.apache.hbase.thirdparty.javax.ws.rs.core.Response;
@@ -40,6 +46,8 @@
public class MultiRowResource extends ResourceBase implements Constants {
private static final Logger LOG = LoggerFactory.getLogger(MultiRowResource.class);

private static final Decoder base64Urldecoder = Base64.getUrlDecoder();

TableResource tableResource;
Integer versions = null;
String[] columns = null;
@@ -65,15 +73,34 @@ public MultiRowResource(TableResource tableResource, String versions, String col
@GET
@Produces({ MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF, MIMETYPE_PROTOBUF_IETF })
public Response get(final @Context UriInfo uriInfo,
final @HeaderParam("Encoding") String keyEncodingHeader) {
final @HeaderParam("Encoding") String keyEncodingHeader,
@QueryParam(Constants.FILTER_B64) @Encoded String paramFilterB64,
@QueryParam(Constants.FILTER) String paramFilter) {
MultivaluedMap<String, String> params = uriInfo.getQueryParameters();
String keyEncoding = (keyEncodingHeader != null)
? keyEncodingHeader
: params.getFirst(KEY_ENCODING_QUERY_PARAM_NAME);

servlet.getMetrics().incrementRequests(1);

byte[] filterBytes = null;
if (paramFilterB64 != null) {
filterBytes = base64Urldecoder.decode(paramFilterB64);
} else if (paramFilter != null) {
// Not binary clean
filterBytes = paramFilter.getBytes();
}

try {
Filter parsedParamFilter = null;
if (filterBytes != null) {
// Note that this is a completely different representation of the filters
// than the JSON one used in the /table/scanner endpoint
ParseFilter pf = new ParseFilter();
parsedParamFilter = pf.parseFilterString(filterBytes);
}
CellSetModel model = new CellSetModel();
// TODO map this to a Table.get(List<Get> gets) call instead of multiple get calls
for (String rk : params.get(ROW_KEYS_PARAM_NAME)) {
RowSpec rowSpec = new RowSpec(rk, keyEncoding);

@@ -88,7 +115,7 @@ public Response get(final @Context UriInfo uriInfo,
}

ResultGenerator generator = ResultGenerator.fromRowSpec(this.tableResource.getName(),
rowSpec, null, !params.containsKey(NOCACHE_PARAM_NAME));
rowSpec, parsedParamFilter, !params.containsKey(NOCACHE_PARAM_NAME));
Cell value = null;
RowModel rowModel = new RowModel(rowSpec.getRow());
if (generator.hasNext()) {
Original file line number Diff line number Diff line change
@@ -18,8 +18,8 @@
package org.apache.hadoop.hbase.rest;

import java.io.IOException;
import java.util.Base64.Decoder;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Scan;
@@ -46,6 +46,8 @@ public class TableResource extends ResourceBase {
String table;
private static final Logger LOG = LoggerFactory.getLogger(TableResource.class);

private static final Decoder base64Urldecoder = java.util.Base64.getUrlDecoder();

/**
* Constructor
*/
@@ -103,6 +105,7 @@ public RowResource getRowResource(
return new RowResource(this, rowspec, versions, check, returnResult, keyEncoding);
}

// TODO document
@Path("{suffixglobbingspec: .*\\*/.+}")
public RowResource getRowResourceWithSuffixGlobbing(
// We need the @Encoded decorator so Jersey won't urldecode before
@@ -117,6 +120,8 @@ public RowResource getRowResourceWithSuffixGlobbing(
return new RowResource(this, suffixglobbingspec, versions, check, returnResult, keyEncoding);
}

// TODO document
// FIXME handle binary rowkeys (like put and delete does)
@Path("{scanspec: .*[*]$}")
public TableScanResource getScanResource(final @PathParam("scanspec") String scanSpec,
@DefaultValue(Integer.MAX_VALUE + "") @QueryParam(Constants.SCAN_LIMIT) int userRequestedLimit,
@@ -129,7 +134,8 @@ public TableScanResource getScanResource(final @PathParam("scanspec") String sca
@DefaultValue(Long.MAX_VALUE + "") @QueryParam(Constants.SCAN_END_TIME) long endTime,
@DefaultValue("true") @QueryParam(Constants.SCAN_CACHE_BLOCKS) boolean cacheBlocks,
@DefaultValue("false") @QueryParam(Constants.SCAN_REVERSED) boolean reversed,
@DefaultValue("") @QueryParam(Constants.SCAN_FILTER) String paramFilter) {
@QueryParam(Constants.FILTER) String paramFilter,
@QueryParam(Constants.FILTER_B64) @Encoded String paramFilterB64) {
try {
Filter prefixFilter = null;
Scan tableScan = new Scan();
@@ -173,15 +179,24 @@ public TableScanResource getScanResource(final @PathParam("scanspec") String sca
}
}
FilterList filterList = new FilterList();
if (StringUtils.isNotEmpty(paramFilter)) {
byte[] filterBytes = null;
if (paramFilterB64 != null) {
filterBytes = base64Urldecoder.decode(paramFilterB64);
} else if (paramFilter != null) {
// Not binary clean
filterBytes = paramFilter.getBytes();
}
if (filterBytes != null) {
// Note that this is a completely different representation of the filters
// than the JSON one used in the /table/scanner endpoint
ParseFilter pf = new ParseFilter();
Filter parsedParamFilter = pf.parseFilterString(paramFilter);
Filter parsedParamFilter = pf.parseFilterString(filterBytes);
if (parsedParamFilter != null) {
filterList.addFilter(parsedParamFilter);
}
if (prefixFilter != null) {
filterList.addFilter(prefixFilter);
}
}
if (prefixFilter != null) {
filterList.addFilter(prefixFilter);
}
if (filterList.size() > 0) {
tableScan.setFilter(filterList);
Original file line number Diff line number Diff line change
@@ -21,11 +21,11 @@

import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.IOException;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
import java.util.Base64.Encoder;
import java.util.Collection;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.Marshaller;
import javax.xml.bind.Unmarshaller;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
@@ -76,10 +76,9 @@ public class TestMultiRowResource {
private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private static final HBaseRESTTestingUtility REST_TEST_UTIL = new HBaseRESTTestingUtility();

private static final Encoder base64UrlEncoder = java.util.Base64.getUrlEncoder();

private static Client client;
private static JAXBContext context;
private static Marshaller marshaller;
private static Unmarshaller unmarshaller;
private static Configuration conf;

private static Header extraHdr = null;
@@ -104,9 +103,6 @@ public static void setUpBeforeClass() throws Exception {
extraHdr = new BasicHeader(RESTServer.REST_CSRF_CUSTOM_HEADER_DEFAULT, "");
TEST_UTIL.startMiniCluster();
REST_TEST_UTIL.startServletContainer(conf);
context = JAXBContext.newInstance(CellModel.class, CellSetModel.class, RowModel.class);
marshaller = context.createMarshaller();
unmarshaller = context.createUnmarshaller();
client = new Client(new Cluster().add("localhost", REST_TEST_UTIL.getServletPort()));
Admin admin = TEST_UTIL.getAdmin();
if (admin.tableExists(TABLE)) {
@@ -336,4 +332,71 @@ public void testMultiCellGetWithColsInQueryPathJSON() throws IOException {
client.delete(row_5_url, extraHdr);
client.delete(row_6_url, extraHdr);
}

@Test
public void testMultiCellGetFilterJSON() throws IOException {
String row_5_url = "/" + TABLE + "/" + ROW_1 + "/" + COLUMN_1;
String row_6_url = "/" + TABLE + "/" + ROW_2 + "/" + COLUMN_2;

StringBuilder path = new StringBuilder();
path.append("/");
path.append(TABLE);
path.append("/multiget/?row=");
path.append(ROW_1);
path.append("&row=");
path.append(ROW_2);

if (csrfEnabled) {
Response response = client.post(row_5_url, Constants.MIMETYPE_BINARY, Bytes.toBytes(VALUE_1));
assertEquals(400, response.getCode());
}

client.post(row_5_url, Constants.MIMETYPE_BINARY, Bytes.toBytes(VALUE_1), extraHdr);
client.post(row_6_url, Constants.MIMETYPE_BINARY, Bytes.toBytes(VALUE_2), extraHdr);

Response response = client.get(path.toString(), Constants.MIMETYPE_JSON);
assertEquals(200, response.getCode());
assertEquals(Constants.MIMETYPE_JSON, response.getHeader("content-type"));

// If the filter is used, then we get the same result
String positivePath = path.toString() + ("&" + Constants.FILTER_B64 + "=" + base64UrlEncoder
.encodeToString("PrefixFilter('testrow')".getBytes(StandardCharsets.UTF_8.toString())));
response = client.get(positivePath, Constants.MIMETYPE_JSON);
checkMultiCellGetJSON(response);

// Same with non binary clean param
positivePath = path.toString() + ("&" + Constants.FILTER + "="
+ URLEncoder.encode("PrefixFilter('testrow')", StandardCharsets.UTF_8.name()));
response = client.get(positivePath, Constants.MIMETYPE_JSON);
checkMultiCellGetJSON(response);

// This filter doesn't match the found rows
String negativePath = path.toString() + ("&" + Constants.FILTER_B64 + "=" + base64UrlEncoder
.encodeToString("PrefixFilter('notfound')".getBytes(StandardCharsets.UTF_8.toString())));
response = client.get(negativePath, Constants.MIMETYPE_JSON);
assertEquals(404, response.getCode());

// Same with non binary clean param
negativePath = path.toString() + ("&" + Constants.FILTER + "="
+ URLEncoder.encode("PrefixFilter('notfound')", StandardCharsets.UTF_8.name()));
response = client.get(negativePath, Constants.MIMETYPE_JSON);
assertEquals(404, response.getCode());

// Check with binary parameters
// positive case
positivePath = path.toString() + ("&" + Constants.FILTER_B64 + "=" + base64UrlEncoder
.encodeToString(Bytes.toBytesBinary("ColumnRangeFilter ('\\x00', true, '\\xff', true)")));
response = client.get(positivePath, Constants.MIMETYPE_JSON);
checkMultiCellGetJSON(response);

// negative case
negativePath = path.toString() + ("&" + Constants.FILTER_B64 + "=" + base64UrlEncoder
.encodeToString(Bytes.toBytesBinary("ColumnRangeFilter ('\\x00', true, '1', false)")));
response = client.get(negativePath, Constants.MIMETYPE_JSON);
assertEquals(404, response.getCode());

client.delete(row_5_url, extraHdr);
client.delete(row_6_url, extraHdr);
}

}
Original file line number Diff line number Diff line change
@@ -34,6 +34,7 @@
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Base64.Encoder;
import java.util.Collections;
import java.util.List;
import javax.xml.bind.JAXBContext;
@@ -93,6 +94,7 @@ public class TestTableScan {
private static Configuration conf;

private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private static final Encoder base64UrlEncoder = java.util.Base64.getUrlEncoder();
private static final HBaseRESTTestingUtility REST_TEST_UTIL = new HBaseRESTTestingUtility();

@BeforeClass
@@ -442,7 +444,33 @@ public void testSimpleFilter() throws IOException, JAXBException {
builder.append("&");
builder.append(Constants.SCAN_END_ROW + "=aay");
builder.append("&");
builder.append(Constants.SCAN_FILTER + "=" + URLEncoder.encode("PrefixFilter('aab')", "UTF-8"));
builder.append(Constants.FILTER + "=" + URLEncoder.encode("PrefixFilter('aab')", "UTF-8"));
Response response = client.get("/" + TABLE + builder.toString(), Constants.MIMETYPE_XML);
assertEquals(200, response.getCode());
JAXBContext ctx = JAXBContext.newInstance(CellSetModel.class);
Unmarshaller ush = ctx.createUnmarshaller();
CellSetModel model = (CellSetModel) ush.unmarshal(response.getStream());
int count = TestScannerResource.countCellSet(model);
assertEquals(1, count);
assertEquals("aab",
new String(model.getRows().get(0).getCells().get(0).getValue(), StandardCharsets.UTF_8));
}

// This only tests the Base64Url encoded filter definition.
// base64 encoded row values are not implemented for this endpoint
@Test
public void testSimpleFilterBase64() throws IOException, JAXBException {
StringBuilder builder = new StringBuilder();
builder.append("/*");
builder.append("?");
builder.append(Constants.SCAN_COLUMN + "=" + COLUMN_1);
builder.append("&");
builder.append(Constants.SCAN_START_ROW + "=aaa");
builder.append("&");
builder.append(Constants.SCAN_END_ROW + "=aay");
builder.append("&");
builder.append(Constants.FILTER_B64 + "=" + base64UrlEncoder
.encodeToString("PrefixFilter('aab')".getBytes(StandardCharsets.UTF_8.toString())));
Response response = client.get("/" + TABLE + builder.toString(), Constants.MIMETYPE_XML);
assertEquals(200, response.getCode());
JAXBContext ctx = JAXBContext.newInstance(CellSetModel.class);
@@ -459,8 +487,8 @@ public void testQualifierAndPrefixFilters() throws IOException, JAXBException {
StringBuilder builder = new StringBuilder();
builder.append("/abc*");
builder.append("?");
builder.append(
Constants.SCAN_FILTER + "=" + URLEncoder.encode("QualifierFilter(=,'binary:1')", "UTF-8"));
builder
.append(Constants.FILTER + "=" + URLEncoder.encode("QualifierFilter(=,'binary:1')", "UTF-8"));
Response response = client.get("/" + TABLE + builder.toString(), Constants.MIMETYPE_XML);
assertEquals(200, response.getCode());
JAXBContext ctx = JAXBContext.newInstance(CellSetModel.class);
@@ -477,7 +505,7 @@ public void testCompoundFilter() throws IOException, JAXBException {
StringBuilder builder = new StringBuilder();
builder.append("/*");
builder.append("?");
builder.append(Constants.SCAN_FILTER + "="
builder.append(Constants.FILTER + "="
+ URLEncoder.encode("PrefixFilter('abc') AND QualifierFilter(=,'binary:1')", "UTF-8"));
Response response = client.get("/" + TABLE + builder.toString(), Constants.MIMETYPE_XML);
assertEquals(200, response.getCode());
@@ -497,7 +525,7 @@ public void testCustomFilter() throws IOException, JAXBException {
builder.append("?");
builder.append(Constants.SCAN_COLUMN + "=" + COLUMN_1);
builder.append("&");
builder.append(Constants.SCAN_FILTER + "=" + URLEncoder.encode("CustomFilter('abc')", "UTF-8"));
builder.append(Constants.FILTER + "=" + URLEncoder.encode("CustomFilter('abc')", "UTF-8"));
Response response = client.get("/" + TABLE + builder.toString(), Constants.MIMETYPE_XML);
assertEquals(200, response.getCode());
JAXBContext ctx = JAXBContext.newInstance(CellSetModel.class);
@@ -516,7 +544,7 @@ public void testNegativeCustomFilter() throws IOException, JAXBException {
builder.append("?");
builder.append(Constants.SCAN_COLUMN + "=" + COLUMN_1);
builder.append("&");
builder.append(Constants.SCAN_FILTER + "=" + URLEncoder.encode("CustomFilter('abc')", "UTF-8"));
builder.append(Constants.FILTER + "=" + URLEncoder.encode("CustomFilter('abc')", "UTF-8"));
Response response = client.get("/" + TABLE + builder.toString(), Constants.MIMETYPE_XML);
assertEquals(200, response.getCode());
JAXBContext ctx = JAXBContext.newInstance(CellSetModel.class);

0 comments on commit 4d3a8b8

Please sign in to comment.