Filtering on addendum namespaces

helper/geojsonify.js

@@ -8,8 +8,9 @@ const codec = require('pelias-model').codec;
 const field = require('./fieldValue');
 const decode_gid = require('./decode_gid');
 const iso3166 = require('./iso3166');
+const peliasConfig = require('pelias-config').generate();
 
-function geojsonifyPlaces( params, docs ){
+function geojsonifyPlaces(params, docs) {
 
   // flatten & expand data for geojson conversion
   const geodata = docs
@@ -28,8 +29,8 @@ function geojsonifyPlaces( params, docs ){
   const extentPoints = extractExtentPoints(geodata);
 
   // convert to geojson
-  const geojson             = GeoJSON.parse( geodata, { Point: ['lat', 'lng'] });
-  const geojsonExtentPoints = GeoJSON.parse( extentPoints, { Point: ['lat', 'lng'] });
+  const geojson = GeoJSON.parse(geodata, { Point: ['lat', 'lng'] });
+  const geojsonExtentPoints = GeoJSON.parse(extentPoints, { Point: ['lat', 'lng'] });
 
   // to insert the bbox property at the top level of each feature, it must be done separately after
   // initial geojson construction is finished
@@ -72,16 +73,26 @@ function geojsonifyPlace(params, place) {
   // add addendum data if available
   // note: this should be the last assigned property, for aesthetic reasons.
   if (_.has(place, 'addendum')) {
-    let addendum = {};
-    for(let namespace in place.addendum){
-      try {
-        addendum[namespace] = codec.decode(place.addendum[namespace]);
-      } catch( e ){
-        logger.warn(`doc ${doc.gid} failed to decode addendum namespace ${namespace}`);
+    const configuredAddendumNamespaces = peliasConfig.get('addendum_namespaces');
+    let nonConfiguredNamespaces = {};
+    let configuredNamespaces = {};
+    for (const namespace in place.addendum) {
+      const isConfigured = configuredAddendumNamespaces[namespace];
+      if (isConfigured) {
+        configuredNamespaces[namespace] = place.addendum[namespace];
+      } else {
+        try {
+          nonConfiguredNamespaces[namespace] = codec.decode(place.addendum[namespace]);
+        } catch (e) {
+          logger.warn(`doc ${doc.gid} failed to decode addendum namespace ${namespace}`);
+        }
       }
     }
-    if( Object.keys(addendum).length ){
-      doc.addendum = addendum;
+    if (Object.keys(nonConfiguredNamespaces).length) {
+      doc.addendum = nonConfiguredNamespaces;
+    }
+    for (let namespace in configuredNamespaces) {
+      doc[namespace] = configuredNamespaces[namespace];
     }
   }
 
@@ -133,8 +144,7 @@ function extractExtentPoints(geodata) {
         lat: place.bounding_box.max_lat
       });
 
-    }
-    else {
+    } else {
       // otherwise, use the point for the extent
       extentPoints.push({
         lng: place.lng,
@@ -158,13 +168,13 @@ function computeBBox(geojson, geojsonExtentPoints) {
   // @note: extent() sometimes throws Errors for unusual data
   // eg: https://github.com/pelias/pelias/issues/84
   try {
-    var bbox = extent( geojsonExtentPoints );
-    if( !!bbox ){
+    const bbox = extent(geojsonExtentPoints);
+    if (!!bbox) {
       geojson.bbox = bbox;
     }
-  } catch( e ){
-    logger.error( 'bbox error', e.message, e.stack );
-    logger.error( 'geojson', geojsonExtentPoints );
+  } catch (e) {
+    logger.error('bbox error', e.message, e.stack);
+    logger.error('geojson', geojsonExtentPoints);
   }
 }
 
@@ -176,11 +186,15 @@ function computeBBox(geojson, geojsonExtentPoints) {
 function addISO3166PropsPerFeature(geojson) {
   geojson.features.forEach(feature => {
     let code = _.get(feature, 'properties.country_a') || _.get(feature, 'properties.dependency_a') || '';
-    if (!_.isString(code) || _.isEmpty(code)){ return; }
+    if (!_.isString(code) || _.isEmpty(code)) {
+      return;
+    }
 
     let info = iso3166.info(code);
     let alpha2 = _.get(info, 'alpha2');
-    if (!_.isString(alpha2) || _.size(alpha2) !== 2) { return; }
+    if (!_.isString(alpha2) || _.size(alpha2) !== 2) {
+      return;
+    }
 
     _.set(feature, 'properties.country_code', alpha2);
   });

query/autocomplete.js

@@ -8,16 +8,17 @@ const toSingleField = require('./view/helper').toSingleField;
 
 // additional views (these may be merged in to pelias/query at a later date)
 var views = {
-  custom_boosts:              require('./view/boost_sources_and_layers'),
-  ngrams_strict:              require('./view/ngrams_strict'),
-  ngrams_last_token_only:     require('./view/ngrams_last_token_only'),
-  ngrams_last_token_only_multi: require('./view/ngrams_last_token_only_multi'),
-  admin_multi_match_first: require('./view/admin_multi_match_first'),
-  admin_multi_match_last: require('./view/admin_multi_match_last'),
-  phrase_first_tokens_only:   require('./view/phrase_first_tokens_only'),
-  boost_exact_matches:        require('./view/boost_exact_matches'),
+  custom_boosts:                      require('./view/boost_sources_and_layers'),
+  ngrams_strict:                      require('./view/ngrams_strict'),
+  ngrams_last_token_only:             require('./view/ngrams_last_token_only'),
+  ngrams_last_token_only_multi:       require('./view/ngrams_last_token_only_multi'),
+  admin_multi_match_first:            require('./view/admin_multi_match_first'),
+  admin_multi_match_last:             require('./view/admin_multi_match_last'),
+  phrase_first_tokens_only:           require('./view/phrase_first_tokens_only'),
+  boost_exact_matches:                require('./view/boost_exact_matches'),
   max_character_count_layer_filter:   require('./view/max_character_count_layer_filter'),
-  focus_point_filter:         require('./view/focus_point_distance_filter')
+  focus_point_filter:                 require('./view/focus_point_distance_filter'),
+  addendum_namespace_filter:          require('./view/addendum_namespace_filter')
 };
 
 // add abbrevations for the fields pelias/parser is able to detect.
@@ -65,6 +66,11 @@ query.filter( peliasQuery.view.categories );
 query.filter( peliasQuery.view.boundary_gid );
 query.filter( views.focus_point_filter );
 
+const configuredAddendumNamespaces = config.get('addendum_namespaces');
+Object.keys(configuredAddendumNamespaces).forEach(namespace => {
+  query.filter( views.addendum_namespace_filter(namespace, configuredAddendumNamespaces[namespace].type) );
+});
+
 // --------------------------------
 
 /**
@@ -75,6 +81,14 @@ function generateQuery( clean ){
 
   const vs = new peliasQuery.Vars( defaults );
 
+  //addendum
+  const configuredAddendumNamespaces = config.get('addendum_namespaces');
+  Object.keys(configuredAddendumNamespaces)
+    .filter(namespace => clean[namespace])
+    .forEach(namespace => {
+      vs.var( namespace, clean[namespace] );
+    });
+
   // sources
   if( _.isArray(clean.sources) && !_.isEmpty(clean.sources) ){
     vs.var( 'sources', clean.sources );
@@ -104,7 +118,7 @@ function generateQuery( clean ){
   vs.var( 'input:name', clean.text );
 
   // if the tokenizer has run then we set 'input:name' to as the combination of the
-  // 'complete' tokens with the 'incomplete' tokens, the resuting array differs
+  // 'complete' tokens with the 'incomplete' tokens, the resulting array differs
   // slightly from the 'input:name:tokens' array as some tokens might have been
   // removed in the process; such as single grams which are not present in then
   // ngrams index.

query/reverse.js

@@ -1,6 +1,8 @@
 const _ = require('lodash');
 const peliasQuery = require('pelias-query');
 const defaults = require('./reverse_defaults');
+const config = require('pelias-config').generate();
+const addendum_namespace_filter = require('./view/addendum_namespace_filter');
 
 //------------------------------
 // reverse geocode query
@@ -21,12 +23,25 @@ query.filter( peliasQuery.view.categories );
 query.filter( peliasQuery.view.boundary_country );
 query.filter( peliasQuery.view.boundary_gid );
 
+const configuredAddendumNamespaces = config.get('addendum_namespaces');
+Object.keys(configuredAddendumNamespaces).forEach(namespace => {
+  query.filter( addendum_namespace_filter(namespace, configuredAddendumNamespaces[namespace].type) );
+});
+
 // --------------------------------
 
 function generateQuery( clean ){
 
   const vs = new peliasQuery.Vars( defaults );
 
+  //addendum
+  const configuredAddendumNamespaces = config.get('addendum_namespaces');
+  Object.keys(configuredAddendumNamespaces)
+    .filter(namespace => clean[namespace])
+    .forEach(namespace => {
+      vs.var( namespace, clean[namespace] );
+    });
+
   // set size
   if( clean.querySize ){
     vs.var( 'size', clean.querySize);
@@ -55,7 +70,7 @@ function generateQuery( clean ){
   // bounding circle
   // note: the sanitizers will take care of the case
   // where point.lan/point.lon are provided in the
-  // absense of boundary.circle.lat/boundary.circle.lon
+  // absence of boundary.circle.lat/boundary.circle.lon
   if( _.isFinite(clean['boundary.circle.lat']) &&
       _.isFinite(clean['boundary.circle.lon']) ){
 

query/search.js

@@ -2,6 +2,8 @@ const _ = require('lodash');
 const peliasQuery = require('pelias-query');
 const defaults = require('./search_defaults');
 const textParser = require('./text_parser');
+const config = require('pelias-config').generate();
+const addendum_namespace_filter = require('./view/addendum_namespace_filter');
 
 //------------------------------
 // general-purpose search query
@@ -22,6 +24,12 @@ fallbackQuery.filter( peliasQuery.view.sources );
 fallbackQuery.filter( peliasQuery.view.layers );
 fallbackQuery.filter( peliasQuery.view.categories );
 fallbackQuery.filter( peliasQuery.view.boundary_gid );
+
+const configuredAddendumNamespaces = config.get('addendum_namespaces');
+Object.keys(configuredAddendumNamespaces).forEach(namespace => {
+  fallbackQuery.filter( addendum_namespace_filter(namespace, configuredAddendumNamespaces[namespace].type) );
+});
+
 // --------------------------------
 
 /**
@@ -36,6 +44,14 @@ function generateQuery( clean ){
   // input text
   vs.var( 'input:name', clean.text );
 
+  //addendum
+  const configuredAddendumNamespaces = config.get('addendum_namespaces');
+  Object.keys(configuredAddendumNamespaces)
+    .filter(namespace => clean[namespace])
+    .forEach(namespace => {
+      vs.var( namespace, clean[namespace] );
+    });
+
   // sources
   if( _.isArray(clean.sources) && !_.isEmpty(clean.sources) ) {
     vs.var( 'sources', clean.sources);
@@ -113,11 +129,7 @@ function generateQuery( clean ){
     textParser( clean.parsed_text, vs );
   }
 
-  var q = getQuery(vs);
-
-  //console.log(JSON.stringify(q, null, 2));
-
-  return q;
+  return getQuery(vs);
 }
 
 function getQuery(vs) {

query/structured_geocoding.js

@@ -2,6 +2,8 @@ const _ = require('lodash');
 const peliasQuery = require('pelias-query');
 const defaults = require('./search_defaults');
 const textParser = require('./text_parser');
+const config = require('pelias-config').generate();
+const addendum_namespace_filter = require('./view/addendum_namespace_filter');
 
 //------------------------------
 // general-purpose search query
@@ -22,6 +24,11 @@ structuredQuery.filter( peliasQuery.view.sources );
 structuredQuery.filter( peliasQuery.view.layers );
 structuredQuery.filter( peliasQuery.view.categories );
 structuredQuery.filter( peliasQuery.view.boundary_gid );
+
+const configuredAddendumNamespaces = config.get('addendum_namespaces');
+Object.keys(configuredAddendumNamespaces).forEach(namespace => {
+  structuredQuery.filter( addendum_namespace_filter(namespace, configuredAddendumNamespaces[namespace].type) );
+});
 // --------------------------------
 
 /**
@@ -35,6 +42,14 @@ function generateQuery( clean ){
   // input text
   vs.var( 'input:name', clean.text );
 
+  //addendum
+  const configuredAddendumNamespaces = config.get('addendum_namespaces');
+  Object.keys(configuredAddendumNamespaces)
+    .filter(namespace => clean[namespace])
+    .forEach(namespace => {
+      vs.var( namespace, clean[namespace] );
+    });
+
   // sources
   vs.var( 'sources', clean.sources);
 
@@ -103,11 +118,7 @@ function generateQuery( clean ){
     textParser( clean.parsed_text, vs );
   }
 
-  const q = getQuery(vs);
-
-  // console.log(JSON.stringify(q.body, null, 2));
-
-  return q;
+  return getQuery(vs);
 }
 
 function getQuery(vs) {

query/view/addendum_namespace_filter.js

@@ -0,0 +1,22 @@
+module.exports = function (addendumParameter, type) {
+  return function (vs) {
+    if (!vs.isset(addendumParameter)) {
+      return null;
+    }
+
+    switch (type) {
+      case 'array':
+        return {
+          terms: {
+            [`addendum.${addendumParameter}`]: vs.var(addendumParameter)
+          }
+        };
+      default:
+        return {
+          term: {
+            [`addendum.${addendumParameter}`]: vs.var(addendumParameter)
+          }
+        };
+    }
+  };
+};

sanitizer/_addendum.js

@@ -0,0 +1,65 @@
+const _ = require('lodash');
+const peliasConfig = require('pelias-config').generate();
+
+const nonEmptyString = (v) => _.isString(v) && !_.isEmpty(v);
+
+function _sanitize(raw, clean) {
+
+  // error & warning messages
+  const messages = { errors: [], warnings: [] };
+
+  // target input params
+  const configuredAddendumNamespaces = peliasConfig.get('addendum_namespaces');
+
+  Object.keys(raw)
+    .filter(namespace => configuredAddendumNamespaces.hasOwnProperty(namespace))
+    .forEach(namespace => {
+
+      if (!nonEmptyString(raw[namespace])) {
+        messages.errors.push(namespace + ' should be a non empty string');
+      } else {
+        const rawValue = raw[namespace].trim();
+        const validationType = configuredAddendumNamespaces[namespace].type;
+        switch (validationType.toLowerCase()) {
+          case 'array':
+            const valuesArray = rawValue.split(',').filter(nonEmptyString);
+            if (_.isArray(valuesArray) && _.isEmpty(valuesArray)) {
+              messages.errors.push(namespace + ' should not be empty');
+            } else {
+              clean[namespace] = valuesArray;
+            }
+            break;
+
+          case 'string':
+            clean[namespace] = rawValue;
+            break;
+
+          case 'number':
+            if (isNaN(rawValue)) {
+              messages.errors.push(namespace + ': Invalid parameter type, expecting: ' + validationType + ', got NaN: ' + rawValue);
+            } else {
+              clean[namespace] = Number(rawValue);
+            }
+            break;
+
+          case 'boolean':
+            if ('true' !== rawValue && 'false' !== rawValue) {
+              messages.errors.push(namespace + ': Invalid parameter type, expecting: ' + validationType + ', got: ' + rawValue);
+            } else {
+              clean[namespace] = 'true' === rawValue;
+            }
+            break;
+
+          default:
+            messages.errors.push(namespace + ': Unsupported configured type ' + validationType + ', ' +
+              'valid types are array, string, number and boolean');
+        }
+      }
+    });
+
+  return messages;
+}
+
+module.exports = () => ({
+  sanitize: _sanitize,
+});