Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

remove phrase field in Document model #148

Merged
merged 1 commit into from
Jun 21, 2022
Merged

remove phrase field in Document model #148

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 1 addition & 8 deletions Document.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ const parentFields = [

function Document( source, layer, source_id ){
this.name = {};
this.phrase = {};
this.parent = {};
this.address_parts = {};
this.center_point = {};
Expand Down Expand Up @@ -93,7 +92,7 @@ Document.prototype.toESDocument = function() {

var doc = {
name: this.name,
phrase: this.phrase,
phrase: this.name,
parent: this.parent,
address_parts: this.address_parts,
center_point: this.center_point,
Expand Down Expand Up @@ -249,13 +248,10 @@ Document.prototype.setName = function( prop, value ){
validate.truthy(value);
validate.regex.nomatch(value, /https?:\/\//);

// must copy name to 'phrase' index
if( Array.isArray( this.name[ prop ] ) ){
this.name[ prop ][ 0 ] = value;
this.phrase[ prop ][ 0 ] = value;
} else {
this.name[ prop ] = value;
this.phrase[ prop ] = value;
}

return this;
Expand All @@ -270,14 +266,12 @@ Document.prototype.setNameAlias = function( prop, value ){
// is this the first time setting this prop? ensure it's an array
if( !this.hasName( prop ) ){
this.name[ prop ] = [];
this.phrase[ prop ] = [];
}

// is casting required to convert a scalar field to an array?
else if( 'string' === typeof this.name[ prop ] ){
var stringValue = this.name[ prop ];
this.name[ prop ] = [ stringValue ];
this.phrase[ prop ] = [ stringValue ];
}

// is the array empty? ie. no prior call to setName()
Expand All @@ -288,7 +282,6 @@ Document.prototype.setNameAlias = function( prop, value ){

// set the alias as the second, third, fourth, etc value in the array
this.name[ prop ].push( value );
this.phrase[ prop ].push( value );

return this;
};
Expand Down
2 changes: 1 addition & 1 deletion post/deduplication.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
*/

const _ = require('lodash');
const prefixes = ['name', 'phrase', 'address_parts'];
const prefixes = ['name', 'address_parts'];
const punctuation = /[\.]+/g;
const normalize = (v) => _.isString(v) ? _.replace(v.toLowerCase(), punctuation, '') : v;

Expand Down
60 changes: 29 additions & 31 deletions post/language_field_trimming.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,50 +15,48 @@
*/

const _ = require('lodash');
const prefixes = ['name', 'phrase'];
const prefix = 'name';

function deduplication(doc) {
prefixes.forEach(prefix => {

// load the field data
// ie: an object keyed by language codes, each value is an array of names
let field = doc[prefix];
if (!_.isPlainObject(field)) { return; }
// load the field data
// ie: an object keyed by language codes, each value is an array of names
let field = doc[prefix];
if (!_.isPlainObject(field)) { return; }

// fetch the 'default' language
var defaults = _.get(field, 'default');
// fetch the 'default' language
var defaults = _.get(field, 'default');

// no default names, nothing to do; continue
if (_.isEmpty(defaults)) { return; }
// no default names, nothing to do; continue
if (_.isEmpty(defaults)) { return; }

// convert scalar values to arrays
defaults = _.castArray(defaults);
// convert scalar values to arrays
defaults = _.castArray(defaults);

// iterate over other languages in the field
_.each(field, (names, lang) => {
// iterate over other languages in the field
_.each(field, (names, lang) => {

// skip the 'default' language
if (lang === 'default'){ return; }
// skip the 'default' language
if (lang === 'default'){ return; }

// no names, nothing to do; continue
if (_.isEmpty(names)) { return; }
// no names, nothing to do; continue
if (_.isEmpty(names)) { return; }

// convert scalar values to arrays
names = _.castArray(names);
// convert scalar values to arrays
names = _.castArray(names);

// filter entries from this language which appear in the 'default' lang
field[lang] = _.difference(names, defaults);
// filter entries from this language which appear in the 'default' lang
field[lang] = _.difference(names, defaults);

// clean up empty language arrays
if (_.isEmpty(field[lang])) {
delete field[lang];
}
// clean up empty language arrays
if (_.isEmpty(field[lang])) {
delete field[lang];
}

// flatten single-value arrays
else if(_.size(field[lang]) === 1) {
field[lang] = _.first(field[lang]);
}
});
// flatten single-value arrays
else if(_.size(field[lang]) === 1) {
field[lang] = _.first(field[lang]);
}
});
}

Expand Down
7 changes: 0 additions & 7 deletions test/document/name.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ module.exports.tests.setName = function(test) {
var doc = new Document('mysource','mylayer','myid');
t.equal(doc.setName('foo','bar'), doc, 'chainable');
t.equal(doc.name.foo, 'bar', 'setter works');
t.equal(doc.phrase.foo, 'bar', 'setter works');
t.end();
});
test('setName - validate key', function(t) {
Expand Down Expand Up @@ -70,9 +69,6 @@ module.exports.tests.setNameAlias = function(test) {
t.equal(doc.name.foo[0], 'bar', 'setter works');
t.equal(doc.name.foo[1], 'bar', 'setter works');
t.equal(doc.name.foo[2], 'baz', 'setter works');
t.equal(doc.phrase.foo[0], 'bar', 'setter works');
t.equal(doc.phrase.foo[1], 'bar', 'setter works');
t.equal(doc.phrase.foo[2], 'baz', 'setter works');
t.equal(doc.getName('foo'), 'bar', 'name set');
t.deepEqual(doc.getNameAliases('foo'), ['bar','baz'], 'aliases set');
t.end();
Expand All @@ -85,9 +81,6 @@ module.exports.tests.setNameAlias = function(test) {
t.equal(doc.name.foo[0], 'bar', 'setter works');
t.equal(doc.name.foo[1], 'baz', 'setter works');
t.equal(doc.name.foo[2], 'boo', 'setter works');
t.equal(doc.phrase.foo[0], 'bar', 'setter works');
t.equal(doc.phrase.foo[1], 'baz', 'setter works');
t.equal(doc.phrase.foo[2], 'boo', 'setter works');
t.equal(doc.getName('foo'), 'bar', 'name set');
t.deepEqual(doc.getNameAliases('foo'), ['baz','boo'], 'aliases set');
t.end();
Expand Down
16 changes: 0 additions & 16 deletions test/post/deduplication.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,22 +35,6 @@ module.exports.tests.dedupe = function (test) {

t.end();
});

test('dedupe - phrase', function (t) {
var doc = new Document('mysource', 'mylayer', 'myid');

doc.setNameAlias('default', 'test');
doc.setName('default', 'test');
doc.setNameAlias('default', 'test');
doc.setNameAlias('default', 'test 2');
doc.setNameAlias('default', 'test');
doc.setNameAlias('default', '...Te...st...');

deduplication(doc);
t.deepEquals(doc.phrase.default, ['test', 'test 2']);

t.end();
});
};

module.exports.all = function (tape, common) {
Expand Down
23 changes: 0 additions & 23 deletions test/post/language_field_trimming.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,29 +28,6 @@ module.exports.tests.dedupe = function (test) {
t.end();
});

test('dedupe - phrase', function (t) {
var doc = new Document('mysource', 'mylayer', 'myid');

doc.setName('default', 'test1');
doc.setNameAlias('default', 'test2');
doc.setNameAlias('default', 'test3');

doc.setName('en', 'test1');
doc.setNameAlias('en', 'test3');
doc.setNameAlias('en', 'test4');

doc.setName('de', 'test1');
doc.setNameAlias('de', 'test2');

language_field_trimming(doc);

t.deepEquals(doc.phrase.default, ['test1', 'test2', 'test3']);
t.deepEquals(doc.phrase.en, 'test4');
t.false(doc.phrase.de);

t.end();
});

test('dedupe - two default names, one from a language code', function (t) {
var doc = new Document('mysource', 'mylayer', 'myid');

Expand Down
7 changes: 0 additions & 7 deletions test/serialize/test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ module.exports.tests.minimal = function(test) {
'layer': 'mylayer',
'source_id': 'myid',
'name': {},
'phrase': {},
'parent': {},
'address_parts': {},
'category': [],
Expand Down Expand Up @@ -82,12 +81,6 @@ module.exports.tests.complete = function(test) {
'alt': 'Haggerston City Farm'
},

// place name (phrase analysis)
'phrase':{
'default': 'Hackney City Farm',
'alt': 'Haggerston City Farm'
},

// address data
'address_parts':{
'number': '10',
Expand Down