Skip to content

Commit

Permalink
Updated I.T. management #10; new Hidden Fields #11
Browse files Browse the repository at this point in the history 
Updated back-end management for Intermediate Templates inputs: mechanism for creating multiple Records (+ related named graphs) based on different Templates; made them available for corrections and reviews; fixed some DOM-related script. Final visualizations (Explore section) is not available yet.
Created an option to make fields hidden. Related mapping behaviours are still to be defined.
  • Loading branch information
@Sebastiano-G
Sebastiano-G committed Feb 12, 2024
1 parent 516d63f commit ff320cf
Show file tree
Hide file tree
Showing 8 changed files with 276 additions and 153 deletions.
238 changes: 120 additions & 118 deletions forms.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,49 +54,72 @@ def get_form(json_form, from_dict=False, subtemplate=False):
res_class = res_class[0] if len(res_class) > 0 else "none"

for field in fields:
# all
myid = field['id']
description = field['label'] if 'label' in field and len(field['label']) > 0 else 'input'
pre_a = '<span class="tip" data-toggle="tooltip" data-placement="bottom" title="'
pre_b = '"><i class="fas fa-info-circle"></i></span>'
prepend = pre_a+field['prepend']+pre_b if 'prepend' in field and len(field['prepend']) > 0 else ''
disabled = 'disabled' if 'disabled' in field and field['disabled'] == "True" else ''
classes = field['class'] if 'class' in field and len(field['class']) > 0 else ''
if 'vocab' in field:
for vocab in field['vocab']:
classes = classes + " " + vocab
classes = classes+' searchWikidata' if 'searchWikidata' in field and field['searchWikidata'] == 'True' else classes
classes = classes+' searchGeonames' if 'searchGeonames' in field and field['searchGeonames'] == 'True' else classes
classes = classes+' urlField' if 'url' in field and field['url'] == 'True' else classes
classes = classes+' disambiguate' if "disambiguate" in field and field["disambiguate"] == 'True' else classes
classes = classes+' multimediaField '+ field['multimedia'] if field['type'] == 'Multimedia' else classes
classes = classes+' vocabularyField' if field['type'] == 'Vocab' else classes
classes = classes+' oneVocableAccepted' if 'vocables' in field and field['vocables'] == 'oneVocable' else classes
classes = classes+' websitePreview' if field['type'] == 'WebsitePreview' else classes
classes = classes+' ('+res_class+') '+disabled
autocomplete = field['cache_autocomplete'] if 'cache_autocomplete' in field and len(field['cache_autocomplete']) > 0 else ''
mandatory = field['mandatory'] if 'mandatory' in field and field['mandatory'] == 'True' else 'False'

# text box
placeholder = field['placeholder'] if 'placeholder' in field else None
default = field['defaultvalue'] if 'defaultvalue' in field else ''
# dropdown
dropdown_values = [(k,v) for k,v in field['values'].items()] if 'values' in field else None

# Text box
if field['type'] in ['Textbox','Vocab', 'WebsitePreview']:
if "disambiguate" in field and field["disambiguate"] == 'True':
vpass = form.regexp(r".{1,200}$", 'must be between 1 and 200 characters')
params = params + (form.Textbox(myid, vpass,
if 'hidden' in field and field['hidden'] == 'False': # do not include hidden fields
# all
myid = field['id']
description = field['label'] if 'label' in field and len(field['label']) > 0 else 'input'
pre_a = '<span class="tip" data-toggle="tooltip" data-placement="bottom" title="'
pre_b = '"><i class="fas fa-info-circle"></i></span>'
prepend = pre_a+field['prepend']+pre_b if 'prepend' in field and len(field['prepend']) > 0 else ''
disabled = 'disabled' if 'disabled' in field and field['disabled'] == "True" else ''
classes = field['class'] if 'class' in field and len(field['class']) > 0 else ''
if 'vocab' in field:
for vocab in field['vocab']:
classes = classes + " " + vocab
classes = classes+' searchWikidata' if 'searchWikidata' in field and field['searchWikidata'] == 'True' else classes
classes = classes+' searchGeonames' if 'searchGeonames' in field and field['searchGeonames'] == 'True' else classes
classes = classes+' urlField' if 'url' in field and field['url'] == 'True' else classes
classes = classes+' disambiguate' if "disambiguate" in field and field["disambiguate"] == 'True' else classes
classes = classes+' multimediaField '+ field['multimedia'] if field['type'] == 'Multimedia' else classes
classes = classes+' vocabularyField' if field['type'] == 'Vocab' else classes
classes = classes+' oneVocableAccepted' if 'vocables' in field and field['vocables'] == 'oneVocable' else classes
classes = classes+' websitePreview' if field['type'] == 'WebsitePreview' else classes
classes = classes+' ('+res_class+') '+disabled
autocomplete = field['cache_autocomplete'] if 'cache_autocomplete' in field and len(field['cache_autocomplete']) > 0 else ''
mandatory = field['mandatory'] if 'mandatory' in field and field['mandatory'] == 'True' else 'False'

# text box
placeholder = field['placeholder'] if 'placeholder' in field else None
default = field['defaultvalue'] if 'defaultvalue' in field else ''
# dropdown
dropdown_values = [(k,v) for k,v in field['values'].items()] if 'values' in field else None

# Text box
if field['type'] in ['Textbox','Vocab', 'WebsitePreview']:
if "disambiguate" in field and field["disambiguate"] == 'True':
vpass = form.regexp(r".{1,200}$", 'must be between 1 and 200 characters')
params = params + (form.Textbox(myid, vpass,
description = description,
id=myid,
placeholder=placeholder,
pre = prepend,
class_= classes,
value=default,
mandatory = mandatory) , )
else:
params = params + (form.Textbox(myid,
description = description,
id=myid,
placeholder=placeholder,
pre = prepend,
class_= classes,
value=default,
mandatory = mandatory), )

# Multimedia Link
if field['type'] == 'Multimedia':
params = params + (form.Textbox(myid,
description = description,
id=myid,
placeholder=placeholder,
pre = prepend,
class_= classes,
value=default,
mandatory = mandatory) , )
else:
params = params + (form.Textbox(myid,


# Text box
if field['type'] == 'Textarea':
params = params + (form.Textarea(myid,
description = description,
id=myid,
placeholder=placeholder,
Expand All @@ -105,96 +128,75 @@ def get_form(json_form, from_dict=False, subtemplate=False):
value=default,
mandatory = mandatory), )

# Multimedia Link
if field['type'] == 'Multimedia':
params = params + (form.Textbox(myid,
description = description,
id=myid,
pre = prepend,
class_= classes,
value=default,
mandatory = mandatory) , )


# Text box
if field['type'] == 'Textarea':
params = params + (form.Textarea(myid,
description = description,
id=myid,
placeholder=placeholder,
pre = prepend,
class_= classes,
value=default,
mandatory = mandatory), )

if field['type'] == 'Date':
if field['calendar'] == 'Month':
params = params + (Month(myid,
description = description,
id=myid,
pre = prepend,
class_= classes,
mandatory=mandatory), )
elif field['calendar'] == 'Day':
params = params + (form.Date(myid,
description = description,
id=myid,
pre = prepend,
class_= classes,
mandatory=mandatory), )
elif field['calendar'] == 'Year':
params = params + (form.Textbox(myid,
if field['type'] == 'Date':
if field['calendar'] == 'Month':
params = params + (Month(myid,
description = description,
id=myid,
pre = prepend,
class_= classes,
mandatory=mandatory), )
elif field['calendar'] == 'Day':
params = params + (form.Date(myid,
description = description,
id=myid,
pre = prepend,
class_= classes,
mandatory=mandatory), )
elif field['calendar'] == 'Year':
params = params + (form.Textbox(myid,
description = description,
id=myid,
pre = prepend,
class_= classes,
value=default,
mandatory=mandatory), )

if field['type'] == 'Dropdown':
params = params + (form.Dropdown(myid,
description = description,
args=dropdown_values,
placeholder=placeholder,
id=myid,
pre = prepend,
class_= classes,
value=default,
mandatory=mandatory), )

if field['type'] == 'Dropdown':
params = params + (form.Dropdown(myid,
description = description,
args=dropdown_values,
placeholder=placeholder,
id=myid,
pre = prepend,
class_= classes,
mandatory = mandatory), )

if field['type'] == 'Checkbox':
prepend_title = '<section class="checkbox_group_label label col-12">'+description+'</section>'
i = 0
params = params + (form.Checkbox(myid+'-'+str(i),
value=dropdown_values[0][0]+','+dropdown_values[0][1],
description = dropdown_values[0][1],
id=myid,
pre = prepend_title+prepend,
class_= classes+' checkbox_group',
checked=False,
mandatory = mandatory), )

for value in dropdown_values[1:]:
i += 1
mandatory = mandatory), )

if field['type'] == 'Checkbox':
prepend_title = '<section class="checkbox_group_label label col-12">'+description+'</section>'
i = 0
params = params + (form.Checkbox(myid+'-'+str(i),
value=value[0]+','+value[1],
description = value[1],
value=dropdown_values[0][0]+','+dropdown_values[0][1],
description = dropdown_values[0][1],
id=myid,
pre = '',
class_= classes+' checkbox_group following_checkbox',
pre = prepend_title+prepend,
class_= classes+' checkbox_group',
checked=False,
mandatory = mandatory), )

if field['type'] == 'Subtemplate':
resource_class = [t["type"] for t in tpl_list if t["template"] == field['import_subtemplate']][0]
params = params + (form.Textbox(myid,
description = description,
id=myid,
placeholder=placeholder,
pre = prepend,
class_= classes,
value=default,
mandatory = mandatory,
subtemplate = resource_class), ) + get_form(field['import_subtemplate'], subtemplate=True)
for value in dropdown_values[1:]:
i += 1
params = params + (form.Checkbox(myid+'-'+str(i),
value=value[0]+','+value[1],
description = value[1],
id=myid,
pre = '',
class_= classes+' checkbox_group following_checkbox',
checked=False,
mandatory = mandatory), )

# subtemplate
if field['type'] == 'Subtemplate':
resource_class = [t["type"] for t in tpl_list if t["template"] == field['import_subtemplate']][0]
params = params + (form.Textbox(myid,
description = description,
id=myid,
placeholder=placeholder,
pre = prepend,
class_= classes,
value=default,
mandatory = mandatory,
subtemplate = resource_class), ) + get_form(field['import_subtemplate'], subtemplate=True)

if subtemplate:
return params
Expand Down
69 changes: 63 additions & 6 deletions mapping.py
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
# -*- coding: utf-8 -*-
import os
import datetime
import time
import json
import urllib.parse
import web
from web import form
from web import form, storify
import rdflib
from rdflib import URIRef , XSD, Namespace , Literal
from rdflib.namespace import OWL, DC , DCTERMS, RDF , RDFS
Expand Down Expand Up @@ -60,14 +61,15 @@ def getRightURIbase(value):
return WD+value if value.startswith('Q') else GEO+value if value.isdecimal() else VIAF+value[4:] if value.startswith("viaf") else ''+value if value.startswith("http") else base+value


def inputToRDF(recordData, userID, stage, knowledge_extraction, graphToClear=None,tpl_form=None):
def inputToRDF(recordData, userID, stage, knowledge_extraction, graphToClear=None,tpl_form=None, subrecords_dict=None):
""" transform input data into RDF, upload data to the triplestore, dump data locally """

# MAPPING FORM / PROPERTIES
if tpl_form:
with open(tpl_form) as config_form:
fields = json.load(config_form)
else:
else:
#should this be deleted?
with open(conf.myform) as config_form:
fields = json.load(config_form)

Expand Down Expand Up @@ -109,7 +111,7 @@ def inputToRDF(recordData, userID, stage, knowledge_extraction, graphToClear=Non
wd.add(( URIRef(base+graph_name+'/'), RDFS.label, Literal("no title") ))

for field in fields:
if field['type'] != 'KnowledgeExtractor':
if field['type'] not in ['KnowledgeExtractor', 'Subtemplate']:
# URI, Textarea (only text at this stage), Literals
value = getValuesFromFields(field['id'], recordData, fields) \
if 'value' in field and field['value'] in ['URI','Place'] else getValuesFromFields(field['id'], recordData, field_type=field['type']) if 'value' in field and field['value'] == 'URL' else recordData[field['id']]
Expand Down Expand Up @@ -158,7 +160,7 @@ def inputToRDF(recordData, userID, stage, knowledge_extraction, graphToClear=Non
wd.add(( URIRef(base+graph_name), SCHEMA.keywords, URIRef(entityURI) ))
wd.add(( URIRef( entityURI ), RDFS.label, Literal(entity[1].lstrip().rstrip(), datatype="http://www.w3.org/2001/XMLSchema#string") ))
# KNOWLEDGE EXTRACTION: import graphs
else:
elif field['type']=="KnowledgeExtractor":
with open(knowledge_extraction) as extraction_file:
extraction = json.load(extraction_file)
imported_graphs = extraction[recordID] if recordID in extraction else []
Expand All @@ -176,7 +178,18 @@ def inputToRDF(recordData, userID, stage, knowledge_extraction, graphToClear=Non
wd_extraction.add(( URIRef(urllib.parse.unquote(recordData[label])), RDFS.label, Literal(label.replace(start, ""))))
wd_extraction.serialize(destination='records/'+recordID+"-extraction-"+str(graph['internalID'])+'.ttl', format='ttl', encoding='utf-8')
server.update('load <file:///'+dir_path+'/records/'+recordID+"-extraction-"+str(graph['internalID'])+'.ttl> into graph <'+base+extraction_graph_name+'/>')

# SUBTEMPLATE
elif field['type']=="Subtemplate":
subrecords = process_subrecords(recordData, field['id']) if not subrecords_dict else subrecords_dict
if field['id'] in subrecords:
for subrecord_idx, subrecord in subrecords[field['id']].items():
ID = str(int(time.time() * 1000))
subrecord['recordID'] = ID
label = find_label(field['import_subtemplate'], subrecord, field['label'])
inputToRDF(storify(subrecord),userID,stage,knowledge_extraction,tpl_form=field['import_subtemplate'],subrecords_dict=subrecord)
wd.add(( URIRef(base+graph_name), URIRef(field['property']), URIRef(base+ID) ))
wd.add(( URIRef(base+ID), RDFS.label, Literal(label, datatype="http://www.w3.org/2001/XMLSchema#string")))


# get keywords (record modify)
if stage == 'modified' and any([k for k,v in recordData.items() if k.startswith('keywords')]):
Expand All @@ -195,3 +208,47 @@ def inputToRDF(recordData, userID, stage, knowledge_extraction, graphToClear=Non
server.update('load <file:///'+dir_path+'/records/'+recordID+'.ttl> into graph <'+base+graph_name+'/>')

return 'records/'+recordID+'.ttl'

# convert the dict of inputs into a series of nested dictionaries to be parsed as single records
def process_subrecords(data, id):
results = {}
created_subrecords = [key for key in data if key.startswith(id+"-")]
if created_subrecords != []:
for subrecord in created_subrecords:
add_results = {}
subrecord_split = subrecord.split('-')
prefix, num = subrecord_split[0], subrecord_split[-1]
subrecord_fields = data[subrecord].split(',')
inner_subrecords = [key for item in subrecord_fields for key in data.keys() if key.startswith(item + "-")]
for key in subrecord_fields:
if data[key] != "":
add_results[key.split('-')[0]] = data[key]
else:
inner_subrecords = [inner_subrecord for inner_subrecord in data.keys() if inner_subrecord.startswith(key + "-")]
for inner_subrecord in inner_subrecords:
if inner_subrecord.startswith(key + '-'):
inner_subrecord_split = inner_subrecord.split('-')
inner_prefix, inner_num = inner_subrecord_split[0], inner_subrecord_split[-1]
add_results[inner_prefix] = {
inner_num: process_subrecords(data, inner_subrecord)
}
if prefix in results:
results[prefix][num] = add_results
else:
results[prefix] = { num: add_results }
else:
for el in data[id].split(','):
results[el.split('-')[0]] = data[el]
return results

def find_label(tpl, subrecord, alternative_label):
print(tpl)
# Retrieve the field associated with the Primary Key (i.e., the label) of the Record
with open(tpl) as tpl_file:
tpl_fields = json.load(tpl_file)
label_field_id = [field['id'] for field in tpl_fields if field['disambiguate'] == "True"][0]

# Add a mechanism to handle potential Templates without a Primary Key (e.g. the primary key has been set to "hidden")
label = subrecord[label_field_id] if label_field_id in subrecord else alternative_label+"-"+subrecord['recordID']
return label

2 changes: 1 addition & 1 deletion resource_templates/template_list.json
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
[{"name": "Example template", "short_name": "example_template", "type": "http://example.org/ExampleClass", "template": "resource_templates/template-example_template.json"}]
[{"name": "Example template", "short_name": "example_template", "type": "http://example.org/ExampleClass", "hidden": "False", "template": "resource_templates/template-example_template.json"}]
Loading

0 comments on commit ff320cf

Please sign in to comment.