Update: Hidden Templates and new UI draft

app.py

@@ -254,9 +254,9 @@ def GET(self, res_name):
 		with open(TEMPLATE_LIST,'r') as tpl_file:
 			tpl_list = json.load(tpl_file)
 
-		print(res_name)
 		res_type = ";  ".join([i['type'] for i in tpl_list if i["short_name"] == res_name][0])
 		res_full_name = [i['name'] for i in tpl_list if i["short_name"] == res_name][0]
+		res_status = [i['hidden'] for i in tpl_list if i["short_name"] == res_name][0]
 
 		# if does not exist create the template json file
 		template_path = RESOURCE_TEMPLATES+'template-'+res_name+'.json'
@@ -280,8 +280,9 @@ def GET(self, res_name):
 
 		return render.template(f=fields,user=session['username'],
 								res_type=res_type,res_name=res_full_name,
-								is_git_auth=is_git_auth,project=conf.myProject,
-								skos_vocabs=skos_file,templates=templates)
+								res_status=res_status,is_git_auth=is_git_auth,
+								project=conf.myProject,skos_vocabs=skos_file,
+								templates=templates,)
 
 	def POST(self, res_name):
 		""" Save the form template for data entry and reload config files
@@ -565,7 +566,8 @@ def GET(self, name):
 		logged_in = True if user != 'anonymous' else False
 		block_user, limit = u.check_ip(str(web.ctx['ip']), str(datetime.datetime.now()) )
 		u.check_ask_class()
-		ask_form = u.change_template_names()
+		print("gitauth",is_git_auth)
+		ask_form = u.change_template_names(is_git_auth)
 		f = forms.get_form(ask_form,True)
 
 		return render.record(record_form=f, pageID=name, user=user,
@@ -957,13 +959,14 @@ def GET(self):
 
 		records_by_template , count_by_template , filters_by_template = {} , {} , {}
 		for template in templates:
-			res_class=template["type"]
-			records = queries.getRecords(res_class)
-			records_by_template[template["name"]] = records
-			alll = queries.countAll(res_class,True)
-			count_by_template[template["name"]] = alll
-			filtersBrowse = queries.getBrowsingFilters(template["template"])
-			filters_by_template[template["name"]] = filtersBrowse
+			if not (is_git_auth==False and template["hidden"] =='True'):
+				res_class=template["type"]
+				records = queries.getRecords(res_class)
+				records_by_template[template["name"]] = records
+				alll = queries.countAll(res_class,True)
+				count_by_template[template["name"]] = alll
+				filtersBrowse = queries.getBrowsingFilters(template["template"])
+				filters_by_template[template["name"]] = filtersBrowse
 		return render.records(user=session['username'], data=records_by_template,
 							title='Latest resources', r_base=conf.base,
 							alll=count_by_template, filters=filters_by_template,
@@ -992,14 +995,14 @@ def GET(self, name):
 		base = conf.base
 		record = base+name
 		res_class = queries.getClass(conf.base+name)
-		data, stage, title, properties, data_labels = None, None, None, None, {}
+		data, stage, title, properties, data_labels, extractions_data = None, None, None, None, {}, {}
 
 		try:
 			res_template = u.get_template_from_class(res_class)
 			data = dict(queries.getData(record+'/',res_template))
 			stage = data['stage'][0] if 'stage' in data else 'draft'
 			previous_extractors = u.has_extractor(res_template, name)
-			extractions_data = queries.retrieve_extractions(previous_extractors)
+			extractions_data = queries.retrieve_extractions(previous_extractors,view=True)
 
 			with open(res_template) as tpl_form:
 				fields = json.load(tpl_form)

forms.py

@@ -191,13 +191,13 @@ def get_form(json_form, from_dict=False, subtemplate=False):
 				data_class=res_class), )
 
 			if field['type'] == 'Checkbox':
-				prepend_title = '<section class="checkbox_group_label label col-12">'+description+'</section>'
+				prepend_title = '<section class="checkbox_group_label label col-12">'+prepend+"\n"+'<span class="title">'+description+'</span></section>'
 				i = 0
 				params = params + (form.Checkbox(myid+'-'+str(i),
 				value=dropdown_values[0][0]+','+dropdown_values[0][1],
 				description = dropdown_values[0][1],
 				id=myid,
-				pre = prepend_title+prepend,
+				pre = prepend_title,
 				class_= classes+' checkbox_group',
 				checked=False,
 				data_mandatory = mandatory,
@@ -217,7 +217,7 @@ def get_form(json_form, from_dict=False, subtemplate=False):
 
 			# Subtemplate
 			if field['type'] == 'Subtemplate':
-				resource_class = [t["type"] for t in tpl_list if t["template"] == field['import_subtemplate']][0]
+				resource_class = ";  ".join([t["type"] for t in tpl_list if t["template"] == field['import_subtemplate']][0])
 				params = params + (form.Textbox(myid,
 					description = description,
 					id=myid,
@@ -239,7 +239,7 @@ def get_form(json_form, from_dict=False, subtemplate=False):
 
 searchRecord = form.Form(
 	form.Textbox("search",
-    	class_="searchWikidata col-md-11",
+    	class_="searchWikidata col-md-12",
     	description="Search",
     	autocomplete="off")
 )

mapping.py

@@ -219,7 +219,8 @@ def inputToRDF(recordData, userID, stage, graphToClear=None,tpl_form=None):
 		elif field['type']=="KnowledgeExtractor" and "extractions-dict" in recordData:
 			# process extraction parameters
 			extractions_dict = json.loads(urllib.parse.unquote(recordData["extractions-dict"]))
-			extractions_array = extractions_dict[recordID] if recordID in extractions_dict else []
+			extractions_array_unfiltered = extractions_dict[recordID] if recordID in extractions_dict else []
+			extractions_array = [extraction for extraction in extractions_array_unfiltered if 'metadata' in extraction and 'type' in extraction['metadata']]
 
 			for extraction in extractions_array:
 				extraction_num = str(extraction['internalId'])
@@ -256,6 +257,7 @@ def inputToRDF(recordData, userID, stage, graphToClear=None,tpl_form=None):
 					wd.add(( URIRef(base+graph_name+'/'), URIRef(field['property']), URIRef(base+extraction_graph_name+'/') ))
 
 					# store the extraction metadata
+					queries.clearGraph(base+extraction_graph_name+'/')
 					wd_extraction = rdflib.Graph(identifier=URIRef(base+extraction_graph_name+'/'))
 					wd_extraction.add(( URIRef(base+extraction_graph_name+'/'), PROV.wasAttributedTo, URIRef(base+userID) ))
 					wd_extraction.add(( URIRef(base+extraction_graph_name+'/'), PROV.generatedAtTime, Literal(datetime.datetime.now(),datatype=XSD.dateTime)  ))

queries.py

@@ -411,19 +411,21 @@ def get_subrecords(rdf_property,record_name):
 	return subrecords_list
 
 
-def retrieve_extractions(res_uri_list):
-	"""Return a dictionary of Extractions given a list of tuples: [(named_graph_uri, extraction_rdf_property)]'
+def retrieve_extractions(res_uri_list, view=False):
+	"""Return a dictionary of Extractions given a list of tuples: [(named_graph_uri, extraction_rdf_property, extraction_field_name)]'
 	
 	Parameters
 	----------
 	res_uri_list: list
 		a list of uri (Named Graphs) which may contain Knowledge Extractions
+	view: bool
+		whether to return the extraction_rdf_property and the extraction_field_name for visualization purposes
 	"""
 	res_dict = {}
 
 
 	# Retrieve the extraction graphs (URI) for each Record/Subrecord
-	for uri, rdf_property in res_uri_list:
+	for uri, rdf_property, field_name in res_uri_list:
 		uri_id = uri.replace(conf.base,'')[:-1]
 		query_var_id = uri.rsplit('/',2)[1].replace('-','_')
 		query_pattern = """<"""+uri+"""> <"""+rdf_property+"""> ?extraction_graph_"""+query_var_id+"""."""+\
@@ -435,6 +437,12 @@ def retrieve_extractions(res_uri_list):
 		results = hello_blazegraph(q)
 
 		res_dict[uri_id] = []
+		if view==True:
+			res_dict[uri_id+'_view'] = {
+				'property': rdf_property,
+				'field_name': field_name
+			}
+
 		pattern = re.compile(r'<x-sparql-anything:(.*?)>')
 
 		for result in results["results"]["bindings"]:
@@ -459,6 +467,7 @@ def retrieve_extractions(res_uri_list):
 			comment = metadata['comment']
 
 			res_dict[uri_id].append({"graph":graph, "metadata": {}})
+
 
 			# Api metadata
 			if comment: