diff --git a/Dockerfile b/Dockerfile index 14c171c4f15..b52647d85a5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 as base +FROM ubuntu:22.04 as base USER root ## Setting default environment variables @@ -30,17 +30,17 @@ RUN set -ex \ docbook-mathml \ libgdal-dev \ libpq-dev \ - python3.8 \ - python3.8-dev \ + python3.10 \ + python3.10-dev \ curl \ - python3.8-distutils \ + python3.10-distutils \ libldap2-dev libsasl2-dev ldap-utils \ dos2unix \ " \ && apt-get update -y \ && apt-get install -y --no-install-recommends $BUILD_DEPS \ && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \ - && python3.8 get-pip.py + && python3.10 get-pip.py RUN pip3 wheel --no-cache-dir -r ${WHEELS}/requirements.txt \ && pip3 wheel --no-cache-dir -r ${WHEELS}/requirements_dev.txt \ @@ -70,20 +70,23 @@ RUN set -ex \ libgdal-dev \ python3-venv \ postgresql-client-12 \ - python3.8 \ - python3.8-distutils \ - python3.8-venv \ + python3.10 \ + python3.10-distutils \ + python3.10-venv \ " \ && apt-get update -y \ - && apt-get install -y --no-install-recommends curl \ - && curl -sL https://deb.nodesource.com/setup_14.x | bash - \ + && apt-get install -y --no-install-recommends curl ca-certificates gnupg \ + && mkdir -p /etc/apt/keyrings \ + && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \ + && NODE_MAJOR=16 \ + && (echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" > /etc/apt/sources.list.d/nodesource.list) \ && curl -sL https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \ && add-apt-repository "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -sc)-pgdg main" \ && apt-get update -y \ && apt-get install -y --no-install-recommends $RUN_DEPS \ && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \ - && python3.8 get-pip.py \ - && apt-get install -y nodejs \ + && python3.10 get-pip.py \ + && apt-get -y install --no-install-recommends nodejs \ && npm install -g yarn # Install Yarn components @@ -99,7 +102,7 @@ WORKDIR ${WEB_ROOT} RUN mv ${WHEELS}/entrypoint.sh entrypoint.sh -RUN python3.8 -m venv ENV \ +RUN python3.10 -m venv ENV \ && . ENV/bin/activate \ && pip install wheel setuptools requests \ && pip install rjsmin==1.2.0 MarkupSafe==2.0.0 \ diff --git a/arches/app/datatypes/base.py b/arches/app/datatypes/base.py index 23425368f03..0b5dd6a6f53 100644 --- a/arches/app/datatypes/base.py +++ b/arches/app/datatypes/base.py @@ -222,14 +222,14 @@ def get_search_terms(self, nodevalue, nodeid=None): """ return [] - def append_search_filters(self, value, node, query, request): + def append_search_filters(self, value, node, query, parameters): """ Allows for modification of an elasticsearch bool query for use in advanced search """ pass - def append_null_search_filters(self, value, node, query, request): + def append_null_search_filters(self, value, node, query, parameters): """ Appends the search query dsl to search for fields that have not been populated """ @@ -298,7 +298,7 @@ def get_default_language_value_from_localized_node(self, tile, nodeid): """ return tile.data[str(nodeid)] - def post_tile_save(self, tile, nodeid, request): + def post_tile_save(self, tile, nodeid, parameters, user): """ Called after the tile is saved to the database diff --git a/arches/app/datatypes/concept_types.py b/arches/app/datatypes/concept_types.py index f805fc77aaf..2a0391ca188 100644 --- a/arches/app/datatypes/concept_types.py +++ b/arches/app/datatypes/concept_types.py @@ -124,10 +124,10 @@ def append_to_document(self, document, nodevalue, nodeid, tile, provisional=Fals ) document["strings"].append({"string": value.value, "nodegroup_id": tile.nodegroup_id, "provisional": provisional}) - def append_search_filters(self, value, node, query, request): + def append_search_filters(self, value, node, query, parameters): try: if value["op"] == "null" or value["op"] == "not_null": - self.append_null_search_filters(value, node, query, request) + self.append_null_search_filters(value, node, query, parameters) elif value["val"] != "": match_query = Match(field="tiles.data.%s" % (str(node.pk)), type="phrase", query=value["val"]) if "!" in value["op"]: @@ -385,4 +385,4 @@ def collects_multiple_values(self): return True def ignore_keys(self): - return ["http://www.w3.org/2000/01/rdf-schema#label http://www.w3.org/2000/01/rdf-schema#Literal"] \ No newline at end of file + return ["http://www.w3.org/2000/01/rdf-schema#label http://www.w3.org/2000/01/rdf-schema#Literal"] diff --git a/arches/app/datatypes/datatypes.py b/arches/app/datatypes/datatypes.py index b8f3324b5b1..2d726fc9c20 100644 --- a/arches/app/datatypes/datatypes.py +++ b/arches/app/datatypes/datatypes.py @@ -188,7 +188,7 @@ def transform_export_values(self, value, *args, **kwargs): else: return value[get_language()]["value"] except KeyError: - # sometimes certain requested language values aren't populated. Just pass back with implicit None. + # sometimes certain parametersed language values aren't populated. Just pass back with implicit None. pass def get_search_terms(self, nodevalue, nodeid=None): @@ -203,7 +203,7 @@ def get_search_terms(self, nodevalue, nodeid=None): pass return terms - def append_null_search_filters(self, value, node, query, request): + def append_null_search_filters(self, value, node, query, parameters): """ Appends the search query dsl to search for fields that have not been populated or are empty strings """ @@ -228,10 +228,10 @@ def append_null_search_filters(self, value, node, query, request): non_blank_string_query = Term(field=f"tiles.data.{str(node.pk)}.{value['lang']}.value.keyword", query="") query.should(Nested(path="tiles", query=non_blank_string_query)) - def append_search_filters(self, value, node, query, request): + def append_search_filters(self, value, node, query, parameters): try: if value["op"] == "null" or value["op"] == "not_null": - self.append_null_search_filters(value, node, query, request) + self.append_null_search_filters(value, node, query, parameters) elif value["val"] != "": exact_terms = re.search('"(?P.*)"', value["val"]) if exact_terms: @@ -430,10 +430,10 @@ def pre_tile_save(self, tile, nodeid): def append_to_document(self, document, nodevalue, nodeid, tile, provisional=False): document["numbers"].append({"number": nodevalue, "nodegroup_id": tile.nodegroup_id, "provisional": provisional}) - def append_search_filters(self, value, node, query, request): + def append_search_filters(self, value, node, query, parameters): try: if value["op"] == "null" or value["op"] == "not_null": - self.append_null_search_filters(value, node, query, request) + self.append_null_search_filters(value, node, query, parameters) elif value["val"] != "": if value["op"] != "eq": operators = {"gte": None, "lte": None, "lt": None, "gt": None} @@ -521,11 +521,11 @@ def to_json(self, tile, node): def transform_value_for_tile(self, value, **kwargs): return bool(util.strtobool(str(value))) - def append_search_filters(self, value, node, query, request): + def append_search_filters(self, value, node, query, parameters): try: if value["val"] == "null" or value["val"] == "not_null": value["op"] = value["val"] - self.append_null_search_filters(value, node, query, request) + self.append_null_search_filters(value, node, query, parameters) elif value["val"] != "" and value["val"] is not None: term = True if value["val"] == "t" else False query.must(Term(field="tiles.data.%s" % (str(node.pk)), term=term)) @@ -655,10 +655,10 @@ def append_to_document(self, document, nodevalue, nodeid, tile, provisional=Fals {"date": ExtendedDateFormat(nodevalue).lower, "nodegroup_id": tile.nodegroup_id, "nodeid": nodeid, "provisional": provisional} ) - def append_search_filters(self, value, node, query, request): + def append_search_filters(self, value, node, query, parameters): try: if value["op"] == "null" or value["op"] == "not_null": - self.append_null_search_filters(value, node, query, request) + self.append_null_search_filters(value, node, query, parameters) elif value["val"] != "" and value["val"] is not None: try: date_value = datetime.strptime(value["val"], "%Y-%m-%d %H:%M:%S%z").astimezone().isoformat() @@ -781,7 +781,7 @@ def add_date_to_doc(document, edtf): add_date_to_doc(document, edtf) add_date_to_doc(tile.data[nodeid], edtf) - def append_search_filters(self, value, node, query, request): + def append_search_filters(self, value, node, query, parameters): def add_date_to_doc(query, edtf): if value["op"] == "eq": if edtf.lower != edtf.upper: @@ -809,7 +809,7 @@ def add_date_to_doc(query, edtf): raise Exception(_("Invalid date specified.")) if value["op"] == "null" or value["op"] == "not_null": - self.append_null_search_filters(value, node, query, request) + self.append_null_search_filters(value, node, query, parameters) elif value["val"] != "" and value["val"] is not None: edtf = ExtendedDateFormat(value["val"]) if edtf.result_set: @@ -1570,14 +1570,20 @@ def to_json(self, tile, node): if data: return self.compile_json(tile, node, file_details=data[str(node.nodeid)]) - def post_tile_save(self, tile, nodeid, request): - if request is not None: + def post_tile_save(self, tile, nodeid, parameters, user=None): + if parameters is not None: # this does not get called when saving data from the mobile app previously_saved_tile = models.TileModel.objects.filter(pk=tile.tileid) - user = request.user - if hasattr(request.user, "userprofile") is not True: - models.UserProfile.objects.create(user=request.user) - user_is_reviewer = user_is_resource_reviewer(request.user) + user_is_reviewer = False + if user is True: + user_is_reviewer = True + elif user: + if hasattr(parameters.user, "userprofile") is not True: + models.UserProfile.objects.create(user=parameters.user) + user_is_reviewer = user_is_resource_reviewer(parameters.user) + else: + # There must be a user to be able to upload files. + return current_tile_data = self.get_tile_data(tile) if previously_saved_tile.count() == 1: previously_saved_tile_data = self.get_tile_data(previously_saved_tile[0]) @@ -1594,7 +1600,7 @@ def post_tile_save(self, tile, nodeid, request): except models.File.DoesNotExist: logger.exception(_("File does not exist")) - files = request.FILES.getlist("file-list_" + nodeid + "_preloaded", []) + request.FILES.getlist("file-list_" + nodeid, []) + files = parameters["FILES"].getlist("file-list_" + nodeid + "_preloaded", []) + parameters["FILES"].getlist("file-list_" + nodeid, []) tile_exists = models.TileModel.objects.filter(pk=tile.tileid).exists() for file_data in files: @@ -1649,7 +1655,7 @@ def transform_value_for_tile(self, value, **kwargs): Accepts a comma delimited string of file paths as 'value' to create a file datatype value with corresponding file record in the files table for each path. Only the basename of each path is used, so the accuracy of the full path is not important. However the name of each file must match the name of a file in - the directory from which Arches will request files. By default, this is the directory in a project as defined + the directory from which Arches will parameters files. By default, this is the directory in a project as defined in settings.UPLOADED_FILES_DIR. """ @@ -1696,7 +1702,7 @@ def transform_value_for_tile(self, value, **kwargs): return json.loads(json.dumps(tile_data)) def pre_tile_save(self, tile, nodeid): - # TODO If possible this method should probably replace 'handle request' + # TODO If possible this method should probably replace 'handle parameters' if tile.data[nodeid]: for file in tile.data[nodeid]: try: @@ -1960,10 +1966,10 @@ def transform_export_values(self, value, *args, **kwargs): ret = value return ret - def append_search_filters(self, value, node, query, request): + def append_search_filters(self, value, node, query, parameters): try: if value["op"] == "null" or value["op"] == "not_null": - self.append_null_search_filters(value, node, query, request) + self.append_null_search_filters(value, node, query, parameters) elif value["val"] != "": search_query = Match(field="tiles.data.%s" % (str(node.pk)), type="phrase", query=value["val"]) if "!" in value["op"]: @@ -2136,10 +2142,10 @@ def transform_export_values(self, value, *args, **kwargs): new_values.append(val) return ",".join(new_values) - def append_search_filters(self, value, node, query, request): + def append_search_filters(self, value, node, query, parameters): try: if value["op"] == "null" or value["op"] == "not_null": - self.append_null_search_filters(value, node, query, request) + self.append_null_search_filters(value, node, query, parameters) elif value["val"] != "" and value["val"] != []: search_query = Match(field="tiles.data.%s" % (str(node.pk)), type="phrase", query=value["val"]) if "!" in value["op"]: @@ -2235,7 +2241,7 @@ def pre_tile_save(self, tile, nodeid): for relationship in relationships: relationship["resourceXresourceId"] = str(uuid.uuid4()) - def post_tile_save(self, tile, nodeid, request): + def post_tile_save(self, tile, nodeid, parameters, user): ret = False sql = """ SELECT * FROM __arches_create_resource_x_resource_relationships('%s') as t; @@ -2303,10 +2309,16 @@ def transform_value_for_tile(self, value, **kwargs): return json.loads(value) except ValueError: # do this if json (invalid) is formatted with single quotes, re #6390 - try: - return ast.literal_eval(value) - except: + if "'" in value: + try: + return ast.literal_eval(value) + except: + return value + elif isinstance(value, str): + return [{"resourceId": val} for val in value.split(";")] + else: return value + except TypeError: # data should come in as json but python list is accepted as well if isinstance(value, list): @@ -2315,10 +2327,10 @@ def transform_value_for_tile(self, value, **kwargs): def transform_export_values(self, value, *args, **kwargs): return json.dumps(value) - def append_search_filters(self, value, node, query, request): + def append_search_filters(self, value, node, query, parameters): try: if value["op"] == "null" or value["op"] == "not_null": - self.append_null_search_filters(value, node, query, request) + self.append_null_search_filters(value, node, query, parameters) elif value["val"] != "" and value["val"] != []: # search_query = Match(field="tiles.data.%s.resourceId" % (str(node.pk)), type="phrase", query=value["val"]) search_query = Terms(field="tiles.data.%s.resourceId.keyword" % (str(node.pk)), terms=value["val"]) @@ -2454,7 +2466,7 @@ def get_display_value(self, tile, node, **kwargs): def append_to_document(self, document, nodevalue, nodeid, tile, provisional=False): pass - def append_search_filters(self, value, node, query, request): + def append_search_filters(self, value, node, query, parameters): pass diff --git a/arches/app/datatypes/url.py b/arches/app/datatypes/url.py index c543be22a12..9d98bfad4d7 100644 --- a/arches/app/datatypes/url.py +++ b/arches/app/datatypes/url.py @@ -151,7 +151,7 @@ def get_search_terms(self, nodevalue, nodeid=None): # terms.append(nodevalue['url']) FIXME: URLs searchable? return terms - def append_search_filters(self, value, node, query, request): + def append_search_filters(self, value, node, query, parameters): # Match the label in the same manner as a String datatype try: if value["val"] != "": diff --git a/arches/app/media/css/arches.scss b/arches/app/media/css/arches.scss index a1f755fa650..31b35d9abc4 100644 --- a/arches/app/media/css/arches.scss +++ b/arches/app/media/css/arches.scss @@ -10339,6 +10339,11 @@ ul.pagination { padding: 10px 5px 0px 10px; } +.search-listing-title.principal-user { + font-weight: 1000; + background: rgb(204,230,244); + padding-bottom: 5px; +} .search-listing-title.i18n-alt a span { font-size: 13px; } diff --git a/arches/app/media/js/views/components/search/search-results.js b/arches/app/media/js/views/components/search/search-results.js index 70a7d9efae3..88874257937 100644 --- a/arches/app/media/js/views/components/search/search-results.js +++ b/arches/app/media/js/views/components/search/search-results.js @@ -202,6 +202,9 @@ function($, _, BaseFilter, bootstrap, arches, select2, ko, koMapping, GraphModel resourceinstanceid: result._source.resourceinstanceid, displaydescription: result._source.displaydescription, alternativelanguage: result._source.displayname_language != arches.activeLanguage, + principaluser: ko.computed(function () { + return result._source.permissions.principal_user && result._source.permissions.principal_user.includes(ko.unwrap(self.userid)); + }), "map_popup": result._source.map_popup, "provisional_resource": result._source.provisional_resource, geometries: ko.observableArray(result._source.geometries), @@ -252,4 +255,4 @@ function($, _, BaseFilter, bootstrap, arches, select2, ko, koMapping, GraphModel }), template: searchResultsTemplate }); -}); \ No newline at end of file +}); diff --git a/arches/app/media/js/views/user-profile-manager.js b/arches/app/media/js/views/user-profile-manager.js index 2e4d6cb5290..30c620f3738 100644 --- a/arches/app/media/js/views/user-profile-manager.js +++ b/arches/app/media/js/views/user-profile-manager.js @@ -21,6 +21,7 @@ define([ self.viewModel.mismatchedPasswords = ko.observable(); self.viewModel.changePasswordSuccess = ko.observable(); self.viewModel.notifTypeObservables = ko.observableArray(); + self.viewModel.roleObservables = ko.observableArray(); self.viewModel.isTwoFactorAuthenticationEnabled = data.two_factor_authentication_settings['ENABLE_TWO_FACTOR_AUTHENTICATION']; self.viewModel.isTwoFactorAuthenticationForced = data.two_factor_authentication_settings['FORCE_TWO_FACTOR_AUTHENTICATION']; @@ -54,6 +55,31 @@ define([ }; self.viewModel.getNotifTypes(); + self.viewModel.getRoles = function() { + self.viewModel.roleObservables.removeAll(); + $.ajax({ + url: arches.urls.get_user_roles, + context: this, + data: { userids: [] }, + method: 'POST', + dataType: 'json' + }).done(function(data) { + var koType; + var user = Object.keys(data); + if (user.length == 1) { + const userId = user[0]; + Object.values(data[userId]).forEach(function(role) { + if (!role.name || role.name === "") { + role.name = "(direct)"; + } + koRole = ko.mapping.fromJS(role); + self.viewModel.roleObservables.push(koRole); + }); + } + }); + }; + self.viewModel.getRoles(); + self.viewModel.updateNotifTypes = function() { var modified; var updatedTypes = self.viewModel.notifTypeObservables().map(function(type) { diff --git a/arches/app/models/graph.py b/arches/app/models/graph.py index ec99b5c5344..e3d6c54e6a7 100644 --- a/arches/app/models/graph.py +++ b/arches/app/models/graph.py @@ -188,10 +188,9 @@ def check_default_configs(default_configs, configs): edge_lookup = {edge["edgeid"]: edge for edge in json.loads(JSONSerializer().serialize(edges))} - for card in cards: - widgets = list(card.cardxnodexwidget_set.all()) - for widget in widgets: - self.widgets[widget.pk] = widget + widgets = models.CardXNodeXWidget.objects.filter(card__in=cards).all() + for widget in widgets: + self.widgets[widget.pk] = widget node_lookup = {} for node in nodes: diff --git a/arches/app/models/migrations/10999_add_principaluserid_to_resources.py b/arches/app/models/migrations/10999_add_principaluserid_to_resources.py new file mode 100644 index 00000000000..4c80ac4d794 --- /dev/null +++ b/arches/app/models/migrations/10999_add_principaluserid_to_resources.py @@ -0,0 +1,16 @@ +# Generated by Django 2.2.6 on 2019-12-13 11:56 + +from django.conf import settings +from django.db import migrations, models +from django.db.models import deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("models", "9604_relational_data_model_handle_user_accounts"), + ] + + operations = [ + migrations.AddField(model_name="resourceinstance", name="principaluser", field=models.ForeignKey(on_delete=deletion.SET_NULL, to=settings.AUTH_USER_MODEL, null=True)), + ] diff --git a/arches/app/models/migrations/9604_relational_data_model_handle_user_accounts.py b/arches/app/models/migrations/9604_relational_data_model_handle_user_accounts.py new file mode 100644 index 00000000000..be7634ece7d --- /dev/null +++ b/arches/app/models/migrations/9604_relational_data_model_handle_user_accounts.py @@ -0,0 +1,400 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("models", "10121_workflowhistory"), + ] + + operations = [ + migrations.RunSQL( + """ + create or replace function __arches_get_node_value_sql( + node public.nodes + ) returns text as $$ + declare + node_value_sql text; + select_sql text = '(t.tiledata->>%L)'; + datatype text = 'text'; + begin + select_sql = format(select_sql, node.nodeid); + case node.datatype + when 'geojson-feature-collection' then + select_sql = format(' + st_collect( + array( + select st_transform(geom, 4326) from geojson_geometries + where geojson_geometries.tileid = t.tileid and nodeid = %L + ) + )', + node.nodeid + ); + datatype = 'geometry'; + when 'number' then datatype = 'numeric'; + when 'boolean' then datatype = 'boolean'; + when 'resource-instance' then datatype = 'jsonb'; + when 'resource-instance-list' then datatype = 'jsonb'; + when 'annotation' then datatype = 'jsonb'; + when 'file-list' then datatype = 'jsonb'; + when 'url' then datatype = 'jsonb'; + when 'date' then select_sql = format( + 'to_date( + t.tiledata->>%L::text, + %L + )', + node.nodeid, + node.config->>'dateFormat' + ); + datatype = 'timestamp'; + when 'node-value' then datatype = 'uuid'; + when 'domain-value' then datatype = 'uuid'; + when 'domain-value-list' then select_sql = format( + '( + CASE + WHEN t.tiledata->>%1$L is null THEN null + ELSE ARRAY( + SELECT jsonb_array_elements_text( + t.tiledata->%1$L + )::uuid + ) + END + )', + node.nodeid + ); + datatype = 'uuid[]'; + when 'concept' then datatype = 'uuid'; + when 'concept-list' then + select_sql = format('( + CASE + WHEN t.tiledata->>%1$L is null THEN null + ELSE ARRAY( + SELECT jsonb_array_elements_text( + t.tiledata->%1$L + )::uuid + ) + END + )', node.nodeid + ); + datatype = 'uuid[]'; + else + datatype = 'text'; + end case; + + node_value_sql = format( + '%s::%s as "%s"', + select_sql, + datatype, + __arches_slugify(node.name) + ); + return node_value_sql; + end + $$ language plpgsql volatile; + + create or replace function __arches_get_json_data_for_view( + view_row anyelement, + schema_name text, + view_name text + ) returns json as $$ + declare + column_info record; + query text; + result jsonb; + geom geometry; + geometry_type text; + geometry_query text; + node public.nodes; + tiledata jsonb = '{}'::jsonb; + begin + for column_info in select a.attname as column_name, + d.description + from pg_class as c + inner join pg_attribute as a on c.oid = a.attrelid + left join pg_namespace n on n.oid = c.relnamespace + left join pg_tablespace t on t.oid = c.reltablespace + left join pg_description as d on ( + d.objoid = c.oid + and d.objsubid = a.attnum + ) + where c.relkind in('r', 'v') + and n.nspname = schema_name + and c.relname = view_name + and d.description is not null + and d.description != 'parenttileid' + loop + select n.* into node + from nodes n where n.nodeid = column_info.description::uuid; + if node.datatype = 'geojson-feature-collection' then + query = format( + 'select st_geometrytype( + ($1::text::%s.%s).%s + )', + schema_name, + view_name, + column_info.column_name + ); + execute query into geometry_type using view_row; + if geometry_type = 'ST_GeometryCollection' or geometry_type like 'ST_Multi%' then + geometry_query = E'from ( + select st_asgeojson( + st_dump( + ($1::text::%s. %s).%s + ) + )::json->\\'geometry\\' as geom + ) as g'; + else + geometry_query = 'from ( + select st_asgeojson( + ($1::text::%s. %s).%s + ) as geom + ) as g'; + end if; + query = format( + E'select json_build_object( + \\'type\\', + \\'FeatureCollection\\', + \\'features\\', + json_agg( + json_build_object( + \\'type\\', + \\'Feature\\', + \\'geometry\\', + g.geom::json, + \\'properties\\', + json_build_object() + ) + ) + )' || geometry_query, + schema_name, + view_name, + column_info.column_name + ); + elsif node.datatype = 'date' then + query = format( + 'select to_json( + to_char( + ($1::text::%s.%s).%s, + %L + ) + )', + schema_name, + view_name, + column_info.column_name, + node.config->>'dateFormat' + ); + else + query = format( + 'select to_json( + ($1::text::%s.%s).%s + )', + schema_name, + view_name, + column_info.column_name + ); + end if; + execute query into result using view_row; + if node.datatype in ('resource-instance-list', 'resource-instance') then + select jsonb_agg( + case + when e->>'resourceXresourceId' = '' then jsonb_set( + e, + '{resourceXresourceId}', + to_jsonb(public.uuid_generate_v1mc()) + ) + else e + end + ) into result + from jsonb_array_elements(result) e(e); + end if; + tiledata = tiledata || jsonb_build_object(column_info.description, result); + end loop; + + return tiledata::json; + end + $$ language plpgsql volatile; + """, + """ + create or replace function __arches_get_node_value_sql( + node public.nodes + ) returns text as $$ + declare + node_value_sql text; + select_sql text = '(t.tiledata->>%L)'; + datatype text = 'text'; + begin + select_sql = format(select_sql, node.nodeid); + case node.datatype + when 'geojson-feature-collection' then + select_sql = format(' + st_collect( + array( + select st_transform(geom, 4326) from geojson_geometries + where geojson_geometries.tileid = t.tileid and nodeid = %L + ) + )', + node.nodeid + ); + datatype = 'geometry'; + when 'number' then datatype = 'numeric'; + when 'boolean' then datatype = 'boolean'; + when 'resource-instance' then datatype = 'jsonb'; + when 'resource-instance-list' then datatype = 'jsonb'; + when 'annotation' then datatype = 'jsonb'; + when 'file-list' then datatype = 'jsonb'; + when 'url' then datatype = 'jsonb'; + when 'date' then datatype = 'timestamp'; + when 'node-value' then datatype = 'uuid'; + when 'domain-value' then datatype = 'uuid'; + when 'domain-value-list' then select_sql = format( + '( + CASE + WHEN t.tiledata->>%1$L is null THEN null + ELSE ARRAY( + SELECT jsonb_array_elements_text( + t.tiledata->%1$L + )::uuid + ) + END + )', + node.nodeid + ); + datatype = 'uuid[]'; + when 'concept' then datatype = 'uuid'; + when 'concept-list' then + select_sql = format('( + CASE + WHEN t.tiledata->>%1$L is null THEN null + ELSE ARRAY( + SELECT jsonb_array_elements_text( + t.tiledata->%1$L + )::uuid + ) + END + )', node.nodeid + ); + datatype = 'uuid[]'; + else + datatype = 'text'; + end case; + + node_value_sql = format( + '%s::%s as "%s"', + select_sql, + datatype, + __arches_slugify(node.name) + ); + return node_value_sql; + end + $$ language plpgsql volatile; + + create or replace function __arches_get_json_data_for_view( + view_row anyelement, + schema_name text, + view_name text + ) returns json as $$ + declare + column_info record; + query text; + result jsonb; + geom geometry; + geometry_type text; + geometry_query text; + node_datatype text; + tiledata jsonb = '{}'::jsonb; + begin + for column_info in select a.attname as column_name, + d.description + from pg_class as c + inner join pg_attribute as a on c.oid = a.attrelid + left join pg_namespace n on n.oid = c.relnamespace + left join pg_tablespace t on t.oid = c.reltablespace + left join pg_description as d on ( + d.objoid = c.oid + and d.objsubid = a.attnum + ) + where c.relkind in('r', 'v') + and n.nspname = schema_name + and c.relname = view_name + and d.description is not null + and d.description != 'parenttileid' + loop + select datatype into node_datatype + from nodes where nodeid = column_info.description::uuid; + if node_datatype = 'geojson-feature-collection' then + query = format( + 'select st_geometrytype( + ($1::text::%s.%s).%s + )', + schema_name, + view_name, + column_info.column_name + ); + execute query into geometry_type using view_row; + if geometry_type = 'ST_GeometryCollection' or geometry_type like 'ST_Multi%' then + geometry_query = E'from ( + select st_asgeojson( + st_dump( + ($1::text::%s. %s).%s + ) + )::json->\\'geometry\\' as geom + ) as g'; + else + geometry_query = 'from ( + select st_asgeojson( + ($1::text::%s. %s).%s + ) as geom + ) as g'; + end if; + query = format( + E'select json_build_object( + \\'type\\', + \\'FeatureCollection\\', + \\'features\\', + json_agg( + json_build_object( + \\'type\\', + \\'Feature\\', + \\'geometry\\', + g.geom::json, + \\'properties\\', + json_build_object() + ) + ) + )' || geometry_query, + schema_name, + view_name, + column_info.column_name + ); + else + query = format( + 'select to_json( + ($1::text::%s.%s).%s + )', + schema_name, + view_name, + column_info.column_name + ); + end if; + execute query into result using view_row; + if node_datatype in ('resource-instance-list', 'resource-instance') then + select jsonb_agg( + case + when e->>'resourceXresourceId' = '' then jsonb_set( + e, + '{resourceXresourceId}', + to_jsonb(public.uuid_generate_v1mc()) + ) + else e + end + ) into result + from jsonb_array_elements(result) e(e); + end if; + tiledata = tiledata || jsonb_build_object(column_info.description, result); + end loop; + + return tiledata::json; + end + $$ language plpgsql volatile; + """, + ) + ] diff --git a/arches/app/models/models.py b/arches/app/models/models.py index 127bb9692c0..141164f8a14 100644 --- a/arches/app/models/models.py +++ b/arches/app/models/models.py @@ -25,7 +25,7 @@ from django.template.loader import get_template, render_to_string from django.core.validators import RegexValidator from django.db.models import Q, Max -from django.db.models.signals import post_delete, pre_save, post_save +from django.db.models.signals import post_delete, pre_save, post_save, m2m_changed from django.dispatch import receiver from django.utils import translation from django.utils.translation import gettext as _ @@ -33,8 +33,6 @@ from django.contrib.auth.models import Group from django.contrib.contenttypes.models import ContentType from django.core.validators import validate_slug -from guardian.models import GroupObjectPermission -from guardian.shortcuts import assign_perm # can't use "arches.app.models.system_settings.SystemSettings" because of circular refernce issue # so make sure the only settings we use in this file are ones that are static (fixed at run time) @@ -956,6 +954,14 @@ class ResourceInstance(models.Model): descriptors = models.JSONField(blank=True, null=True) legacyid = models.TextField(blank=True, unique=True, null=True) createdtime = models.DateTimeField(auto_now_add=True) + # This could be used as a lock, but primarily addresses the issue that a creating user + # may not yet match the criteria to edit a ResourceInstance (via Set/LogicalSet) simply + # because the details may not yet be complete. Only one user can create, as it is an + # action, not a state, so we do not need an array here. That may be desirable depending on + # future use of this field (e.g. locking to a group). + # Note that this is intended to bypass normal permissions logic, so a resource type must + # prevent a user who created the resource from editing it, by updating principaluserid logic. + principaluser = models.ForeignKey(User, on_delete=models.SET_NULL, null=True) def save(self, *args, **kwargs): try: @@ -1343,19 +1349,19 @@ def is_reviewer(self): def viewable_nodegroups(self): from arches.app.utils.permission_backend import get_nodegroups_by_perm - return set(str(nodegroup.pk) for nodegroup in get_nodegroups_by_perm(self.user, ["models.read_nodegroup"], any_perm=True)) + return set(str(nodegroup_pk) for nodegroup_pk in get_nodegroups_by_perm(self.user, ["models.read_nodegroup"], any_perm=True)) @property def editable_nodegroups(self): from arches.app.utils.permission_backend import get_nodegroups_by_perm - return set(str(nodegroup.pk) for nodegroup in get_nodegroups_by_perm(self.user, ["models.write_nodegroup"], any_perm=True)) + return set(str(nodegroup_pk) for nodegroup_pk in get_nodegroups_by_perm(self.user, ["models.write_nodegroup"], any_perm=True)) @property def deletable_nodegroups(self): from arches.app.utils.permission_backend import get_nodegroups_by_perm - return set(str(nodegroup.pk) for nodegroup in get_nodegroups_by_perm(self.user, ["models.delete_nodegroup"], any_perm=True)) + return set(str(nodegroup_pk) for nodegroup_pk in get_nodegroups_by_perm(self.user, ["models.delete_nodegroup"], any_perm=True)) class Meta: managed = True @@ -1367,23 +1373,6 @@ def save_profile(sender, instance, **kwargs): UserProfile.objects.get_or_create(user=instance) -@receiver(post_save, sender=User) -def create_permissions_for_new_users(sender, instance, created, **kwargs): - from arches.app.models.resource import Resource - - if created: - ct = ContentType.objects.get(app_label="models", model="resourceinstance") - resourceInstanceIds = list(GroupObjectPermission.objects.filter(content_type=ct).values_list("object_pk", flat=True).distinct()) - for resourceInstanceId in resourceInstanceIds: - resourceInstanceId = uuid.UUID(resourceInstanceId) - resources = ResourceInstance.objects.filter(pk__in=resourceInstanceIds) - assign_perm("no_access_to_resourceinstance", instance, resources) - for resource_instance in resources: - resource = Resource(resource_instance.resourceinstanceid) - resource.graph_id = resource_instance.graph_id - resource.createdtime = resource_instance.createdtime - resource.index() - class UserXTask(models.Model): id = models.UUIDField(primary_key=True, serialize=False) @@ -1498,6 +1487,33 @@ class Meta: managed = True db_table = "user_x_notification_types" +@receiver(post_save, sender=User) +def create_permissions_for_new_users(sender, instance, created, **kwargs): + from arches.app.utils.permission_backend import process_new_user + + if created: + process_new_user(instance, created) + +@receiver(m2m_changed, sender=User.groups.through) +def update_groups_for_user(sender, instance, action, **kwargs): + from arches.app.utils.permission_backend import update_groups_for_user + + if action in ("post_add", "post_remove"): + update_groups_for_user(instance) + +@receiver(m2m_changed, sender=User.user_permissions.through) +def update_permissions_for_user(sender, instance, action, **kwargs): + from arches.app.utils.permission_backend import update_permissions_for_user + + if action in ("post_add", "post_remove"): + update_permissions_for_user(instance) + +@receiver(m2m_changed, sender=Group.permissions.through) +def update_permissions_for_group(sender, instance, action, **kwargs): + from arches.app.utils.permission_backend import update_permissions_for_group + + if action in ("post_add", "post_remove"): + update_permissions_for_group(instance) @receiver(post_save, sender=UserXNotification) def send_email_on_save(sender, instance, **kwargs): diff --git a/arches/app/models/resource.py b/arches/app/models/resource.py index 2e5fb9c72fe..897f9b87bcf 100644 --- a/arches/app/models/resource.py +++ b/arches/app/models/resource.py @@ -40,8 +40,7 @@ from arches.app.utils import import_class_from_string, task_management from arches.app.utils.label_based_graph import LabelBasedGraph from arches.app.utils.label_based_graph_v2 import LabelBasedGraph as LabelBasedGraphV2 -from guardian.shortcuts import assign_perm, remove_perm -from guardian.exceptions import NotUserNorGroup +from arches.app.utils.permission_backend import assign_perm, remove_perm, NotUserNorGroup from arches.app.utils.betterJSONSerializer import JSONSerializer, JSONDeserializer from arches.app.utils.exceptions import ( InvalidNodeNameException, @@ -52,10 +51,13 @@ get_restricted_users, get_restricted_instances, ) +import django.dispatch from arches.app.datatypes.datatypes import DataTypeFactory logger = logging.getLogger(__name__) +resource_indexed = django.dispatch.Signal() + class Resource(models.ResourceInstance): class Meta: @@ -222,16 +224,20 @@ def save(self, *args, **kwargs): index = kwargs.pop("index", True) context = kwargs.pop("context", None) transaction_id = kwargs.pop("transaction_id", None) - super(Resource, self).save(*args, **kwargs) - for tile in self.tiles: - tile.resourceinstance_id = self.resourceinstanceid - tile.save(request=request, index=False, transaction_id=transaction_id, context=context) + if request is None: if user is None: user = {} else: user = request.user + if not self.principaluser_id and user: + self.principaluser_id = user.id + + super(Resource, self).save(*args, **kwargs) + for tile in self.tiles: + tile.resourceinstance_id = self.resourceinstanceid + tile.save(request=request, index=False, transaction_id=transaction_id, context=context) try: for perm in ("view_resourceinstance", "change_resourceinstance", "delete_resourceinstance"): assign_perm(perm, user, self) @@ -352,6 +358,7 @@ def index(self, context=None): es_index.index_document(document=doc, id=doc_id) super(Resource, self).save() + resource_indexed.send(sender=self.__class__, instance=self) def get_documents_to_index(self, fetchTiles=True, datatype_factory=None, node_datatypes=None, context=None): """ @@ -377,6 +384,7 @@ def get_documents_to_index(self, fetchTiles=True, datatype_factory=None, node_da document["displayname"] = [] document["displaydescription"] = [] + document["sets"] = [] document["map_popup"] = [] for lang in settings.LANGUAGES: if context is None: @@ -420,6 +428,10 @@ def get_documents_to_index(self, fetchTiles=True, datatype_factory=None, node_da document["permissions"]["users_without_edit_perm"] = restrictions["cannot_write"] document["permissions"]["users_without_delete_perm"] = restrictions["cannot_delete"] document["permissions"]["users_with_no_access"] = restrictions["no_access"] + if self.principaluser_id: + document["permissions"]["principal_user"] = [int(self.principaluser_id)] + else: + document["permissions"]["principal_user"] = [] document["strings"] = [] document["dates"] = [] document["domains"] = [] @@ -528,6 +540,32 @@ def delete(self, user={}, index=True, transaction_id=None): return permit_deletion + def get_index(self, resourceinstanceid=None): + """ + Gets the indexed document for a resource + + Keyword Arguments: + resourceinstanceid -- the resource instance id to delete from related indexes, if supplied will use this over self.resourceinstanceid + """ + + if resourceinstanceid is None: + resourceinstanceid = self.resourceinstanceid + resourceinstanceid = str(resourceinstanceid) + + # delete any related terms + query = Query(se) + bool_query = Bool() + bool_query.must(Terms(field="_id", terms=[resourceinstanceid])) + query.add_query(bool_query) + query.include("sets") + query.include("permissions.principal_user") + results = query.search(index=RESOURCES_INDEX) + if len(results["hits"]["hits"]) < 1: + raise UnindexedError("This resource is not (yet) indexed") + if len(results["hits"]["hits"]) > 1: + raise RuntimeError("Resource instance ID exists multiple times in search index") + return results["hits"]["hits"][0] + def delete_index(self, resourceinstanceid=None): """ Deletes all references to a resource from all indexes @@ -873,3 +911,12 @@ def __init__(self, message, code=None): def __str__(self): return repr(self.message) + +class UnindexedError(Exception): + def __init__(self, message, code=None): + self.title = _("Unindexed Error") + self.message = message + self.code = code + + def __str__(self): + return repr(self.message) diff --git a/arches/app/models/tile.py b/arches/app/models/tile.py index dedbd417c7f..fae8d3cba90 100644 --- a/arches/app/models/tile.py +++ b/arches/app/models/tile.py @@ -338,6 +338,10 @@ def validate(self, errors=None, raise_early=True, strict=False, request=None): except TypeError: # will catch if serialized_graph is None node = models.Node.objects.get(nodeid=nodeid) datatype = self.datatype_factory.get_instance(node.datatype) + parameters = {} + for bag in (request.GET, request.POST): + for key in bag: + parameters[key] = bag[key] error = datatype.validate(value, node=node, strict=strict, request=request) tile_errors += error for error_instance in error: @@ -382,7 +386,11 @@ def datatype_post_save_actions(self, request=None): except: node = models.Node.objects.get(nodeid=nodeid) datatype = self.datatype_factory.get_instance(node.datatype) - datatype.post_tile_save(self, nodeid, request) + datatype.post_tile_save(self, nodeid, { + "GET": request.GET if request else {}, + "POST": request.POST if request else {}, + "FILES": request.FILES if request else {} + }, request.user if request else None) def save(self, *args, **kwargs): request = kwargs.pop("request", None) @@ -402,7 +410,8 @@ def save(self, *args, **kwargs): try: if user is None and request is not None: user = request.user - user_is_reviewer = user_is_resource_reviewer(user) + if user is not None: + user_is_reviewer = user_is_resource_reviewer(user) except AttributeError: # no user - probably importing data user = None @@ -506,7 +515,10 @@ def delete(self, *args, **kwargs): tile.delete(*args, request=request, **kwargs) try: user = request.user - user_is_reviewer = user_is_resource_reviewer(user) + if user is not None: + user_is_reviewer = user_is_resource_reviewer(user) + else: + user_is_reviewer = True except AttributeError: # no user user = None user_is_reviewer = True diff --git a/arches/app/permissions/__init__.py b/arches/app/permissions/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/arches/app/permissions/arches_allow_with_credentials.py b/arches/app/permissions/arches_allow_with_credentials.py new file mode 100644 index 00000000000..657f51ba370 --- /dev/null +++ b/arches/app/permissions/arches_allow_with_credentials.py @@ -0,0 +1,30 @@ +from arches.app.permissions.arches_standard import ArchesStandardPermissionFramework + +class ArchesAllowWithCredentialsFramework(ArchesStandardPermissionFramework): + def get_sets_for_user(self, user, perm): + # We do not do set filtering - None is allow-all for sets. + return None if user and user.username != "anonymous" else set() + + def check_resource_instance_permissions(self, user, resourceid, permission): + result = super().check_resource_instance_permissions(user, resourceid, permission) + + if result and result.get("permitted", None) is not None: + if result["permitted"] == "unknown": + if not user or user.username == "anonymous": + result["permitted"] = False + elif result["permitted"] == False: + + # This covers the case where one group denies permission and another + # allows it. Ideally, the deny would override (as normal in Arches) but + # this prevents us from having a default deny rule that another group + # can override (as deny rules in Arches must be explicit for a resource). + resource = ResourceInstance.objects.get(resourceinstanceid=resourceid) + user_permissions = get_user_perms(user, resource) + if "no_access_to_resourceinstance" not in user_permissions: + group_permissions = get_group_perms(user, resource) + + # This should correspond to the exact case we wish to flip. + if permission in group_permissions: + result["permitted"] = True + + return result diff --git a/arches/app/permissions/arches_default_deny.py b/arches/app/permissions/arches_default_deny.py new file mode 100644 index 00000000000..2322f4f2c8d --- /dev/null +++ b/arches/app/permissions/arches_default_deny.py @@ -0,0 +1,33 @@ +from arches.app.permissions.arches_standard import ArchesStandardPermissionFramework +from arches.app.search.components.resource_type_filter import get_permitted_graphids + +class ArchesDefaultDenyPermissionFramework(ArchesStandardPermissionFramework): + def get_sets_for_user(self, user, perm): + # We do not do set filtering - None is allow-all for sets. + return None if user and user.username != "anonymous" else set() + + def check_resource_instance_permissions(self, user, resourceid, permission): + result = super().check_resource_instance_permissions(user, resourceid, permission) + + if result and result.get("permitted", None) is not None: + # This is a safety check - we don't want an unpermissioned user + # defaulting to having access (allowing anonymous users is still + # possible by assigning appropriate group permissions). + if result["permitted"] == "unknown": + result["permitted"] = False + elif result["permitted"] == False: + + # This covers the case where one group denies permission and another + # allows it. Ideally, the deny would override (as normal in Arches) but + # this prevents us from having a default deny rule that another group + # can override (as deny rules in Arches must be explicit for a resource). + resource = ResourceInstance.objects.get(resourceinstanceid=resourceid) + user_permissions = get_user_perms(user, resource) + if "no_access_to_resourceinstance" not in user_permissions: + group_permissions = get_group_perms(user, resource) + + # This should correspond to the exact case we wish to flip. + if permission in group_permissions: + result["permitted"] = True + + return result diff --git a/arches/app/permissions/arches_standard.py b/arches/app/permissions/arches_standard.py new file mode 100644 index 00000000000..b98202294a6 --- /dev/null +++ b/arches/app/permissions/arches_standard.py @@ -0,0 +1,596 @@ +from django.core.exceptions import ObjectDoesNotExist +from arches.app.models.system_settings import settings, SystemSettings +from django.contrib.auth.models import User, Group +from django.contrib.gis.db.models import Model +from django.core.cache import caches +from guardian.backends import check_support, ObjectPermissionBackend +from guardian.core import ObjectPermissionChecker +from guardian.shortcuts import ( + get_perms, + get_group_perms, + get_user_perms, + get_users_with_perms, + get_groups_with_perms, + get_perms_for_model, +) +from guardian.exceptions import NotUserNorGroup +from arches.app.models.resource import Resource + +from guardian.models import GroupObjectPermission, UserObjectPermission +from guardian.exceptions import WrongAppError +from guardian.shortcuts import assign_perm, get_perms, remove_perm, get_group_perms, get_user_perms + +import inspect +from arches.app.models.models import * +from arches.app.models.models import ResourceInstance, MapLayer +from arches.app.search.elasticsearch_dsl_builder import Bool, Query, Terms, Nested +from arches.app.search.mappings import RESOURCES_INDEX +from arches.app.utils.permission_backend import PermissionFramework, NotUserNorGroup as ArchesNotUserNorGroup + +class ArchesStandardPermissionFramework(PermissionFramework): + def setup(self): + ... + + def get_perms_for_model(self, cls): + return get_perms_for_model(cls) + + def assign_perm(self, perm, user_or_group, obj=None): + try: + return assign_perm(perm, user_or_group, obj=obj) + except NotUserNorGroup: + raise ArchesNotUserNorGroup() + + def get_permission_backend(self): + return PermissionBackend() + + def remove_perm(self, perm, user_or_group=None, obj=None): + return remove_perm(perm, user_or_group=user_or_group, obj=obj) + + def get_perms(self, user_or_group, obj): + return get_perms(user_or_group, obj) + + def get_group_perms(self, user_or_group, obj): + return get_group_perms(user_or_group, obj) + + def get_user_perms(self, user, obj): + return get_user_perms(user, obj) + + def process_new_user(self, instance, created): + ct = ContentType.objects.get(app_label="models", model="resourceinstance") + resourceInstanceIds = list(GroupObjectPermission.objects.filter(content_type=ct).values_list("object_pk", flat=True).distinct()) + for resourceInstanceId in resourceInstanceIds: + resourceInstanceId = uuid.UUID(resourceInstanceId) + resources = ResourceInstance.objects.filter(pk__in=resourceInstanceIds) + self.assign_perm("no_access_to_resourceinstance", instance, resources) + for resource_instance in resources: + resource = Resource(resource_instance.resourceinstanceid) + resource.graph_id = resource_instance.graph_id + resource.createdtime = resource_instance.createdtime + resource.index() + + def get_map_layers_by_perm(self, user, perms, any_perm=True): + """ + returns a list of node groups that a user has the given permission on + + Arguments: + user -- the user to check + perms -- the permssion string eg: "read_map_layer" or list of strings + any_perm -- True to check ANY perm in "perms" or False to check ALL perms + + """ + + if not isinstance(perms, list): + perms = [perms] + + formatted_perms = [] + # in some cases, `perms` can have a `model.` prefix + for perm in perms: + if len(perm.split(".")) > 1: + formatted_perms.append(perm.split(".")[1]) + else: + formatted_perms.append(perm) + + if user.is_superuser is True: + return MapLayer.objects.all() + else: + permitted_map_layers = list() + + user_permissions = ObjectPermissionChecker(user) + + for map_layer in MapLayer.objects.all(): + if map_layer.addtomap is True and map_layer.isoverlay is False: + permitted_map_layers.append(map_layer) + else: # if no explicit permissions, object is considered accessible by all with group permissions + explicit_map_layer_perms = user_permissions.get_perms(map_layer) + if len(explicit_map_layer_perms): + if any_perm: + if len(set(formatted_perms) & set(explicit_map_layer_perms)): + permitted_map_layers.append(map_layer) + else: + if set(formatted_perms) == set(explicit_map_layer_perms): + permitted_map_layers.append(map_layer) + elif map_layer.ispublic: + permitted_map_layers.append(map_layer) + + return permitted_map_layers + + def user_can_read_map_layers(self, user): + + map_layers_with_read_permission = self.get_map_layers_by_perm(user, ['models.read_maplayer']) + map_layers_allowed = [] + + for map_layer in map_layers_with_read_permission: + if ('no_access_to_maplayer' not in get_user_perms(user, map_layer)) or (map_layer.addtomap is False and map_layer.isoverlay is False): + map_layers_allowed.append(map_layer) + + return map_layers_allowed + + + def user_can_write_map_layers(self, user): + map_layers_with_write_permission = self.get_map_layers_by_perm(user, ['models.write_maplayer']) + map_layers_allowed = [] + + for map_layer in map_layers_with_write_permission: + if ('no_access_to_maplayer' not in get_user_perms(user, map_layer)) or (map_layer.addtomap is False and map_layer.isoverlay is False): + map_layers_allowed.append(map_layer) + + return map_layers_allowed + + def get_nodegroups_by_perm(self, user, perms, any_perm=True): + """ + returns a list of node groups that a user has the given permission on + + Arguments: + user -- the user to check + perms -- the permssion string eg: "read_nodegroup" or list of strings + any_perm -- True to check ANY perm in "perms" or False to check ALL perms + + """ + return list(set(str(nodegroup.pk) for nodegroup in get_nodegroups_by_perm_for_user_or_group(user, perms, any_perm=any_perm))) + + def check_resource_instance_permissions(self, user, resourceid, permission): + """ + Checks if a user has permission to access a resource instance + + Arguments: + user -- the user to check + resourceid -- the id of the resource + permission -- the permission codename (e.g. 'view_resourceinstance') for which to check + + """ + result = {} + try: + resource = ResourceInstance.objects.get(resourceinstanceid=resourceid) + result["resource"] = resource + + all_perms = self.get_perms(user, resource) + + if len(all_perms) == 0: # no permissions assigned. permission implied + result["permitted"] = "unknown" + return result + else: + user_permissions = self.get_user_perms(user, resource) + if "no_access_to_resourceinstance" in user_permissions: # user is restricted + result["permitted"] = False + return result + elif permission in user_permissions: # user is permitted + result["permitted"] = True + return result + + group_permissions = self.get_group_perms(user, resource) + if "no_access_to_resourceinstance" in group_permissions: # group is restricted - no user override + result["permitted"] = False + return result + elif permission in group_permissions: # group is permitted - no user override + result["permitted"] = True + return result + + if permission not in all_perms: # neither user nor group explicitly permits or restricts. + result["permitted"] = False # restriction implied + return result + + except ObjectDoesNotExist: + result["permitted"] = True # if the object does not exist, no harm in returning true - this prevents strange 403s. + + return result + + def get_users_with_perms(self, obj, attach_perms=False, with_superusers=False, with_group_users=True, only_with_perms_in=None): + return get_users_with_perms(obj, attach_perms=attach_perms, with_superusers=with_superusers, with_group_users=with_group_users, only_with_perms_in=only_with_perms_in) + + def get_groups_with_perms(self, obj, attach_perms=False): + return get_groups_with_perms(obj, attach_perms=attach_perms) + + def get_restricted_users(self, resource): + """ + Takes a resource instance and identifies which users are explicitly restricted from + reading, editing, deleting, or accessing it. + + """ + + user_perms = get_users_with_perms(resource, attach_perms=True, with_group_users=False) + user_and_group_perms = get_users_with_perms(resource, attach_perms=True, with_group_users=True) + + result = { + "no_access": [], + "cannot_read": [], + "cannot_write": [], + "cannot_delete": [], + } + + for user, perms in user_and_group_perms.items(): + if user.is_superuser: + pass + elif user in user_perms and "no_access_to_resourceinstance" in user_perms[user]: + for k, v in result.items(): + v.append(user.id) + else: + if "view_resourceinstance" not in perms: + result["cannot_read"].append(user.id) + if "change_resourceinstance" not in perms: + result["cannot_write"].append(user.id) + if "delete_resourceinstance" not in perms: + result["cannot_delete"].append(user.id) + if "no_access_to_resourceinstance" in perms and len(perms) == 1: + result["no_access"].append(user.id) + + return result + + def get_groups_for_object(self, perm, obj): + """ + returns a list of group objects that have the given permission on the given object + + Arguments: + perm -- the permssion string eg: "read_nodegroup" + obj -- the model instance to check + + """ + + def has_group_perm(group, perm, obj): + explicitly_defined_perms = self.get_perms(group, obj) + if len(explicitly_defined_perms) > 0: + if "no_access_to_nodegroup" in explicitly_defined_perms: + return False + else: + return perm in explicitly_defined_perms + else: + default_perms = [] + for permission in group.permissions.all(): + if perm in permission.codename: + return True + return False + + ret = [] + for group in Group.objects.all(): + if has_group_perm(group, perm, obj): + ret.append(group) + return ret + + + def get_sets_for_user(self, user, perm): + # We do not do set filtering - None is allow-all for sets. + return None + + def get_users_for_object(self, perm, obj): + """ + Returns a list of user objects that have the given permission on the given object + + Arguments: + perm -- the permssion string eg: "read_nodegroup" + obj -- the model instance to check + + """ + + ret = [] + for user in User.objects.all(): + if user.has_perm(perm, obj): + ret.append(user) + return ret + + + def get_restricted_instances(self, user, search_engine=None, allresources=False): + if allresources is False and user.is_superuser is True: + return [] + + if allresources is True: + restricted_group_instances = { + perm["object_pk"] + for perm in GroupObjectPermission.objects.filter(permission__codename="no_access_to_resourceinstance").values("object_pk") + } + restricted_user_instances = { + perm["object_pk"] + for perm in UserObjectPermission.objects.filter(permission__codename="no_access_to_resourceinstance").values("object_pk") + } + all_restricted_instances = list(restricted_group_instances | restricted_user_instances) + return all_restricted_instances + else: + terms = Terms(field="permissions.users_with_no_access", terms=[str(user.id)]) + query = Query(search_engine, start=0, limit=settings.SEARCH_RESULT_LIMIT) + has_access = Bool() + nested_term_filter = Nested(path="permissions", query=terms) + has_access.must(nested_term_filter) + query.add_query(has_access) + results = query.search(index=RESOURCES_INDEX, scroll="1m") + scroll_id = results["_scroll_id"] + total = results["hits"]["total"]["value"] + if total > settings.SEARCH_RESULT_LIMIT: + pages = total // settings.SEARCH_RESULT_LIMIT + for page in range(pages): + results_scrolled = query.se.es.scroll(scroll_id=scroll_id, scroll="1m") + results["hits"]["hits"] += results_scrolled["hits"]["hits"] + restricted_ids = [res["_id"] for res in results["hits"]["hits"]] + return restricted_ids + + def update_groups_for_user(self, user): + """Hook for spotting group updates on a user.""" + ... + + def update_permissions_for_user(self, user): + """Hook for spotting permission updates on a user.""" + ... + + def update_permissions_for_group(self, group): + """Hook for spotting permission updates on a group.""" + ... + + def user_has_resource_model_permissions(self, user, perms, resource): + """ + Checks if a user has any explicit permissions to a model's nodegroups + + Arguments: + user -- the user to check + perms -- the permssion string eg: "read_nodegroup" or list of strings + resource -- a resource instance to check if a user has permissions to that resource's type specifically + + """ + + nodegroups = self.get_nodegroups_by_perm(user, perms) + nodes = Node.objects.filter(nodegroup__in=nodegroups).filter(graph_id=resource.graph_id).select_related("graph") + return nodes.exists() + + + def user_can_read_resource(self, user, resourceid=None): + """ + Requires that a user be able to read an instance and read a single nodegroup of a resource + + """ + if user.is_authenticated: + if user.is_superuser: + return True + if resourceid not in [None, ""]: + result = self.check_resource_instance_permissions(user, resourceid, "view_resourceinstance") + if result is not None: + if result["permitted"] == "unknown": + return self.user_has_resource_model_permissions(user, ["models.read_nodegroup"], result["resource"]) + else: + return result["permitted"] + else: + return None + + return len(self.get_resource_types_by_perm(user, ["models.read_nodegroup"])) > 0 + return False + + def get_resource_types_by_perm(self, user, perms): + graphs = set() + nodegroups = self.get_nodegroups_by_perm(user, perms) + for node in Node.objects.filter(nodegroup__in=nodegroups).prefetch_related("graph"): + if node.graph.isresource and str(node.graph_id) != SystemSettings.SYSTEM_SETTINGS_RESOURCE_MODEL_ID: + graphs.add(str(node.graph.pk)) + return list(graphs) + + + def user_can_edit_resource(self, user, resourceid=None): + """ + Requires that a user be able to edit an instance and delete a single nodegroup of a resource + + """ + if user.is_authenticated: + if user.is_superuser: + return True + if resourceid not in [None, ""]: + result = self.check_resource_instance_permissions(user, resourceid, "change_resourceinstance") + if result is not None: + if result["permitted"] == "unknown": + return user.groups.filter(name__in=settings.RESOURCE_EDITOR_GROUPS).exists() or self.user_can_edit_model_nodegroups( + user, result["resource"] + ) + else: + return result["permitted"] + else: + return None + + return user.groups.filter(name__in=settings.RESOURCE_EDITOR_GROUPS).exists() or len(self.get_editable_resource_types(user)) > 0 + return False + + + def user_can_delete_resource(self, user, resourceid=None): + """ + Requires that a user be permitted to delete an instance + + """ + if user.is_authenticated: + if user.is_superuser: + return True + if resourceid not in [None, ""]: + result = self.check_resource_instance_permissions(user, resourceid, "delete_resourceinstance") + if result is not None: + if result["permitted"] == "unknown": + nodegroups = self.get_nodegroups_by_perm(user, "models.delete_nodegroup") + tiles = TileModel.objects.filter(resourceinstance_id=resourceid) + protected_tiles = {str(tile.nodegroup_id) for tile in tiles} - set(nodegroups) + if len(protected_tiles) > 0: + return False + return user.groups.filter(name__in=settings.RESOURCE_EDITOR_GROUPS).exists() or self.user_can_delete_model_nodegroups( + user, result["resource"] + ) + else: + return result["permitted"] + else: + return None + return False + + + def user_can_read_concepts(self, user): + """ + Requires that a user is a part of the RDM Administrator group + + """ + + if user.is_authenticated: + return user.groups.filter(name="RDM Administrator").exists() + return False + + + def user_is_resource_editor(self, user): + """ + Single test for whether a user is in the Resource Editor group + """ + + return user.groups.filter(name="Resource Editor").exists() + + + def user_is_resource_reviewer(self, user): + """ + Single test for whether a user is in the Resource Reviewer group + """ + + return user.groups.filter(name="Resource Reviewer").exists() + + + def user_is_resource_exporter(self, user): + """ + Single test for whether a user is in the Resource Exporter group + """ + + return user.groups.filter(name="Resource Exporter").exists() + + def user_in_group_by_name(self, user, names): + return bool(user.groups.filter(name__in=names)) + + + +class PermissionBackend(ObjectPermissionBackend): + def has_perm(self, user_obj, perm, obj=None): + # check if user_obj and object are supported (pulled directly from guardian) + support, user_obj = check_support(user_obj, obj) + if not support: + return False + + if "." in perm: + app_label, perm = perm.split(".") + if app_label != obj._meta.app_label: + raise WrongAppError("Passed perm has app label of '%s' and " "given obj has '%s'" % (app_label, obj._meta.app_label)) + + ObjPermissionChecker = CachedObjectPermissionChecker(user_obj, obj) + explicitly_defined_perms = ObjPermissionChecker.get_perms(obj) + + if len(explicitly_defined_perms) > 0: + if "no_access_to_nodegroup" in explicitly_defined_perms: + return False + else: + return bool(perm in explicitly_defined_perms) + else: + UserPermissionChecker = CachedUserPermissionChecker(user_obj) + return bool(UserPermissionChecker.user_has_permission(perm)) + + +class CachedUserPermissionChecker: + """ + A permission checker that leverages the 'user_permission' cache to check user-level user permissions. + """ + + def __init__(self, user): + user_permission_cache = caches["user_permission"] + current_user_cached_permissions = user_permission_cache.get(str(user.pk), {}) + + if current_user_cached_permissions.get("user_permissions"): + user_permissions = current_user_cached_permissions.get("user_permissions") + else: + user_permissions = set() + + for group in user.groups.all(): + for group_permission in group.permissions.all(): + user_permissions.add(group_permission.codename) + + for user_permission in user.user_permissions.all(): + user_permissions.add(user_permission.codename) + + current_user_cached_permissions["user_permissions"] = user_permissions + user_permission_cache.set(str(user.pk), current_user_cached_permissions) + + self.user_permissions = user_permissions + + def user_has_permission(self, permission): + if permission in self.user_permissions: + return True + else: + return False + +class CachedObjectPermissionChecker: + """ + A permission checker that leverages the 'user_permission' cache to check object-level user permissions. + """ + + def __new__(cls, user, input): + if inspect.isclass(input): + classname = input.__name__ + elif isinstance(input, Model): + classname = input.__class__.__name__ + elif isinstance(input, str) and globals().get(input): + classname = input + else: + raise Exception("Cannot derive model from input.") + + user_permission_cache = caches["user_permission"] + + key = f"g:{user.pk}" if isinstance(user, Group) else str(user.pk) + current_user_cached_permissions = user_permission_cache.get(key, {}) + + if current_user_cached_permissions.get(classname): + checker = current_user_cached_permissions.get(classname) + else: + checker = ObjectPermissionChecker(user) + checker.prefetch_perms(globals()[classname].objects.all()) + + current_user_cached_permissions[classname] = checker + user_permission_cache.set(key, current_user_cached_permissions) + + return checker + +def get_nodegroups_by_perm_for_user_or_group(user_or_group, perms=None, any_perm=True, ignore_perms=False): + formatted_perms = [] + if perms is None: + if not ignore_perms: + raise RuntimeError("Must provide perms or explicitly ignore") + else: + if not isinstance(perms, list): + perms = [perms] + + # in some cases, `perms` can have a `model.` prefix + for perm in perms: + if len(perm.split(".")) > 1: + formatted_perms.append(perm.split(".")[1]) + else: + formatted_perms.append(perm) + + permitted_nodegroups = {} + NodegroupPermissionsChecker = CachedObjectPermissionChecker( + user_or_group, + NodeGroup, + ) + + for nodegroup in NodeGroup.objects.all(): + explicit_perms = NodegroupPermissionsChecker.get_perms(nodegroup) + + if len(explicit_perms): + if ignore_perms: + permitted_nodegroups[nodegroup] = explicit_perms + elif any_perm: + if len(set(formatted_perms) & set(explicit_perms)): + permitted_nodegroups[nodegroup] = explicit_perms + else: + if set(formatted_perms) == set(explicit_perms): + permitted_nodegroups[nodegroup] = explicit_perms + else: # if no explicit permissions, object is considered accessible by all with group permissions + permitted_nodegroups[nodegroup] = set() + + return permitted_nodegroups diff --git a/arches/app/search/components/advanced_search.py b/arches/app/search/components/advanced_search.py index acdc986ee39..aa067693397 100644 --- a/arches/app/search/components/advanced_search.py +++ b/arches/app/search/components/advanced_search.py @@ -21,7 +21,7 @@ class AdvancedSearch(BaseSearchFilter): def append_dsl(self, search_results_object, permitted_nodegroups, include_provisional): - querysting_params = self.request.GET.get(details["componentname"], "") + querysting_params = self.parameters.get(details["componentname"], "") advanced_filters = JSONDeserializer().deserialize(querysting_params) datatype_factory = DataTypeFactory() search_query = Bool() @@ -34,15 +34,15 @@ def append_dsl(self, search_results_object, permitted_nodegroups, include_provis for key, val in advanced_filter.items(): if key != "op": node = models.Node.objects.get(pk=key) - if self.request.user.has_perm("read_nodegroup", node.nodegroup): + if self.user.has_perm("read_nodegroup", node.nodegroup): datatype = datatype_factory.get_instance(node.datatype) if ("op" in val and (val["op"] == "null" or val["op"] == "not_null")) or ( "val" in val and (val["val"] == "null" or val["val"] == "not_null") ): # don't use a nested query with the null/not null search - datatype.append_search_filters(val, node, null_query, self.request) + datatype.append_search_filters(val, node, null_query, self.parameters) else: - datatype.append_search_filters(val, node, tile_query, self.request) + datatype.append_search_filters(val, node, tile_query, self.parameters) nested_query = Nested(path="tiles", query=tile_query) if advanced_filter["op"] == "or" and index != 0: grouped_query = Bool() @@ -78,7 +78,7 @@ def view_data(self): # only allow cards that the user has permission to read searchable_cards = [] for card in resource_cards: - if self.request.user.has_perm("read_nodegroup", card.nodegroup): + if self.user.has_perm("read_nodegroup", card.nodegroup): searchable_cards.append(card) ret["graphs"] = resource_graphs diff --git a/arches/app/search/components/base.py b/arches/app/search/components/base.py index 39eac7dc79c..04d11a04e61 100644 --- a/arches/app/search/components/base.py +++ b/arches/app/search/components/base.py @@ -18,12 +18,13 @@ class BaseSearchFilter: - def __init__(self, request=None): - self.request = request + def __init__(self, parameters, user): + self.parameters = parameters + self.user = user def append_dsl(self, search_results_object, permitted_nodegroups, include_provisional): """ - used to append ES query dsl to the search request + used to append ES query dsl to the search """ @@ -47,8 +48,9 @@ def post_search_hook(self, search_results_object, results, permitted_nodegroups) class SearchFilterFactory(object): - def __init__(self, request=None): - self.request = request + def __init__(self, parameters, user): + self.parameters = parameters + self.user = user self.search_filters = {search_filter.componentname: search_filter for search_filter in models.SearchComponent.objects.all()} self.search_filters_instances = {} @@ -63,7 +65,7 @@ def get_filter(self, componentname): search_filter.modulename, search_filter.classname, settings.SEARCH_COMPONENT_LOCATIONS ) if class_method: - filter_instance = class_method(self.request) + filter_instance = class_method(self.parameters, self.user) self.search_filters_instances[search_filter.componentname] = filter_instance return filter_instance else: diff --git a/arches/app/search/components/map_filter.py b/arches/app/search/components/map_filter.py index 97b9234ce49..2f383c93113 100644 --- a/arches/app/search/components/map_filter.py +++ b/arches/app/search/components/map_filter.py @@ -26,7 +26,7 @@ class MapFilter(BaseSearchFilter): def append_dsl(self, search_results_object, permitted_nodegroups, include_provisional): search_query = Bool() - querysting_params = self.request.GET.get(details["componentname"], "") + querysting_params = self.parameters.get(details["componentname"], "") spatial_filter = JSONDeserializer().deserialize(querysting_params) if "features" in spatial_filter: if len(spatial_filter["features"]) > 0: @@ -54,7 +54,8 @@ def append_dsl(self, search_results_object, permitted_nodegroups, include_provis spatial_query.filter(geoshape) # get the nodegroup_ids that the user has permission to search - spatial_query.filter(Terms(field="geometries.nodegroup_id", terms=permitted_nodegroups)) + if self.user is not True: + spatial_query.filter(Terms(field="geometries.nodegroup_id", terms=permitted_nodegroups)) if include_provisional is False: spatial_query.filter(Terms(field="geometries.provisional", terms=["false"])) diff --git a/arches/app/search/components/paging_filter.py b/arches/app/search/components/paging_filter.py index 7877e467a5f..d0ca7459858 100644 --- a/arches/app/search/components/paging_filter.py +++ b/arches/app/search/components/paging_filter.py @@ -18,17 +18,17 @@ class PagingFilter(BaseSearchFilter): def append_dsl(self, search_results_object, permitted_nodegroups, include_provisional): - export = self.request.GET.get("export", None) - mobile_download = self.request.GET.get("mobiledownload", None) - page = 1 if self.request.GET.get(details["componentname"]) == "" else int(self.request.GET.get(details["componentname"], 1)) + export = self.parameters.get("export", None) + mobile_download = self.parameters.get("mobiledownload", None) + page = 1 if self.parameters.get(details["componentname"]) == "" else int(self.parameters.get(details["componentname"], 1)) if export is not None: limit = settings.SEARCH_RESULT_LIMIT elif mobile_download is not None: - limit = self.request.GET["resourcecount"] + limit = self.parameters["resourcecount"] else: limit = settings.SEARCH_ITEMS_PER_PAGE - limit = int(self.request.GET.get("limit", limit)) + limit = int(self.parameters.get("limit", limit)) search_results_object["query"].start = limit * int(page - 1) search_results_object["query"].limit = limit @@ -38,9 +38,9 @@ def post_search_hook(self, search_results_object, results, permitted_nodegroups) if results["hits"]["total"]["value"] <= settings.SEARCH_RESULT_LIMIT else settings.SEARCH_RESULT_LIMIT ) - page = 1 if self.request.GET.get(details["componentname"]) == "" else int(self.request.GET.get(details["componentname"], 1)) + page = 1 if self.parameters.get(details["componentname"]) == "" else int(self.parameters.get(details["componentname"], 1)) - paginator, pages = get_paginator(self.request, results, total, page, settings.SEARCH_ITEMS_PER_PAGE) + paginator, pages = get_paginator(self.parameters, results, total, page, settings.SEARCH_ITEMS_PER_PAGE) page = paginator.page(page) ret = {} diff --git a/arches/app/search/components/resource_type_filter.py b/arches/app/search/components/resource_type_filter.py index 7378facfcc4..50735a5591c 100644 --- a/arches/app/search/components/resource_type_filter.py +++ b/arches/app/search/components/resource_type_filter.py @@ -1,7 +1,7 @@ from arches.app.utils.betterJSONSerializer import JSONDeserializer from arches.app.search.elasticsearch_dsl_builder import Bool, Terms from arches.app.search.components.base import BaseSearchFilter -from arches.app.models.models import Node +from arches.app.models.models import Node, GraphModel from arches.app.utils.permission_backend import get_resource_types_by_perm details = { @@ -18,21 +18,24 @@ } -def get_permitted_graphids(permitted_nodegroups): - permitted_graphids = set() - for node in Node.objects.filter(nodegroup__in=permitted_nodegroups): - permitted_graphids.add(str(node.graph_id)) - return permitted_graphids +def get_permitted_graphids(user, permitted_nodegroups): + return get_resource_types_by_perm(user, "read_nodegroup") class ResourceTypeFilter(BaseSearchFilter): def append_dsl(self, search_results_object, permitted_nodegroups, include_provisional): search_query = Bool() - querystring_params = self.request.GET.get(details["componentname"], "") + querystring_params = self.parameters.get(details["componentname"], "") graph_ids = [] - permitted_graphids = get_permitted_graphids(permitted_nodegroups) + resourceTypeFilters = JSONDeserializer().deserialize(querystring_params) + if self.user is True: + permitted_graphids = [ + str(resourceTypeFilter["graphid"]) for resourceTypeFilter in resourceTypeFilters + ] + else: + permitted_graphids = get_permitted_graphids(self.user, permitted_nodegroups) - for resourceTypeFilter in JSONDeserializer().deserialize(querystring_params): + for resourceTypeFilter in resourceTypeFilters: graphid = str(resourceTypeFilter["graphid"]) if resourceTypeFilter["inverted"] is True: try: @@ -53,4 +56,8 @@ def append_dsl(self, search_results_object, permitted_nodegroups, include_provis search_results_object["query"].add_query(search_query) def view_data(self): - return {"resources": get_resource_types_by_perm(self.request.user, "read_nodegroup")} + return {"resources": list( + GraphModel.objects.filter(pk__in=get_resource_types_by_perm(self.user, "read_nodegroup")) + .order_by("name") + .all() + )} diff --git a/arches/app/search/components/search_results.py b/arches/app/search/components/search_results.py index c4958bac28c..a9185d3c85c 100644 --- a/arches/app/search/components/search_results.py +++ b/arches/app/search/components/search_results.py @@ -1,9 +1,9 @@ from arches.app.models import models from arches.app.models.system_settings import settings -from arches.app.search.elasticsearch_dsl_builder import Bool, Terms, NestedAgg, FiltersAgg, GeoHashGridAgg, GeoBoundsAgg +from arches.app.search.elasticsearch_dsl_builder import Bool, Terms, NestedAgg, FiltersAgg, GeoHashGridAgg, GeoBoundsAgg, Nested from arches.app.search.components.base import BaseSearchFilter from arches.app.search.components.resource_type_filter import get_permitted_graphids -from arches.app.utils.permission_backend import user_is_resource_reviewer +from arches.app.utils.permission_backend import user_is_resource_reviewer, get_sets_for_user details = { "searchcomponentid": "", @@ -30,11 +30,12 @@ def append_dsl(self, search_results_object, permitted_nodegroups, include_provis "graph_id" ] # check if resource_type filter is already applied except (KeyError, IndexError): - resource_model_filter = Bool() - permitted_graphids = get_permitted_graphids(permitted_nodegroups) - terms = Terms(field="graph_id", terms=list(permitted_graphids)) - resource_model_filter.filter(terms) - search_results_object["query"].add_query(resource_model_filter) + if self.user is not True: + resource_model_filter = Bool() + permitted_graphids = get_permitted_graphids(self.user, permitted_nodegroups) + terms = Terms(field="graph_id", terms=list(permitted_graphids)) + resource_model_filter.filter(terms) + search_results_object["query"].add_query(resource_model_filter) if include_provisional is True: geo_agg_filter.filter(Terms(field="points.provisional", terms=["false", "true"])) @@ -46,18 +47,32 @@ def append_dsl(self, search_results_object, permitted_nodegroups, include_provis elif include_provisional == "only provisional": geo_agg_filter.filter(Terms(field="points.provisional", terms=["true"])) - geo_agg_filter.filter(Terms(field="points.nodegroup_id", terms=permitted_nodegroups)) + if self.user is not True: + geo_agg_filter.filter(Terms(field="points.nodegroup_id", terms=permitted_nodegroups)) nested_agg_filter.add_filter(geo_agg_filter) nested_agg_filter.add_aggregation(GeoHashGridAgg(field="points.point", name="grid", precision=settings.HEX_BIN_PRECISION)) nested_agg_filter.add_aggregation(GeoBoundsAgg(field="points.point", name="bounds")) nested_agg.add_aggregation(nested_agg_filter) + + # TODO: It would be preferable to inject this, but would require more changes elsewhere. + if self.user is not True: + sets = get_sets_for_user(self.user, "view_resourceinstance") + if sets is not None: # Only None if no filtering should be done, but may be an empty set. + search_query = Bool() + subsearch_query = Bool() + if sets: + subsearch_query.should(Nested(path="sets", query=Terms(field="sets.id", terms=list(sets)))) + if self.user and self.user.id: + subsearch_query.should(Nested(path="permissions", query=Terms(field="permissions.principal_user", terms=[int(self.user.id)]))) + search_query.must(subsearch_query) + search_results_object["query"].add_query(search_query) search_results_object["query"].add_aggregation(nested_agg) def post_search_hook(self, search_results_object, results, permitted_nodegroups): - user_is_reviewer = user_is_resource_reviewer(self.request.user) + user_is_reviewer = user_is_resource_reviewer(self.user) # only reuturn points and geometries a user is allowed to view - geojson_nodes = get_nodegroups_by_datatype_and_perm(self.request, "geojson-feature-collection", "read_nodegroup") + geojson_nodes = get_nodegroups_by_datatype_and_perm(self.user, "geojson-feature-collection", "read_nodegroup") for result in results["hits"]["hits"]: result["_source"]["points"] = select_geoms_for_results(result["_source"]["points"], geojson_nodes, user_is_reviewer) @@ -65,17 +80,17 @@ def post_search_hook(self, search_results_object, results, permitted_nodegroups) try: permitted_tiles = [] for tile in result["_source"]["tiles"]: - if tile["nodegroup_id"] in permitted_nodegroups: + if str(tile["nodegroup_id"]) in permitted_nodegroups or self.user is True: permitted_tiles.append(tile) result["_source"]["tiles"] = permitted_tiles except KeyError: pass -def get_nodegroups_by_datatype_and_perm(request, datatype, permission): +def get_nodegroups_by_datatype_and_perm(user, datatype, permission): nodes = [] for node in models.Node.objects.filter(datatype=datatype): - if request.user.has_perm(permission, node.nodegroup): + if user.has_perm(permission, node.nodegroup): nodes.append(str(node.nodegroup_id)) return nodes diff --git a/arches/app/search/components/sort_results.py b/arches/app/search/components/sort_results.py index 00dd505623b..73a9c29127e 100644 --- a/arches/app/search/components/sort_results.py +++ b/arches/app/search/components/sort_results.py @@ -18,7 +18,7 @@ class SortResults(BaseSearchFilter): def append_dsl(self, search_results_object, permitted_nodegroups, include_provisional): - sort_param = self.request.GET.get(details["componentname"], None) + sort_param = self.parameters.get(details["componentname"], None) if sort_param is not None and sort_param is not "": search_results_object["query"].sort( diff --git a/arches/app/search/components/term_filter.py b/arches/app/search/components/term_filter.py index 1c160968870..0cca35836a3 100644 --- a/arches/app/search/components/term_filter.py +++ b/arches/app/search/components/term_filter.py @@ -22,8 +22,8 @@ class TermFilter(BaseSearchFilter): def append_dsl(self, search_results_object, permitted_nodegroups, include_provisional): search_query = Bool() - querysting_params = self.request.GET.get(details["componentname"], "") - language = self.request.GET.get("language", "*") + querysting_params = self.parameters.get(details["componentname"], "") + language = self.parameters.get("language", "*") for term in JSONDeserializer().deserialize(querysting_params): if term["type"] == "term" or term["type"] == "string": string_filter = Bool() @@ -56,7 +56,8 @@ def append_dsl(self, search_results_object, permitted_nodegroups, include_provis elif include_provisional == "only provisional": string_filter.must_not(Match(field="strings.provisional", query="false", type="phrase")) - string_filter.filter(Terms(field="strings.nodegroup_id", terms=permitted_nodegroups)) + if self.user is not True: + string_filter.filter(Terms(field="strings.nodegroup_id", terms=permitted_nodegroups)) nested_string_filter = Nested(path="strings", query=string_filter) if term["inverted"]: search_query.must_not(nested_string_filter) @@ -68,7 +69,8 @@ def append_dsl(self, search_results_object, permitted_nodegroups, include_provis concept_ids = _get_child_concepts(term["value"]) conceptid_filter = Bool() conceptid_filter.filter(Terms(field="domains.conceptid", terms=concept_ids)) - conceptid_filter.filter(Terms(field="domains.nodegroup_id", terms=permitted_nodegroups)) + if self.user is not True: + conceptid_filter.filter(Terms(field="domains.nodegroup_id", terms=permitted_nodegroups)) if include_provisional is False: conceptid_filter.must_not(Match(field="domains.provisional", query="true", type="phrase")) diff --git a/arches/app/search/components/time_filter.py b/arches/app/search/components/time_filter.py index dd11d1c7ae4..b6280e282fa 100644 --- a/arches/app/search/components/time_filter.py +++ b/arches/app/search/components/time_filter.py @@ -24,7 +24,7 @@ class TimeFilter(BaseSearchFilter): def append_dsl(self, search_results_object, permitted_nodegroups, include_provisional): search_query = Bool() - querysting_params = self.request.GET.get(details["componentname"], "") + querysting_params = self.parameters.get(details["componentname"], "") temporal_filter = JSONDeserializer().deserialize(querysting_params) if "fromDate" in temporal_filter and "toDate" in temporal_filter: # now = str(datetime.utcnow()) @@ -52,7 +52,8 @@ def append_dsl(self, search_results_object, permitted_nodegroups, include_provis date_query = Bool() date_query.filter(inverted_date_query) - date_query.filter(Terms(field="dates.nodegroup_id", terms=permitted_nodegroups)) + if self.user is not True: + date_query.filter(Terms(field="dates.nodegroup_id", terms=permitted_nodegroups)) if include_provisional is False: date_query.filter(Terms(field="dates.provisional", terms=["false"])) @@ -65,7 +66,8 @@ def append_dsl(self, search_results_object, permitted_nodegroups, include_provis else: date_ranges_query = Bool() date_ranges_query.filter(inverted_date_ranges_query) - date_ranges_query.filter(Terms(field="date_ranges.nodegroup_id", terms=permitted_nodegroups)) + if self.user is not True: + date_ranges_query.filter(Terms(field="date_ranges.nodegroup_id", terms=permitted_nodegroups)) if include_provisional is False: date_ranges_query.filter(Terms(field="date_ranges.provisional", terms=["false"])) @@ -79,7 +81,8 @@ def append_dsl(self, search_results_object, permitted_nodegroups, include_provis else: date_query = Bool() date_query.filter(Range(field="dates.date", gte=start_date.lower, lte=end_date.upper)) - date_query.filter(Terms(field="dates.nodegroup_id", terms=permitted_nodegroups)) + if self.user is not True: + date_query.filter(Terms(field="dates.nodegroup_id", terms=permitted_nodegroups)) if include_provisional is False: date_query.filter(Terms(field="dates.provisional", terms=["false"])) @@ -93,7 +96,8 @@ def append_dsl(self, search_results_object, permitted_nodegroups, include_provis date_ranges_query.filter( Range(field="date_ranges.date_range", gte=start_date.lower, lte=end_date.upper, relation="intersects") ) - date_ranges_query.filter(Terms(field="date_ranges.nodegroup_id", terms=permitted_nodegroups)) + if self.user is not True: + date_ranges_query.filter(Terms(field="date_ranges.nodegroup_id", terms=permitted_nodegroups)) if include_provisional is False: date_ranges_query.filter(Terms(field="date_ranges.provisional", terms=["false"])) @@ -114,7 +118,7 @@ def view_data(self): datatype__in=date_datatypes, graph__isresource=True, graph__publication__isnull=False ).prefetch_related("nodegroup") node_graph_dict = { - str(node.nodeid): str(node.graph_id) for node in date_nodes if self.request.user.has_perm("read_nodegroup", node.nodegroup) + str(node.nodeid): str(node.graph_id) for node in date_nodes if (self.user is True or self.user.has_perm("read_nodegroup", node.nodegroup)) } date_cardxnodesxwidgets = models.CardXNodeXWidget.objects.filter(node_id__in=list(node_graph_dict.keys())) diff --git a/arches/app/search/elasticsearch_dsl_builder.py b/arches/app/search/elasticsearch_dsl_builder.py index 32b14612eb0..003d7c31d4e 100644 --- a/arches/app/search/elasticsearch_dsl_builder.py +++ b/arches/app/search/elasticsearch_dsl_builder.py @@ -42,6 +42,27 @@ def dsl(self, value): self._dsl = value +class UpdateByQuery(Dsl): + def __init__(self, se, **kwargs): + self.se = se + self.query = kwargs.pop("query", Query(se)) + self.script = kwargs.pop("script", None) + if self.query is None: + raise UpdateByQueryDSLException(_('You need to specify a "query"')) + if self.script is None: + raise UpdateByQueryDSLException(_('You need to specify a "script"')) + + + def run(self, index="", **kwargs): + self.query.prepare() + self.dsl = { + "query": self.query.dsl["query"], + "script": self.script + } + self.dsl.update(kwargs) + return self.se.update_by_query(index=index, **self.dsl) + + class Query(Dsl): """ http://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html @@ -653,3 +674,6 @@ def __init__(self, **kwargs): class NestedAggDSLException(Exception): pass + +class UpdateByQueryDSLException(Exception): + pass diff --git a/arches/app/search/mappings.py b/arches/app/search/mappings.py index 5a2eb5dd529..e6626019764 100644 --- a/arches/app/search/mappings.py +++ b/arches/app/search/mappings.py @@ -154,6 +154,7 @@ def prepare_search_index(create=False): "users_without_edit_perm": {"type": "integer"}, "users_without_delete_perm": {"type": "integer"}, "users_with_no_access": {"type": "integer"}, + "principal_user": {"type": "integer"}, }, }, "strings": { @@ -172,6 +173,10 @@ def prepare_search_index(create=False): "type": "nested", "properties": {"id": {"type": "keyword"}, "nodegroup_id": {"type": "keyword"}, "provisional": {"type": "boolean"}}, }, + "sets": { + "type": "nested", + "properties": {"id": {"type": "keyword"}}, + }, "domains": { "type": "nested", "properties": { diff --git a/arches/app/search/search.py b/arches/app/search/search.py index 0a7861cf423..cdcc00324aa 100644 --- a/arches/app/search/search.py +++ b/arches/app/search/search.py @@ -26,6 +26,7 @@ from elasticsearch import Elasticsearch, helpers, ElasticsearchWarning from elasticsearch.exceptions import RequestError from elasticsearch.helpers import BulkIndexError +from elasticsearch.client import TasksClient from arches.app.models.system_settings import settings from arches.app.utils.betterJSONSerializer import JSONSerializer, JSONDeserializer @@ -117,6 +118,30 @@ def delete_index(self, **kwargs): print("deleting index : %s" % kwargs.get("index")) return self.es.options(ignore_status=[400, 404]).indices.delete(**kwargs) + def update_by_query(self, **kwargs): + """ + Update items by a search in the index. + Pass an index and id (or list of ids) to get a specific document(s) + Pass a query with a query dsl to perform a search + + """ + + kwargs = self._add_prefix(**kwargs) + query = kwargs.get("query", None) + script = kwargs.get("script", None) + + if query is None or script is None: + message = "%s: WARNING: update-by-query missing query or script" % (datetime.now()) + self.logger.exception(message) + raise RuntimeError(message) + + ret = None + try: + ret = self.es.update_by_query(**kwargs).body + except RequestError as detail: + self.logger.exception("%s: WARNING: update-by-query failed for query: %s \nException detail: %s\n" % (datetime.now(), query, detail.info)) + return ret + def search(self, **kwargs): """ Search for an item in the index. @@ -231,6 +256,9 @@ def refresh(self, **kwargs): kwargs = self._add_prefix(**kwargs) self.es.indices.refresh(**kwargs) + def make_tasks_client(self): + return TasksClient(self.es) + def BulkIndexer(outer_self, batch_size=500, **kwargs): class _BulkIndexer(object): def __init__(self, **kwargs): diff --git a/arches/app/search/time_wheel.py b/arches/app/search/time_wheel.py index cffa98cd427..c92cac7d09f 100644 --- a/arches/app/search/time_wheel.py +++ b/arches/app/search/time_wheel.py @@ -161,7 +161,7 @@ def appendDateRanges(self, results, range_lookup): return results def get_permitted_nodegroups(self, user): - return [str(nodegroup.pk) for nodegroup in get_nodegroups_by_perm(user, "models.read_nodegroup")] + return get_nodegroups_by_perm(user, "models.read_nodegroup") class d3Item(object): diff --git a/arches/app/templates/base-manager.htm b/arches/app/templates/base-manager.htm index 060716895d0..bb5cf3b09fb 100644 --- a/arches/app/templates/base-manager.htm +++ b/arches/app/templates/base-manager.htm @@ -239,89 +239,17 @@

{% endif %} - - - {% endif %} - - {% if show_language_swtich %} - {% get_current_language as LANGUAGE_CODE %} -
- {% endif %} - - - {% if nav.search %} - -
- -
-
Search
-
- {% endif %} - - - {% if nav.notifs %} - -
-
-
-
- -
-
-
- {% endif %} - - - - {% if user_is_reviewer == False and user_can_edit %} - -
- -
-
- {% endif %} - - {% if nav.res_edit and user_can_edit %} - -
- -
-
- {% endif %} - - {% if nav.print %} - -
- -
-
- {% endif %} - - {% if nav.help %} - -
- -
-
- {% endif %} - - - -