Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Import from other channels search optimized #3399

Merged
merged 32 commits into from
Oct 7, 2022
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
357204c
Optimized import from other channels search
vkWeb Jun 2, 2022
9028b22
Merge branch 'unstable' into optimize/search
vkWeb Jun 2, 2022
d24416c
Add search test for channel filtering and location_ids handling
bjester Jun 15, 2022
676526e
Fix autodiscovery of search tests
vkWeb Jun 30, 2022
6d40c55
Remove location_ids, zero db queries on descendant resource count
vkWeb Jul 9, 2022
094f05f
Upgrade django debug toolbar and fix its settings
vkWeb Jul 12, 2022
701ddc9
Remove unnecessary dev settings
vkWeb Jul 22, 2022
1a14749
Add .envrc to .gitignore
vkWeb Jul 22, 2022
bff595c
Add vector search column & indexes, also GiST trigram index
vkWeb Aug 12, 2022
8b7152e
Merge branch 'unstable' into optimize/search
vkWeb Aug 12, 2022
c0e55ee
Remove cyclic migration conflicts
vkWeb Aug 12, 2022
34e8436
Fix wrong indentation happened due to merge conflict
vkWeb Aug 12, 2022
e00c512
Add a command for setting tsvectors and fix tests
vkWeb Aug 14, 2022
f3280d9
Remove grade_level default to pass failing tests
vkWeb Aug 14, 2022
475aeef
Merge branch 'unstable' into optimize/search
vkWeb Aug 24, 2022
3ff0edd
Full text search models and data migrations
vkWeb Sep 8, 2022
fffd912
Merge branch 'unstable' into optimize/search
vkWeb Sep 8, 2022
974b69e
Resolve conflicts and remove old index refs
vkWeb Sep 8, 2022
e15b015
feat: full text search!
vkWeb Sep 10, 2022
aa62dc2
Sync tsvectors on publish!
vkWeb Sep 14, 2022
2672512
fix: tests and ready for merge! <3
vkWeb Sep 14, 2022
1ffa30d
fix: node command edge case; when published nodes go to trash tree, t…
vkWeb Sep 14, 2022
55e4acf
Merge branch 'unstable' into optimize/search
vkWeb Sep 16, 2022
57724e0
Enforce only-one search entries
vkWeb Sep 16, 2022
e53e56a
Remove unnecessary select_related
vkWeb Sep 16, 2022
4b3d4c7
fix cache tests mock by setting ContentNodeFullTextSearch
vkWeb Sep 16, 2022
44ab74c
fix cache & nodes tests by using db-friendly TestCase
vkWeb Sep 16, 2022
4beaf3c
Merge branch 'unstable' into optimize/search
vkWeb Sep 16, 2022
001e788
Use command for tsv insertion & simpler tsv update on publish
vkWeb Sep 21, 2022
ff59495
Merge branch 'unstable' into optimize/search
vkWeb Sep 27, 2022
9ec60cf
fixes the strict update subquery, lightens it up
vkWeb Sep 28, 2022
aae3be1
Do not output deleted channel nodes on search
vkWeb Sep 28, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions contentcuration/contentcuration/dev_settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,3 +6,14 @@
ROOT_URLCONF = "contentcuration.dev_urls"

INSTALLED_APPS += ("drf_yasg",)

REST_FRAMEWORK = {
'DEFAULT_PERMISSION_CLASSES': (
'rest_framework.permissions.IsAuthenticated',
),
'DEFAULT_AUTHENTICATION_CLASSES': (
'rest_framework.authentication.SessionAuthentication',
'rest_framework.authentication.BasicAuthentication',
'rest_framework.authentication.TokenAuthentication',
)
}
vkWeb marked this conversation as resolved.
Show resolved Hide resolved
18 changes: 10 additions & 8 deletions contentcuration/contentcuration/tests/testdata.py
Original file line number Diff line number Diff line change
Expand Up @@ -195,21 +195,23 @@ def node(data, parent=None):
return new_node


def tree(parent=None):
def tree(parent=None, tree_data=None):
# Read from json fixture
filepath = os.path.sep.join([os.path.dirname(__file__), "fixtures", "tree.json"])
with open(filepath, "rb") as jsonfile:
data = json.load(jsonfile)
if tree_data is None:
filepath = os.path.sep.join([os.path.dirname(__file__), "fixtures", "tree.json"])
with open(filepath, "rb") as jsonfile:
tree_data = json.load(jsonfile)

return node(data, parent)
return node(tree_data, parent)


def channel(name="testchannel"):
def channel(name="testchannel", create_main_tree=True, main_tree_data=None):
channel = cc.Channel.objects.create(name=name)
channel.save()

channel.main_tree = tree()
channel.save()
if create_main_tree:
channel.main_tree = tree(tree_data=main_tree_data)
channel.save()

return channel

Expand Down
3 changes: 2 additions & 1 deletion contentcuration/contentcuration/viewsets/contentnode.py
Original file line number Diff line number Diff line change
Expand Up @@ -610,7 +610,8 @@ def delete_from_changes(self, changes):


def dict_if_none(obj, field_name=None):
return obj[field_name] if obj[field_name] else {}
vkWeb marked this conversation as resolved.
Show resolved Hide resolved
value = obj.get(field_name)
return value if value else {}


# Apply mixin first to override ValuesViewset
Expand Down
55 changes: 55 additions & 0 deletions contentcuration/search/tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,3 +29,58 @@ def test_filter_channels_by_edit(self):
)
self.assertEqual(response.status_code, 200, response.content)
self.assertNotEqual(response.data["results"], [])

def test_search_result(self):
# Create two users
user_a = testdata.user(email="[email protected]")
user_b = testdata.user(email="[email protected]")

# Create two channels with two editors
test_tree_data = {
"node_id": "00000000000000000000000000000000",
"title": "Root topic node",
"kind_id": "topic",
"children": [
{
"node_id": "00000000000000000000000000000001",
"title": "Kolibri video",
"kind_id": "video",
},
]
}

channel_a = testdata.channel(name="user_a_channel", main_tree_data=test_tree_data)
channel_a.editors.add(user_a)

channel_b = testdata.channel(name="user_b_channel", create_main_tree=False)
channel_b.editors.add(user_b)

# Publish channel_a
channel_a.main_tree.publishing = False
channel_a.main_tree.changed = False
channel_a.main_tree.published = True
channel_a.main_tree.save()
channel_a.public = True
channel_a.save()

# Import resources from channel_a to channel_b
channel_a.main_tree.copy_to(channel_b.main_tree, batch_size=1)
channel_b.main_tree.refresh_from_db()

# Send request from user_b to the search endpoint
self.client.force_authenticate(user=user_b)
response = self.client.get(
reverse("search-list"),
data={
"channel_list": "public",
"keywords": "video"
},
format="json",
)

# Assert whether the location_ids are of accessible nodes or not
kolibri_video_node = channel_b.main_tree.get_descendants().filter(title="Kolibri video").first()

# The ids in location_ids should be of channel_b's ContentNode only
self.assertEqual(len(response.data["results"][0]["location_ids"]), 1)
self.assertEqual(response.data["results"][0]["location_ids"][0], kolibri_video_node.cloned_source_id)
2 changes: 1 addition & 1 deletion contentcuration/search/viewsets/contentnode.py
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ def annotate_queryset(self, queryset):
2. Annotate lists of content node and channel pks
"""
# Get accessible content nodes that match the content id
content_id_query = ContentNode.filter_view_queryset(ContentNode.objects.all(), self.request.user).filter(
content_id_query = queryset.filter(
vkWeb marked this conversation as resolved.
Show resolved Hide resolved
content_id=OuterRef("content_id")
)

Expand Down