Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 39 additions & 30 deletions core/concepts/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from django.core.exceptions import ValidationError
from django.core.validators import RegexValidator
from django.db import models, IntegrityError
from django.db.models import F, Q
from django.db.models import F, Q, Exists, OuterRef, Subquery
from django.db.models.functions import MD5, Upper
from pydash import get, compact, has

Expand All @@ -15,7 +15,7 @@
from core.common.tasks import process_hierarchy_for_new_concept, process_hierarchy_for_concept_version, \
process_hierarchy_for_new_parent_concept_version, update_mappings_concept
from core.common.utils import generate_temp_version, drop_version, \
startswith_temp_version, is_versioned_uri, decode_string
startswith_temp_version, decode_string
from core.concepts.constants import CONCEPT_TYPE, LOCALES_FULLY_SPECIFIED, LOCALES_SHORT, LOCALES_SEARCH_INDEX_TERM, \
CONCEPT_WAS_RETIRED, CONCEPT_IS_ALREADY_RETIRED, CONCEPT_IS_ALREADY_NOT_RETIRED, CONCEPT_WAS_UNRETIRED, \
ALREADY_EXISTS, CONCEPT_REGEX, MAX_LOCALES_LIMIT, \
Expand Down Expand Up @@ -1391,35 +1391,55 @@ def child_concept_urls(self):
return self.get_hierarchy_concept_urls('child_concepts')

def get_hierarchy_concept_urls(self, relation, versioned=False):
queryset = get(self, relation).all()
return list(self.get_hierarchy_related_queryset(relation, versioned).values_list('uri', flat=True))

def get_hierarchy_queryset(self, relation, repo_version, filters=None):
filters = filters or {}
return self.get_hierarchy_related_queryset(relation, versioned=not repo_version.is_head).filter(**filters)

def get_hierarchy_ids(self):
"""This concept's rows whose hierarchy links count as its own: itself, its HEAD and its latest version."""
ids = {self.id}
if self.is_latest_version:
queryset |= get(self.versioned_object, relation).all()
ids.add(self.versioned_object_id)
if self.is_versioned_object:
latest_version = self.get_latest_version()
if latest_version:
queryset |= get(latest_version, relation).all()
uris = queryset.values_list('uri', flat=True)
ids.add(latest_version.id)
return ids

def get_hierarchy_related_queryset(self, relation, versioned=False):
"""The concepts linked through relation, in one query: their HEADs, or with versioned the linked versions."""
own, other = {'child_concepts': ('parent', 'child'), 'parent_concepts': ('child', 'parent')}[relation]
links = HierarchicalConcepts.objects.filter(**{f'{own}_id__in': self.get_hierarchy_ids()})
if versioned:
return self.__format_hierarchy_versioned_uris(uris)
return self.__format_hierarchy_uris(uris)
return Concept.objects.filter(id__in=links.values(f'{other}_id')).exclude(id=F('versioned_object_id'))
return Concept.objects.filter(id__in=links.values(f'{other}__versioned_object_id'))

def get_hierarchy_queryset(self, relation, repo_version, filters=None):
filters = filters or {}
queryset = Concept.objects.filter(uri__in=self.get_hierarchy_concept_urls(relation, not repo_version.is_head))
return queryset.filter(**filters)
@staticmethod
def annotate_has_children(queryset):
"""has_children for each row as part of the list query, by the same rules as the has_children property."""
own = HierarchicalConcepts.objects.filter(parent_id=OuterRef('id'))
of_head = HierarchicalConcepts.objects.filter(parent_id=OuterRef('versioned_object_id'))
newest_latest_version = Concept.objects.filter(
versioned_object_id=OuterRef(OuterRef('id')), is_latest_version=True, is_active=True
).order_by('-created_at').values('id')[:1]
of_latest = HierarchicalConcepts.objects.filter(parent_id=Subquery(newest_latest_version))
return queryset.annotate(
has_children_annotated=Exists(own) | (Q(is_latest_version=True) & Exists(of_head)) |
(Q(id=F('versioned_object_id')) & Exists(of_latest)))

def child_concept_queryset(self):
urls = self.child_concept_urls
if urls:
return Concept.objects.filter(uri__in=urls)
return Concept.objects.none()
return self.get_hierarchy_related_queryset('child_concepts')

@property
def children_concepts_count(self):
return len(self.child_concept_urls)
return self.child_concept_queryset().count()

@property
def has_children(self):
if 'has_children_annotated' in self.__dict__:
return self.has_children_annotated
result = self.child_concepts.exists()

if not result and self.is_latest_version:
Expand All @@ -1431,21 +1451,10 @@ def has_children(self):

@property
def parent_concepts_count(self):
return len(self.parent_concept_urls)
return self.parent_concept_queryset().count()

def parent_concept_queryset(self):
urls = self.parent_concept_urls
if urls:
return Concept.objects.filter(uri__in=urls)
return Concept.objects.none()

@staticmethod
def __format_hierarchy_uris(uris):
return list({drop_version(uri) for uri in uris})

@staticmethod
def __format_hierarchy_versioned_uris(uris):
return list({uri for uri in uris if is_versioned_uri(uri)})
return self.get_hierarchy_related_queryset('parent_concepts')

def get_hierarchy_path(self):
result = []
Expand Down
105 changes: 105 additions & 0 deletions core/concepts/tests/tests.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import json
from datetime import timedelta
import threading
from unittest.mock import ANY, Mock, patch

Expand Down Expand Up @@ -2954,6 +2955,110 @@ def test_has_children_via_versioned_object(self):

self.assertTrue(latest_parent.versioned_object.has_children)

def test_hierarchy_querysets_via_versions_match_urls(self):
source = OrganizationSourceFactory()
parent_concept = ConceptFactory(parent=source)
child_concept = ConceptFactory(parent=source)
latest_parent = parent_concept.get_latest_version()
child_concept.get_latest_version().parent_concepts.add(latest_parent)

for concept in (parent_concept, latest_parent):
self.assertEqual(
list(concept.child_concept_queryset().values_list('uri', flat=True)), concept.child_concept_urls)
self.assertEqual(concept.child_concept_urls, [child_concept.uri])
for concept in (child_concept, child_concept.get_latest_version()):
self.assertEqual(
list(concept.parent_concept_queryset().values_list('uri', flat=True)), [parent_concept.uri])

def test_get_hierarchy_queryset_versioned_matches_urls(self):
source = OrganizationSourceFactory()
parent_concept = ConceptFactory(parent=source)
child_concept = ConceptFactory(parent=source)
latest_child = child_concept.get_latest_version()
latest_child.parent_concepts.add(parent_concept.get_latest_version())
child_concept.parent_concepts.add(parent_concept)
version = OrganizationSourceFactory(
version='v1', mnemonic=source.mnemonic, organization=source.organization)

self.assertEqual(
list(parent_concept.get_hierarchy_queryset('child_concepts', version).values_list('uri', flat=True)),
[latest_child.uri])
self.assertEqual(
list(parent_concept.get_hierarchy_queryset('child_concepts', source).values_list('uri', flat=True)),
[child_concept.uri])
self.assertEqual(
parent_concept.get_hierarchy_concept_urls('child_concepts', versioned=True), [latest_child.uri])

def test_get_hierarchy_queryset_for_a_collection_version(self):
source = OrganizationSourceFactory()
parent_concept = ConceptFactory(parent=source)
latest_child = ConceptFactory(parent=source).get_latest_version()
latest_child.parent_concepts.add(parent_concept.get_latest_version())
collection_version = OrganizationCollectionFactory(version='v1')
expansion = ExpansionFactory(collection_version=collection_version)
filters = {'expansion_set__collection_version': collection_version}

self.assertEqual(
parent_concept.get_hierarchy_queryset('child_concepts', collection_version, filters).count(), 0)

expansion.concepts.add(latest_child)

self.assertEqual(
list(parent_concept.get_hierarchy_queryset(
'child_concepts', collection_version, filters).values_list('uri', flat=True)),
[latest_child.uri])

def test_get_hierarchy_related_queryset_rejects_an_unknown_relation(self):
with self.assertRaises(KeyError):
ConceptFactory().get_hierarchy_related_queryset('children_concepts')

def test_hierarchy_counts_match_urls(self):
parent_concept = ConceptFactory()
child_concept = ConceptFactory()
child_concept.parent_concepts.add(parent_concept)

self.assertEqual(parent_concept.children_concepts_count, len(parent_concept.child_concept_urls))
self.assertEqual(child_concept.parent_concepts_count, len(child_concept.parent_concept_urls))
self.assertEqual(parent_concept.children_concepts_count, 1)

def test_annotate_has_children_uses_the_newest_latest_version(self):
source = OrganizationSourceFactory()
parent_concept = ConceptFactory(parent=source)
older_latest = parent_concept.get_latest_version()
newer_latest = ConceptFactory(
parent=source, mnemonic=parent_concept.mnemonic, version='newer',
versioned_object=parent_concept, is_latest_version=True)
Concept.objects.filter(id=older_latest.id).update(created_at=newer_latest.created_at - timedelta(days=1))
child_concept = ConceptFactory(parent=source)
child_concept.parent_concepts.add(older_latest)

def annotated():
return Concept.annotate_has_children(Concept.objects.filter(id=parent_concept.id)).get()

self.assertFalse(Concept.objects.get(id=parent_concept.id).has_children)
self.assertFalse(annotated().has_children)

child_concept.parent_concepts.add(newer_latest)

self.assertTrue(Concept.objects.get(id=parent_concept.id).has_children)
self.assertTrue(annotated().has_children)

def test_annotate_has_children_matches_property(self):
source = OrganizationSourceFactory()
own_parent = ConceptFactory(parent=source)
ConceptFactory(parent=source).parent_concepts.add(own_parent)
version_parent = ConceptFactory(parent=source)
ConceptFactory(parent=source).get_latest_version().parent_concepts.add(version_parent.get_latest_version())
leaf = ConceptFactory(parent=source)

concepts = Concept.annotate_has_children(Concept.objects.filter(parent=source))

self.assertTrue(concepts.filter(has_children_annotated=True).exists())
for concept in concepts:
self.assertEqual(concept.has_children, Concept.objects.get(id=concept.id).has_children, concept.uri)
self.assertFalse(concepts.get(id=leaf.id).has_children)
self.assertTrue(concepts.get(id=version_parent.id).has_children)

def test_cascade_returns_self_without_repo_version(self):
concept = ConceptFactory()
result = concept.cascade(repo_version=None)
Expand Down
6 changes: 5 additions & 1 deletion core/concepts/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -606,7 +606,7 @@ class ConceptChildrenView(ConceptBaseView, ListWithHeadersMixin):
is_page_size_capped = True
is_searchable = False # a search would list the repo's concepts, not this concept's hierarchy
serializer_class = ConceptChildrenSerializer
default_qs_sort_attr = 'mnemonic'
default_qs_sort_attr = ['mnemonic', 'id'] # id breaks ties, so pages never repeat or skip a child

def get_queryset(self):
queryset = super().get_queryset()
Expand All @@ -619,6 +619,9 @@ def get_queryset(self):
self.check_object_permissions(self.request, instance)
return get_viewable_concepts(instance.child_concept_queryset(), self.request)

def filter_queryset(self, queryset=None): # annotated here, not in get_queryset, to keep it out of CSV exports
return Concept.annotate_has_children(super().filter_queryset(queryset))

def get(self, request, *args, **kwargs):
return self.list(request, *args, **kwargs)

Expand All @@ -627,6 +630,7 @@ class ConceptParentsView(ConceptBaseView, ListWithHeadersMixin):
is_page_size_capped = True
is_searchable = False # a search would list the repo's concepts, not this concept's hierarchy
serializer_class = ConceptParentsSerializer
default_qs_sort_attr = ['-updated_at', 'id'] # id breaks ties, so pages never repeat or skip a parent

def get_queryset(self):
queryset = super().get_queryset()
Expand Down
78 changes: 78 additions & 0 deletions core/integration_tests/tests_concepts.py
Original file line number Diff line number Diff line change
Expand Up @@ -4114,6 +4114,23 @@ def test_get_200(self):
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.data), 0)

def test_get_200_paged(self):
child_concept = ConceptFactory()
parents = [ConceptFactory() for _ in range(3)]
child_concept.parent_concepts.set(parents)

first_page = self.client.get(child_concept.url + 'parents/?limit=2')
second_page = self.client.get(child_concept.url + 'parents/?limit=2&page=2')

self.assertEqual(first_page.status_code, 200)
self.assertEqual(first_page['num_found'], '3')
self.assertEqual(len(first_page.data), 2)
self.assertEqual(len(second_page.data), 1)
self.assertEqual(
sorted(data['url'] for data in [*first_page.data, *second_page.data]),
sorted(parent.uri for parent in parents)
)


class ConceptChildrenViewTest(OCLAPITestCase):
def test_get_200(self):
Expand All @@ -4137,6 +4154,67 @@ def test_get_200(self):
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.data), 0)

def test_get_200_paged_with_has_children(self):
source = OrganizationSourceFactory()
parent_concept = ConceptFactory(parent=source)
children = [ConceptFactory(parent=source, mnemonic=f'c{i}') for i in range(3)]
for child in children:
child.parent_concepts.set([parent_concept])
ConceptFactory(parent=source).parent_concepts.set([children[0]])

response = self.client.get(parent_concept.url + 'children/?limit=2')

self.assertEqual(response.status_code, 200)
self.assertEqual(response['num_found'], '3')
self.assertEqual(response['num_returned'], '2')
self.assertIn('page=2', response['next'])
self.assertEqual([data['id'] for data in response.data], ['c0', 'c1'])
self.assertEqual([data['has_children'] for data in response.data], [True, False])

response = self.client.get(parent_concept.url + 'children/?limit=2&page=2')

self.assertEqual(response.status_code, 200)
self.assertEqual(response['num_returned'], '1')
self.assertEqual([data['id'] for data in response.data], ['c2'])
self.assertEqual([data['has_children'] for data in response.data], [False])

def test_get_200_pages_are_stable_for_the_same_mnemonic(self):
parent_concept = ConceptFactory()
children = [ConceptFactory(mnemonic='same') for _ in range(3)] # each in its own source
for child in children:
child.parent_concepts.set([parent_concept])

with CaptureQueriesContext(connection) as queries:
pages = [self.client.get(f'{parent_concept.url}children/?limit=1&page={page}') for page in (1, 2, 3)]

self.assertTrue(
any('ORDER BY "concepts"."mnemonic" ASC, "concepts"."id" ASC' in query['sql'] for query in queries))
self.assertEqual(
[page.data[0]['uuid'] for page in pages], [str(child.id) for child in sorted(children, key=lambda c: c.id)])

@patch('core.common.mixins.write_csv_to_s3', return_value='https://csv-url')
def test_csv_has_no_has_children_column(self, write_csv_mock):
parent_concept = ConceptFactory()
ConceptFactory().parent_concepts.set([parent_concept])

self.client.get(parent_concept.url + 'children/?csv=true')

write_csv_mock.assert_called_once()
rows = list(write_csv_mock.call_args.args[0])
self.assertEqual(len(rows), 1)
self.assertNotIn('has_children_annotated', rows[0])

def test_get_200_includes_retired_children(self):
parent_concept = ConceptFactory()
retired_child = ConceptFactory(retired=True)
retired_child.parent_concepts.set([parent_concept])

response = self.client.get(parent_concept.url + 'children/')

self.assertEqual(response.status_code, 200)
self.assertEqual([data['url'] for data in response.data], [retired_child.uri])
self.assertTrue(response.data[0]['retired'])


class ConceptCollectionMembershipViewTest(OCLAPITestCase):
def test_get_200(self):
Expand Down
Loading