diff --git a/core/concepts/models.py b/core/concepts/models.py index 991558f7d..cd7deffdd 100644 --- a/core/concepts/models.py +++ b/core/concepts/models.py @@ -3,7 +3,7 @@ from django.core.exceptions import ValidationError from django.core.validators import RegexValidator from django.db import models, IntegrityError -from django.db.models import F, Q +from django.db.models import F, Q, Exists, OuterRef, Subquery from django.db.models.functions import MD5, Upper from pydash import get, compact, has @@ -15,7 +15,7 @@ from core.common.tasks import process_hierarchy_for_new_concept, process_hierarchy_for_concept_version, \ process_hierarchy_for_new_parent_concept_version, update_mappings_concept from core.common.utils import generate_temp_version, drop_version, \ - startswith_temp_version, is_versioned_uri, decode_string + startswith_temp_version, decode_string from core.concepts.constants import CONCEPT_TYPE, LOCALES_FULLY_SPECIFIED, LOCALES_SHORT, LOCALES_SEARCH_INDEX_TERM, \ CONCEPT_WAS_RETIRED, CONCEPT_IS_ALREADY_RETIRED, CONCEPT_IS_ALREADY_NOT_RETIRED, CONCEPT_WAS_UNRETIRED, \ ALREADY_EXISTS, CONCEPT_REGEX, MAX_LOCALES_LIMIT, \ @@ -1391,35 +1391,55 @@ def child_concept_urls(self): return self.get_hierarchy_concept_urls('child_concepts') def get_hierarchy_concept_urls(self, relation, versioned=False): - queryset = get(self, relation).all() + return list(self.get_hierarchy_related_queryset(relation, versioned).values_list('uri', flat=True)) + + def get_hierarchy_queryset(self, relation, repo_version, filters=None): + filters = filters or {} + return self.get_hierarchy_related_queryset(relation, versioned=not repo_version.is_head).filter(**filters) + + def get_hierarchy_ids(self): + """This concept's rows whose hierarchy links count as its own: itself, its HEAD and its latest version.""" + ids = {self.id} if self.is_latest_version: - queryset |= get(self.versioned_object, relation).all() + ids.add(self.versioned_object_id) if self.is_versioned_object: latest_version = self.get_latest_version() if latest_version: - queryset |= get(latest_version, relation).all() - uris = queryset.values_list('uri', flat=True) + ids.add(latest_version.id) + return ids + + def get_hierarchy_related_queryset(self, relation, versioned=False): + """The concepts linked through relation, in one query: their HEADs, or with versioned the linked versions.""" + own, other = {'child_concepts': ('parent', 'child'), 'parent_concepts': ('child', 'parent')}[relation] + links = HierarchicalConcepts.objects.filter(**{f'{own}_id__in': self.get_hierarchy_ids()}) if versioned: - return self.__format_hierarchy_versioned_uris(uris) - return self.__format_hierarchy_uris(uris) + return Concept.objects.filter(id__in=links.values(f'{other}_id')).exclude(id=F('versioned_object_id')) + return Concept.objects.filter(id__in=links.values(f'{other}__versioned_object_id')) - def get_hierarchy_queryset(self, relation, repo_version, filters=None): - filters = filters or {} - queryset = Concept.objects.filter(uri__in=self.get_hierarchy_concept_urls(relation, not repo_version.is_head)) - return queryset.filter(**filters) + @staticmethod + def annotate_has_children(queryset): + """has_children for each row as part of the list query, by the same rules as the has_children property.""" + own = HierarchicalConcepts.objects.filter(parent_id=OuterRef('id')) + of_head = HierarchicalConcepts.objects.filter(parent_id=OuterRef('versioned_object_id')) + newest_latest_version = Concept.objects.filter( + versioned_object_id=OuterRef(OuterRef('id')), is_latest_version=True, is_active=True + ).order_by('-created_at').values('id')[:1] + of_latest = HierarchicalConcepts.objects.filter(parent_id=Subquery(newest_latest_version)) + return queryset.annotate( + has_children_annotated=Exists(own) | (Q(is_latest_version=True) & Exists(of_head)) | + (Q(id=F('versioned_object_id')) & Exists(of_latest))) def child_concept_queryset(self): - urls = self.child_concept_urls - if urls: - return Concept.objects.filter(uri__in=urls) - return Concept.objects.none() + return self.get_hierarchy_related_queryset('child_concepts') @property def children_concepts_count(self): - return len(self.child_concept_urls) + return self.child_concept_queryset().count() @property def has_children(self): + if 'has_children_annotated' in self.__dict__: + return self.has_children_annotated result = self.child_concepts.exists() if not result and self.is_latest_version: @@ -1431,21 +1451,10 @@ def has_children(self): @property def parent_concepts_count(self): - return len(self.parent_concept_urls) + return self.parent_concept_queryset().count() def parent_concept_queryset(self): - urls = self.parent_concept_urls - if urls: - return Concept.objects.filter(uri__in=urls) - return Concept.objects.none() - - @staticmethod - def __format_hierarchy_uris(uris): - return list({drop_version(uri) for uri in uris}) - - @staticmethod - def __format_hierarchy_versioned_uris(uris): - return list({uri for uri in uris if is_versioned_uri(uri)}) + return self.get_hierarchy_related_queryset('parent_concepts') def get_hierarchy_path(self): result = [] diff --git a/core/concepts/tests/tests.py b/core/concepts/tests/tests.py index e41947d16..9300cb802 100644 --- a/core/concepts/tests/tests.py +++ b/core/concepts/tests/tests.py @@ -1,4 +1,5 @@ import json +from datetime import timedelta import threading from unittest.mock import ANY, Mock, patch @@ -2954,6 +2955,110 @@ def test_has_children_via_versioned_object(self): self.assertTrue(latest_parent.versioned_object.has_children) + def test_hierarchy_querysets_via_versions_match_urls(self): + source = OrganizationSourceFactory() + parent_concept = ConceptFactory(parent=source) + child_concept = ConceptFactory(parent=source) + latest_parent = parent_concept.get_latest_version() + child_concept.get_latest_version().parent_concepts.add(latest_parent) + + for concept in (parent_concept, latest_parent): + self.assertEqual( + list(concept.child_concept_queryset().values_list('uri', flat=True)), concept.child_concept_urls) + self.assertEqual(concept.child_concept_urls, [child_concept.uri]) + for concept in (child_concept, child_concept.get_latest_version()): + self.assertEqual( + list(concept.parent_concept_queryset().values_list('uri', flat=True)), [parent_concept.uri]) + + def test_get_hierarchy_queryset_versioned_matches_urls(self): + source = OrganizationSourceFactory() + parent_concept = ConceptFactory(parent=source) + child_concept = ConceptFactory(parent=source) + latest_child = child_concept.get_latest_version() + latest_child.parent_concepts.add(parent_concept.get_latest_version()) + child_concept.parent_concepts.add(parent_concept) + version = OrganizationSourceFactory( + version='v1', mnemonic=source.mnemonic, organization=source.organization) + + self.assertEqual( + list(parent_concept.get_hierarchy_queryset('child_concepts', version).values_list('uri', flat=True)), + [latest_child.uri]) + self.assertEqual( + list(parent_concept.get_hierarchy_queryset('child_concepts', source).values_list('uri', flat=True)), + [child_concept.uri]) + self.assertEqual( + parent_concept.get_hierarchy_concept_urls('child_concepts', versioned=True), [latest_child.uri]) + + def test_get_hierarchy_queryset_for_a_collection_version(self): + source = OrganizationSourceFactory() + parent_concept = ConceptFactory(parent=source) + latest_child = ConceptFactory(parent=source).get_latest_version() + latest_child.parent_concepts.add(parent_concept.get_latest_version()) + collection_version = OrganizationCollectionFactory(version='v1') + expansion = ExpansionFactory(collection_version=collection_version) + filters = {'expansion_set__collection_version': collection_version} + + self.assertEqual( + parent_concept.get_hierarchy_queryset('child_concepts', collection_version, filters).count(), 0) + + expansion.concepts.add(latest_child) + + self.assertEqual( + list(parent_concept.get_hierarchy_queryset( + 'child_concepts', collection_version, filters).values_list('uri', flat=True)), + [latest_child.uri]) + + def test_get_hierarchy_related_queryset_rejects_an_unknown_relation(self): + with self.assertRaises(KeyError): + ConceptFactory().get_hierarchy_related_queryset('children_concepts') + + def test_hierarchy_counts_match_urls(self): + parent_concept = ConceptFactory() + child_concept = ConceptFactory() + child_concept.parent_concepts.add(parent_concept) + + self.assertEqual(parent_concept.children_concepts_count, len(parent_concept.child_concept_urls)) + self.assertEqual(child_concept.parent_concepts_count, len(child_concept.parent_concept_urls)) + self.assertEqual(parent_concept.children_concepts_count, 1) + + def test_annotate_has_children_uses_the_newest_latest_version(self): + source = OrganizationSourceFactory() + parent_concept = ConceptFactory(parent=source) + older_latest = parent_concept.get_latest_version() + newer_latest = ConceptFactory( + parent=source, mnemonic=parent_concept.mnemonic, version='newer', + versioned_object=parent_concept, is_latest_version=True) + Concept.objects.filter(id=older_latest.id).update(created_at=newer_latest.created_at - timedelta(days=1)) + child_concept = ConceptFactory(parent=source) + child_concept.parent_concepts.add(older_latest) + + def annotated(): + return Concept.annotate_has_children(Concept.objects.filter(id=parent_concept.id)).get() + + self.assertFalse(Concept.objects.get(id=parent_concept.id).has_children) + self.assertFalse(annotated().has_children) + + child_concept.parent_concepts.add(newer_latest) + + self.assertTrue(Concept.objects.get(id=parent_concept.id).has_children) + self.assertTrue(annotated().has_children) + + def test_annotate_has_children_matches_property(self): + source = OrganizationSourceFactory() + own_parent = ConceptFactory(parent=source) + ConceptFactory(parent=source).parent_concepts.add(own_parent) + version_parent = ConceptFactory(parent=source) + ConceptFactory(parent=source).get_latest_version().parent_concepts.add(version_parent.get_latest_version()) + leaf = ConceptFactory(parent=source) + + concepts = Concept.annotate_has_children(Concept.objects.filter(parent=source)) + + self.assertTrue(concepts.filter(has_children_annotated=True).exists()) + for concept in concepts: + self.assertEqual(concept.has_children, Concept.objects.get(id=concept.id).has_children, concept.uri) + self.assertFalse(concepts.get(id=leaf.id).has_children) + self.assertTrue(concepts.get(id=version_parent.id).has_children) + def test_cascade_returns_self_without_repo_version(self): concept = ConceptFactory() result = concept.cascade(repo_version=None) diff --git a/core/concepts/views.py b/core/concepts/views.py index a0c472ef2..152736492 100644 --- a/core/concepts/views.py +++ b/core/concepts/views.py @@ -606,7 +606,7 @@ class ConceptChildrenView(ConceptBaseView, ListWithHeadersMixin): is_page_size_capped = True is_searchable = False # a search would list the repo's concepts, not this concept's hierarchy serializer_class = ConceptChildrenSerializer - default_qs_sort_attr = 'mnemonic' + default_qs_sort_attr = ['mnemonic', 'id'] # id breaks ties, so pages never repeat or skip a child def get_queryset(self): queryset = super().get_queryset() @@ -619,6 +619,9 @@ def get_queryset(self): self.check_object_permissions(self.request, instance) return get_viewable_concepts(instance.child_concept_queryset(), self.request) + def filter_queryset(self, queryset=None): # annotated here, not in get_queryset, to keep it out of CSV exports + return Concept.annotate_has_children(super().filter_queryset(queryset)) + def get(self, request, *args, **kwargs): return self.list(request, *args, **kwargs) @@ -627,6 +630,7 @@ class ConceptParentsView(ConceptBaseView, ListWithHeadersMixin): is_page_size_capped = True is_searchable = False # a search would list the repo's concepts, not this concept's hierarchy serializer_class = ConceptParentsSerializer + default_qs_sort_attr = ['-updated_at', 'id'] # id breaks ties, so pages never repeat or skip a parent def get_queryset(self): queryset = super().get_queryset() diff --git a/core/integration_tests/tests_concepts.py b/core/integration_tests/tests_concepts.py index a57045132..da9a38241 100644 --- a/core/integration_tests/tests_concepts.py +++ b/core/integration_tests/tests_concepts.py @@ -4114,6 +4114,23 @@ def test_get_200(self): self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 0) + def test_get_200_paged(self): + child_concept = ConceptFactory() + parents = [ConceptFactory() for _ in range(3)] + child_concept.parent_concepts.set(parents) + + first_page = self.client.get(child_concept.url + 'parents/?limit=2') + second_page = self.client.get(child_concept.url + 'parents/?limit=2&page=2') + + self.assertEqual(first_page.status_code, 200) + self.assertEqual(first_page['num_found'], '3') + self.assertEqual(len(first_page.data), 2) + self.assertEqual(len(second_page.data), 1) + self.assertEqual( + sorted(data['url'] for data in [*first_page.data, *second_page.data]), + sorted(parent.uri for parent in parents) + ) + class ConceptChildrenViewTest(OCLAPITestCase): def test_get_200(self): @@ -4137,6 +4154,67 @@ def test_get_200(self): self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 0) + def test_get_200_paged_with_has_children(self): + source = OrganizationSourceFactory() + parent_concept = ConceptFactory(parent=source) + children = [ConceptFactory(parent=source, mnemonic=f'c{i}') for i in range(3)] + for child in children: + child.parent_concepts.set([parent_concept]) + ConceptFactory(parent=source).parent_concepts.set([children[0]]) + + response = self.client.get(parent_concept.url + 'children/?limit=2') + + self.assertEqual(response.status_code, 200) + self.assertEqual(response['num_found'], '3') + self.assertEqual(response['num_returned'], '2') + self.assertIn('page=2', response['next']) + self.assertEqual([data['id'] for data in response.data], ['c0', 'c1']) + self.assertEqual([data['has_children'] for data in response.data], [True, False]) + + response = self.client.get(parent_concept.url + 'children/?limit=2&page=2') + + self.assertEqual(response.status_code, 200) + self.assertEqual(response['num_returned'], '1') + self.assertEqual([data['id'] for data in response.data], ['c2']) + self.assertEqual([data['has_children'] for data in response.data], [False]) + + def test_get_200_pages_are_stable_for_the_same_mnemonic(self): + parent_concept = ConceptFactory() + children = [ConceptFactory(mnemonic='same') for _ in range(3)] # each in its own source + for child in children: + child.parent_concepts.set([parent_concept]) + + with CaptureQueriesContext(connection) as queries: + pages = [self.client.get(f'{parent_concept.url}children/?limit=1&page={page}') for page in (1, 2, 3)] + + self.assertTrue( + any('ORDER BY "concepts"."mnemonic" ASC, "concepts"."id" ASC' in query['sql'] for query in queries)) + self.assertEqual( + [page.data[0]['uuid'] for page in pages], [str(child.id) for child in sorted(children, key=lambda c: c.id)]) + + @patch('core.common.mixins.write_csv_to_s3', return_value='https://csv-url') + def test_csv_has_no_has_children_column(self, write_csv_mock): + parent_concept = ConceptFactory() + ConceptFactory().parent_concepts.set([parent_concept]) + + self.client.get(parent_concept.url + 'children/?csv=true') + + write_csv_mock.assert_called_once() + rows = list(write_csv_mock.call_args.args[0]) + self.assertEqual(len(rows), 1) + self.assertNotIn('has_children_annotated', rows[0]) + + def test_get_200_includes_retired_children(self): + parent_concept = ConceptFactory() + retired_child = ConceptFactory(retired=True) + retired_child.parent_concepts.set([parent_concept]) + + response = self.client.get(parent_concept.url + 'children/') + + self.assertEqual(response.status_code, 200) + self.assertEqual([data['url'] for data in response.data], [retired_child.uri]) + self.assertTrue(response.data[0]['retired']) + class ConceptCollectionMembershipViewTest(OCLAPITestCase): def test_get_200(self):