diff --git a/core/common/management/commands/repair_mapping_source_urls.py b/core/common/management/commands/repair_mapping_source_urls.py new file mode 100644 index 000000000..fb4d4b7ec --- /dev/null +++ b/core/common/management/commands/repair_mapping_source_urls.py @@ -0,0 +1,108 @@ +from django.core.management import BaseCommand +from django.db import transaction +from django.db.models import Exists, F, OuterRef, Q + +from core.common.constants import HEAD +from core.mappings.models import Mapping +from core.sources.models import Source + +# to first: replaced targets are told apart by comparing to_source_url with the not yet repaired from_source_url +SIDES = ( + ('to_source', 'to_source_url', 'to_source_version'), + ('from_source', 'from_source_url', 'from_source_version'), +) + + +class Command(BaseCommand): + help = ( + "Points each HEAD mapping's from_source_url/to_source_url at its from/to source when the URL is missing or " + "is another source's URI or canonical URL (as $clone left them), and clears the source version that came " + "with the old URL. URLs that match the mapping's own source or no known source are left alone. " + "Only HEAD mappings (the versioned objects) are repaired. Mapping versions are left as they are: released " + "source versions and their exports, a mapping's versions list and version detail, and collection " + "expansions holding a version keep the old URL. The next edit of a mapping creates its new version from " + "the repaired HEAD. " + "A mapping whose to_concept was moved into its own source while its to_source_url names a source other " + "than the one it was cloned from (from_source_url, matched by URI or canonical URL) had its target in " + "another code system replaced by $clone. Its to_source_url is the only record of that code system, so it is " + "listed with both URLs to be fixed by hand, and its to_source_url is not repaired. Each source is repaired " + "in its own transaction." + ) + + def add_arguments(self, parser): + parser.add_argument('--dry-run', action='store_true') + + def handle(self, *args, **options): + dry_run = options['dry_run'] + verb = 'would be repaired' if dry_run else 'repaired' + for source_field, url_field, version_field in SIDES: + counts, skipped = self.repair(source_field, url_field, version_field, dry_run) + self.stdout.write(f'{sum(counts.values())} HEAD mapping(s) {verb} on {url_field}') + for source_uri, count in counts.items(): + self.stdout.write(f' {source_uri}: {count}') + if skipped: + self.stdout.write( + f'{len(skipped)} HEAD mapping(s) skipped on {url_field}: target in another code system was ' + f'replaced by a concept in the same source, fix by hand') + for uri, from_url, to_url in skipped: + self.stdout.write(f' {uri} (from_source_url {from_url}, to_source_url {to_url})') + + @staticmethod + def get_head_mappings(): + return Mapping.objects.filter(id=F('versioned_object_id')) + + @staticmethod + def url_missing(url_field): + return Q(**{f'{url_field}__isnull': True}) | Q(**{url_field: ''}) + + @staticmethod + def source_url_is(url_field): + return Q(uri=OuterRef(url_field)) | (Q(canonical_url=OuterRef(url_field)) & ~Q(canonical_url='')) + + def get_replaced_targets(self, rows): + same_source = Source.objects.filter(version=HEAD).filter( + self.source_url_is('to_source_url')).filter(self.source_url_is('from_source_url')) + return rows.filter(to_source_id=F('parent_id')).exclude(Exists(same_source)) + + @staticmethod + def source_url_in(url_field, sources): + return Q(**{f'{url_field}__in': sources.values('uri')}) | Q( + **{f'{url_field}__in': sources.exclude(canonical_url__isnull=True).exclude( + canonical_url='').values('canonical_url')}) + + def get_source_ids(self, source_field, url_field): + candidates = self.url_missing(url_field) | self.source_url_in(url_field, Source.objects.filter(version=HEAD)) + return list( + self.get_head_mappings().filter(**{f'{source_field}__isnull': False}).filter(candidates).exclude( + **{url_field: F(f'{source_field}__uri')} + ).exclude( + Q(**{url_field: F(f'{source_field}__canonical_url')}) & ~Q(**{f'{source_field}__canonical_url': ''}) + ).order_by().values_list(f'{source_field}_id', flat=True).distinct() + ) + + def repair(self, source_field, url_field, version_field, dry_run=False): + counts, skipped = {}, [] + for source in Source.objects.filter(id__in=self.get_source_ids(source_field, url_field)).order_by('uri'): + count, source_skipped = self.repair_source(source, source_field, url_field, version_field, dry_run) + if count: + counts[source.uri] = count + skipped += source_skipped + return counts, skipped + + def repair_source(self, source, source_field, url_field, version_field, dry_run=False): # pylint: disable=too-many-arguments + rows = self.get_head_mappings().filter(**{f'{source_field}_id': source.id}).exclude(**{url_field: source.uri}) + if source.canonical_url: + rows = rows.exclude(**{url_field: source.canonical_url}) + missing = rows.filter(self.url_missing(url_field)) + elsewhere = rows.filter( + self.source_url_in(url_field, Source.objects.filter(version=HEAD).exclude(id=source.id))) + skipped = [] + if source_field == 'to_source': + replaced_target = self.get_replaced_targets(elsewhere) + skipped = list(replaced_target.order_by('uri').values_list('uri', 'from_source_url', 'to_source_url')) + elsewhere = elsewhere.exclude(id__in=replaced_target.values('id')) + if dry_run: + return missing.count() + elsewhere.count(), skipped + with transaction.atomic(): + count = missing.update(**{url_field: source.uri}) + return count + elsewhere.update(**{url_field: source.uri, version_field: None}), skipped diff --git a/core/importers/tests.py b/core/importers/tests.py index 682a17b6e..b2583dd05 100644 --- a/core/importers/tests.py +++ b/core/importers/tests.py @@ -1413,20 +1413,21 @@ def test_mapping_import_with_id_moved_to_another_from_concept_refreshes_the_prev {old_from.versioned_object_id, new_from.versioned_object_id}) def cloned_mapping_in_local_source(self): - """A mapping cloned into a local source: linked to local concepts, with its target URL kept on the origin.""" + """A mapping cloned into a local source before clones fixed source URLs: linked to local concepts, with its + target URL still on the origin.""" organization = OrganizationFactory(mnemonic='CloneOrg') origin = OrganizationSourceFactory(organization=organization, mnemonic='Origin') local = OrganizationSourceFactory(organization=organization, mnemonic='Local') - for source in (origin, local): - ConceptFactory(parent=source, mnemonic='From') + ConceptFactory(parent=origin, mnemonic='From') + local_from = ConceptFactory(parent=local, mnemonic='From') origin_to = ConceptFactory(parent=origin, mnemonic='To') local_to = ConceptFactory(parent=local, mnemonic='To') user = origin_to.created_by mapping = MappingFactory.build( - from_concept=ConceptFactory.build(mnemonic='From'), to_concept=ConceptFactory.build(mnemonic='To'), - parent=local, mnemonic='cloned-map', map_type='Same As', to_source_url=origin.uri, - from_source_url=origin.uri) + from_concept=local_from, to_concept=local_to, parent=local, mnemonic='cloned-map', map_type='Same As') local.clone_mappings([mapping], user) + Mapping.objects.filter(parent=local, from_concept_code='From').update( + from_source_url=origin.uri, to_source_url=origin.uri) mapping = Mapping.objects.get(parent=local, from_concept_code='From', id=F('versioned_object_id')) self.assertEqual((mapping.to_concept_id, mapping.to_source_url), (local_to.id, origin.uri)) return origin, local, mapping diff --git a/core/integration_tests/tests_mappings.py b/core/integration_tests/tests_mappings.py index 154bd54c4..69a395bc7 100644 --- a/core/integration_tests/tests_mappings.py +++ b/core/integration_tests/tests_mappings.py @@ -707,15 +707,16 @@ def test_put_200_sort_weight(self): self.assertEqual(self.mapping.sort_weight, 1.67983) def test_put_200_keeps_the_local_link_of_a_cloned_mapping(self): - """A cloned mapping keeps its target URL on the origin; sending that URL and code back keeps the local link.""" + """A mapping cloned before clones fixed source URLs keeps its target URL on the origin; sending that URL and + code back keeps the local link.""" origin = OrganizationSourceFactory(mnemonic='CloneOrigin') ConceptFactory(parent=origin, mnemonic='To') local_from = ConceptFactory(parent=self.source, mnemonic='From') local_to = ConceptFactory(parent=self.source, mnemonic='To') cloned = MappingFactory.build( - from_concept=ConceptFactory.build(mnemonic='From'), to_concept=ConceptFactory.build(mnemonic='To'), - parent=self.source, map_type=SAME_AS, to_source_url=origin.uri) + from_concept=local_from, to_concept=local_to, parent=self.source, map_type=SAME_AS) self.source.clone_mappings([cloned], self.user) + Mapping.objects.filter(parent=self.source, from_concept_code='From').update(to_source_url=origin.uri) cloned = Mapping.objects.get(parent=self.source, from_concept_code='From', id=F('versioned_object_id')) self.assertEqual((cloned.from_concept_id, cloned.to_concept_id), (local_from.id, local_to.id)) diff --git a/core/integration_tests/tests_sources.py b/core/integration_tests/tests_sources.py index 891fd5f95..b1a0c562a 100644 --- a/core/integration_tests/tests_sources.py +++ b/core/integration_tests/tests_sources.py @@ -10,6 +10,7 @@ from django.core.cache import cache from django.core.files.uploadedfile import SimpleUploadedFile from django.db import transaction +from django.db.models import F from django.test import override_settings, tag from mock import patch, Mock, ANY, PropertyMock from mock.mock import call @@ -2717,6 +2718,40 @@ def test_post_success(self, bundle_clone_mock): resource_budget=100, **parameters # no group: the preview per-call budget (ocl_online#230) ) + def test_post_retargets_answers_to_equivalents_and_keeps_targets_in_other_code_systems(self): # pylint: disable=too-many-locals + def named_concept(parent): + return ConceptFactory(parent=parent, names=[ConceptNameFactory.build(locale_preferred=True)]) + + source = OrganizationSourceFactory() + external = OrganizationSourceFactory() + target = self.clone_to_source + question, answer, external_concept = named_concept(source), named_concept(source), named_concept(external) + existing_answer, existing_external = named_concept(target), named_concept(target) + for existing, original in ((existing_answer, answer), (existing_external, external_concept)): + MappingFactory(parent=target, from_concept=existing, to_concept=original, map_type='SAME-AS') + MappingFactory( + parent=source, from_concept=question, to_concept=answer, map_type='Q-AND-A', + from_source=source, to_source=source, from_source_url=source.uri, to_source_url=source.uri) + MappingFactory( + parent=source, from_concept=question, to_concept=external_concept, map_type='SAME-AS', + from_source=source, to_source=external, from_source_url=source.uri, to_source_url=external.uri) + + response = self.post_clone( + [question.uri], user=UserProfileFactory(is_staff=True), + parameters={'mapTypes': 'Q-AND-A,SAME-AS', 'equivalencyMapType': 'SAME-AS'}) + + self.assertEqual(response.data[question.uri]['status'], 200) + copies = Mapping.objects.filter(parent=target, from_source_url=target.uri, id=F('versioned_object_id')) + answer_copy = copies.get(map_type='Q-AND-A') + self.assertEqual(answer_copy.to_concept_id, existing_answer.id) + self.assertEqual((answer_copy.to_source_id, answer_copy.to_source_url), (target.id, target.uri)) + external_copy = copies.get(map_type='SAME-AS', to_source_id=external.id) + self.assertEqual(external_copy.to_concept_id, external_concept.id) + self.assertEqual(external_copy.to_concept_code, external_concept.mnemonic) + self.assertEqual(external_copy.to_source_url, external.uri) + self.assertFalse(copies.filter(to_concept=existing_external).exists()) + + class SourceVersionsChangelogOutputViewTest(OCLAPITestCase): def _build_changelog_output_fixture(self): # pylint: disable=too-many-locals source = OrganizationSourceFactory() diff --git a/core/mappings/models.py b/core/mappings/models.py index 6655d36aa..a99815b06 100644 --- a/core/mappings/models.py +++ b/core/mappings/models.py @@ -296,13 +296,38 @@ def clone(self, user=None, from_concept=None, to_concept=None): ) if to_concept: mapping.to_concept = to_concept + mapping.set_to_source_from_concept( + to_concept, get(to_concept, 'parent_id') != get(self, 'to_concept.parent_id')) if from_concept: mapping.from_concept = from_concept + mapping.set_from_source_from_concept( + from_concept, get(from_concept, 'parent_id') != get(self, 'from_concept.parent_id')) if user: mapping.created_by = mapping.updated_by = user return mapping + def set_from_source_from_concept(self, concept, source_changed=False): + self.from_source_id, self.from_source_url, self.from_source_version = self.get_source_fields_from_concept( + concept, (self.from_source_id, self.from_source_url, self.from_source_version), source_changed) + + def set_to_source_from_concept(self, concept, source_changed=False): + self.to_source_id, self.to_source_url, self.to_source_version = self.get_source_fields_from_concept( + concept, (self.to_source_id, self.to_source_url, self.to_source_version), source_changed) + + @staticmethod + def get_source_fields_from_concept(concept, source_fields, source_changed=False): + """(source id, URL, version) for concept's source. Unless the source changed, a recorded URL and version + are kept.""" + source_id, url, version = source_fields + if not get(concept, 'parent_id'): + return source_id, url, version + if source_changed or not url: + url = concept.parent.uri + if source_changed: + version = None + return concept.parent_id, url, version + @classmethod def build(cls, map_type, from_concept, to_concept, **kwargs): return cls( diff --git a/core/mappings/tests/tests.py b/core/mappings/tests/tests.py index d783f9572..6a1369eb0 100644 --- a/core/mappings/tests/tests.py +++ b/core/mappings/tests/tests.py @@ -324,6 +324,79 @@ def test_save_as_new_version(self): persisted_mapping.version_url, persisted_mapping.uri ) + def test_repair_mapping_source_urls(self): # pylint: disable=too-many-locals,too-many-statements + origin = OrganizationSourceFactory(version=HEAD, canonical_url='https://example.org/origin') + source = OrganizationSourceFactory(version=HEAD, canonical_url='https://example.org/source') + other = OrganizationSourceFactory(version=HEAD, canonical_url='https://example.org/other') + concept = ConceptFactory(parent=source) + answer = ConceptFactory(parent=source) + other_concept = ConceptFactory(parent=other) + blank = OrganizationSourceFactory(version=HEAD, canonical_url='') + blank_concept = ConceptFactory(parent=blank) + blank.refresh_from_db() + self.assertEqual(blank.canonical_url, '') + + def make(to_concept=other_concept, **urls): + mapping = MappingFactory( + parent=source, from_concept=concept, to_concept=to_concept, from_source=source, + to_source=to_concept.parent, from_source_url=source.uri, to_source_url=to_concept.parent.uri) + Mapping.objects.filter(id=mapping.id).update(**urls) + return mapping + + def urls(mapping): + mapping.refresh_from_db() + return (mapping.from_source_url, mapping.from_source_version, + mapping.to_source_url, mapping.to_source_version) + + stale_uri = make( + from_source_url=other.uri, from_source_version='v1', to_source_url=source.uri, to_source_version='v2') + stale_canonical = make(from_source_url=other.canonical_url, from_source_version='v1') + missing = make(from_source_url='', to_source_url=None) + own_canonical = make(from_source_url=source.canonical_url, from_source_version='v1') + unknown = make(from_source_url='/orgs/foo/sources/bar/') + retargeted_answer = make(to_concept=answer, from_source_url=origin.uri, to_source_url=origin.uri) + canonical_answer = make(to_concept=answer, from_source_url=origin.uri, to_source_url=origin.canonical_url) + replaced_target = make(to_concept=answer, from_source_url=origin.uri, to_source_url=other.uri) + blank_canonical = make(to_concept=blank_concept, to_source_url='') + self.assertEqual(stale_uri.versioned_object_id, stale_uri.id) + stale_version = Mapping.create_initial_version(stale_uri) + Mapping.objects.filter(id=stale_version.id).update(from_source_url=other.uri, from_source_version='v1') + repaired = ( + stale_uri, stale_canonical, missing, retargeted_answer, canonical_answer, replaced_target, blank_canonical) + replaced_line = f' {replaced_target.uri} (from_source_url {origin.uri}, to_source_url {other.uri})\n' + + before = {mapping.id: urls(mapping) for mapping in repaired} + out = StringIO() + call_command('repair_mapping_source_urls', '--dry-run', stdout=out) + to_output, from_output = out.getvalue().split('on from_source_url') + self.assertIn(f' {other.uri}: 2\n', to_output) + self.assertIn(f' {source.uri}: 2\n', to_output) + self.assertIn(replaced_line, to_output) + self.assertIn(f' {blank.uri}: 1\n', to_output) + self.assertIn(f' {source.uri}: 6\n', from_output) + self.assertEqual(before, {mapping.id: urls(mapping) for mapping in repaired}) + + call_command('repair_mapping_source_urls', stdout=StringIO()) + + self.assertEqual(urls(stale_uri), (source.uri, None, other.uri, None)) + self.assertEqual(urls(stale_canonical), (source.uri, None, other.uri, None)) + self.assertEqual(urls(missing), (source.uri, None, other.uri, None)) + self.assertEqual(urls(own_canonical), (source.canonical_url, 'v1', other.uri, None)) + self.assertEqual(urls(unknown), ('/orgs/foo/sources/bar/', None, other.uri, None)) + self.assertEqual(urls(retargeted_answer), (source.uri, None, source.uri, None)) + self.assertEqual(urls(canonical_answer), (source.uri, None, source.uri, None)) + self.assertEqual(urls(blank_canonical), (source.uri, None, blank.uri, None)) + self.assertEqual(urls(replaced_target), (source.uri, None, other.uri, None)) + self.assertEqual(urls(stale_version)[:2], (other.uri, 'v1')) + + out = StringIO() + call_command('repair_mapping_source_urls', stdout=out) + self.assertNotIn(f' {source.uri}:', out.getvalue()) + self.assertNotIn(f' {other.uri}:', out.getvalue()) + self.assertIn(f' {replaced_target.uri} (from_source_url {source.uri}, to_source_url {other.uri})\n', + out.getvalue()) + self.assertEqual(urls(replaced_target), (source.uri, None, other.uri, None)) + def test_save_as_new_version_and_repair_demote_every_version_flagged_latest(self): source = OrganizationSourceFactory(version=HEAD) mapping = MappingFactory(parent=source) diff --git a/core/sources/models.py b/core/sources/models.py index e254b8f4b..d45ab6902 100644 --- a/core/sources/models.py +++ b/core/sources/models.py @@ -845,7 +845,7 @@ def clone_resources(self, user, concepts, mappings, **kwargs): # pylint: disabl concept_errors, mapping_errors = [], [] equivalency_map_types = compact((kwargs.get('equivalency_map_types') or '').split(',')) _concepts_to_add_mappings_for = [] - for concept in concepts: + for concept in concepts.select_related('parent'): if self.get_equivalent_concept(concept, equivalency_map_types): continue @@ -871,8 +871,13 @@ def clone_resources(self, user, concepts, mappings, **kwargs): # pylint: disabl for concept_pair in _concepts_to_add_mappings_for: concept, cloned_concept = concept_pair - for mapping in mappings.filter(from_concept__versioned_object_id=concept.versioned_object_id): - existing_to_concept = self.get_equivalent_concept(mapping.to_concept, equivalency_map_types) + for mapping in mappings.filter( + from_concept__versioned_object_id=concept.versioned_object_id + ).select_related('from_concept', 'to_concept'): + # only targets in the source being cloned from move to their equivalent; other code systems stay + existing_to_concept = self.get_equivalent_concept( + mapping.to_concept, equivalency_map_types + ) if get(mapping, 'to_concept.parent_id') == concept.parent_id else None cloned_mapping = mapping.clone(user, cloned_concept, existing_to_concept) self.clone_mappings([cloned_mapping], user, False) if cloned_mapping.id: @@ -924,7 +929,7 @@ def get_equivalent_mapping(self, concept, equivalency_map_type): return self.mappings_set.filter( map_type__in=equivalency_map_type, to_concept__versioned_object_id=concept.versioned_object_id, from_concept__parent_id=self.id, retired=False, id=F('versioned_object_id') - ).first() if equivalency_map_type and concept else None + ).select_related('from_concept__parent').first() if equivalency_map_type and concept else None def clone_with_cascade(self, concept_to_clone, user, resource_budget=None, **kwargs): """ @@ -960,16 +965,10 @@ def clone_mappings(self, cloned_mappings, user, update_count=True): for mapping in cloned_mappings: to_concept = get(mapping, 'to_concept') from_concept = get(mapping, 'from_concept') - if not get(from_concept, 'id'): - from_concept = self.find_concept_by_mnemonic(from_concept.mnemonic) - if not get(to_concept, 'id') and get(to_concept, 'mnemonic'): - to_concept = self.find_concept_by_mnemonic(to_concept.mnemonic) mapping.from_concept_code = get(from_concept, 'mnemonic') or mapping.from_concept_code - mapping.from_concept_id = get(from_concept, 'id') - mapping.to_concept_id = get(to_concept, 'id') mapping.to_concept_code = get(to_concept, 'mnemonic') or mapping.to_concept_code - mapping.to_source_id = get(to_concept, 'parent_id') or mapping.to_source_id - mapping.from_source_id = get(from_concept, 'parent_id') or mapping.from_source_id + mapping.set_from_source_from_concept(from_concept) + mapping.set_to_source_from_concept(to_concept) self._clone_resource(mapping, user) if mapping.id: added.append(mapping) diff --git a/core/sources/tests/tests.py b/core/sources/tests/tests.py index 351a06e91..c9fe52f77 100644 --- a/core/sources/tests/tests.py +++ b/core/sources/tests/tests.py @@ -2286,27 +2286,127 @@ def test_clone_resources_skips_existing_equivalent_concept(self): self.assertEqual(added_concepts, []) - def test_clone_mappings_resolves_missing_concepts_by_mnemonic(self): + def test_clone_resources_keeps_targets_in_other_code_systems(self): source = OrganizationSourceFactory(version=HEAD) - from_concept = ConceptFactory( - parent=source, names=[ConceptNameFactory.build(locale_preferred=True)]) - to_concept = ConceptFactory( - parent=source, names=[ConceptNameFactory.build(locale_preferred=True)]) + external = OrganizationSourceFactory(version=HEAD) target = OrganizationSourceFactory(version=HEAD) - user = from_concept.created_by - - cloned_from = from_concept.versioned_object.clone() - target.clone_concepts([cloned_from], user, False) - cloned_to = to_concept.versioned_object.clone() - target.clone_concepts([cloned_to], user, False) - - unsaved_from = ConceptFactory.build(mnemonic=cloned_from.mnemonic) - unsaved_to = ConceptFactory.build(mnemonic=cloned_to.mnemonic) - mapping = MappingFactory.build(from_concept=unsaved_from, to_concept=unsaved_to, parent=target) - - added = target.clone_mappings([mapping], user) - - self.assertEqual(len(added), 1) + concept = ConceptFactory(parent=source, names=[ConceptNameFactory.build(locale_preferred=True)]) + external_concept = ConceptFactory(parent=external, names=[ConceptNameFactory.build(locale_preferred=True)]) + existing = ConceptFactory(parent=target, names=[ConceptNameFactory.build(locale_preferred=True)]) + MappingFactory(from_concept=existing, to_concept=external_concept, parent=target, map_type='SAME-AS') + mapping = MappingFactory( + from_concept=concept, to_concept=external_concept, parent=source, map_type='BROADER-THAN', + from_source=source, to_source=external, from_source_url=source.uri, to_source_url=external.uri) + + _, added_mappings = target.clone_resources( + concept.created_by, Concept.objects.filter(id=concept.id), Mapping.objects.filter(id=mapping.id), + equivalency_map_types='SAME-AS') + + cloned_mapping = added_mappings[-1] + cloned_mapping.refresh_from_db() + self.assertEqual(cloned_mapping.map_type, 'BROADER-THAN') + self.assertEqual(cloned_mapping.to_concept_id, external_concept.id) + self.assertEqual(cloned_mapping.to_concept_code, external_concept.mnemonic) + self.assertEqual(cloned_mapping.to_source_id, external.id) + self.assertEqual(cloned_mapping.to_source_url, external.uri) + + def test_clone_resources_points_mapping_source_urls_at_the_mapped_concepts_sources(self): + source = OrganizationSourceFactory(version=HEAD) + external = OrganizationSourceFactory(version=HEAD) + target = OrganizationSourceFactory(version=HEAD) + concept = ConceptFactory(parent=source, names=[ConceptNameFactory.build(locale_preferred=True)]) + external_concept = ConceptFactory(parent=external, names=[ConceptNameFactory.build(locale_preferred=True)]) + mapping = MappingFactory( + from_concept=concept, to_concept=external_concept, parent=source, map_type='SAME-AS', + from_source=source, to_source=external, from_source_url=source.uri, to_source_url=external.uri, + from_source_version='v1', to_source_version='v2') + self.assertEqual(mapping.from_source_url, source.uri) + + added_concepts, added_mappings = target.clone_resources( + concept.created_by, Concept.objects.filter(id=concept.id), Mapping.objects.filter(id=mapping.id), + equivalency_map_types='SAME-AS') + + cloned_concept = added_concepts[0] + equivalency_mapping, cloned_mapping = added_mappings + for added in added_mappings: + added.refresh_from_db() + self.assertEqual(added.from_concept_id, cloned_concept.id) + self.assertEqual(added.from_source_id, target.id) + self.assertEqual(added.from_source_url, target.uri) + self.assertIsNone(added.from_source_version) + self.assertEqual(equivalency_mapping.to_source_id, source.id) + self.assertEqual(equivalency_mapping.to_source_url, source.uri) + self.assertEqual(cloned_mapping.to_concept_id, external_concept.id) + self.assertEqual(cloned_mapping.to_source_id, external.id) + self.assertEqual(cloned_mapping.to_source_url, external.uri) + self.assertEqual(cloned_mapping.to_source_version, 'v2') + + def test_clone_resources_keeps_external_source_url_and_version_without_recorded_source(self): + source = OrganizationSourceFactory(version=HEAD) + external = OrganizationSourceFactory(version=HEAD) + target = OrganizationSourceFactory(version=HEAD) + concept = ConceptFactory(parent=source, names=[ConceptNameFactory.build(locale_preferred=True)]) + external_concept = ConceptFactory(parent=external, names=[ConceptNameFactory.build(locale_preferred=True)]) + mapping = MappingFactory( + from_concept=concept, to_concept=external_concept, parent=source, map_type='SAME-AS', + from_source=source, from_source_url=source.uri, to_source_url='http://example.org/external', + to_source_version='2.74') + self.assertIsNone(mapping.to_source_id) + + _, added_mappings = target.clone_resources( + concept.created_by, Concept.objects.filter(id=concept.id), Mapping.objects.filter(id=mapping.id)) + + cloned_mapping = added_mappings[0] + cloned_mapping.refresh_from_db() + self.assertEqual(cloned_mapping.from_source_url, target.uri) + self.assertEqual(cloned_mapping.to_source_id, external.id) + self.assertEqual(cloned_mapping.to_source_url, 'http://example.org/external') + self.assertEqual(cloned_mapping.to_source_version, '2.74') + + def test_clone_resources_copies_a_self_mapping_onto_the_cloned_concept(self): + source = OrganizationSourceFactory(version=HEAD) + target = OrganizationSourceFactory(version=HEAD) + concept = ConceptFactory(parent=source, names=[ConceptNameFactory.build(locale_preferred=True)]) + mapping = MappingFactory( + from_concept=concept, to_concept=concept, parent=source, map_type='SAME-AS', + from_source=source, to_source=source, from_source_url=source.uri, to_source_url=source.uri) + + added_concepts, added_mappings = target.clone_resources( + concept.created_by, Concept.objects.filter(id=concept.id), Mapping.objects.filter(id=mapping.id), + equivalency_map_types='SAME-AS') + + cloned_concept = added_concepts[0] + self.assertEqual(len(added_mappings), 2) + cloned_mapping = added_mappings[-1] + cloned_mapping.refresh_from_db() + self.assertEqual((cloned_mapping.from_concept_id, cloned_mapping.to_concept_id), + (cloned_concept.id, cloned_concept.id)) + self.assertEqual((cloned_mapping.from_source_url, cloned_mapping.to_source_url), (target.uri, target.uri)) + self.assertEqual(cloned_mapping.to_source_id, target.id) + + def test_clone_resources_points_to_source_url_at_the_equivalent_concepts_source(self): + source = OrganizationSourceFactory(version=HEAD) + target = OrganizationSourceFactory(version=HEAD) + concept = ConceptFactory(parent=source, names=[ConceptNameFactory.build(locale_preferred=True)]) + answer = ConceptFactory(parent=source, names=[ConceptNameFactory.build(locale_preferred=True)]) + existing_answer = ConceptFactory(parent=target, names=[ConceptNameFactory.build(locale_preferred=True)]) + MappingFactory(from_concept=existing_answer, to_concept=answer, parent=target, map_type='SAME-AS') + mapping = MappingFactory( + from_concept=concept, to_concept=answer, parent=source, map_type='Q-AND-A', + from_source=source, to_source=source, from_source_url=source.uri, to_source_url=source.uri, + to_source_version='v1') + self.assertEqual(mapping.to_source_url, source.uri) + + _, added_mappings = target.clone_resources( + concept.created_by, Concept.objects.filter(id=concept.id), Mapping.objects.filter(id=mapping.id), + equivalency_map_types='SAME-AS') + + cloned_mapping = added_mappings[-1] + cloned_mapping.refresh_from_db() + self.assertEqual(cloned_mapping.to_concept_id, existing_answer.id) + self.assertEqual(cloned_mapping.to_source_id, target.id) + self.assertEqual(cloned_mapping.to_source_url, target.uri) + self.assertIsNone(cloned_mapping.to_source_version) @patch('core.sources.models.Source.update_mappings_count') def test_clone_mappings_updates_count_when_update_count_true(self, update_count_mock): @@ -2322,9 +2422,7 @@ def test_clone_mappings_updates_count_when_update_count_true(self, update_count_ cloned_to = to_concept.versioned_object.clone() target.clone_concepts([cloned_to], user, False) - mapping = MappingFactory.build( - from_concept=ConceptFactory.build(mnemonic=cloned_from.mnemonic), - to_concept=ConceptFactory.build(mnemonic=cloned_to.mnemonic), parent=target) + mapping = MappingFactory.build(from_concept=cloned_from, to_concept=cloned_to, parent=target) target.clone_mappings([mapping], user)