Repository navigation
OpenConceptLab/ocl_issues#3012 #3013 | $clone keeps mappings to other code systems and points copied mappings source URLs at their concepts sources; repair command for HEAD mappings #995
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from all commits
d25073f
512332a
908c8c8
e1dd19b
c73da96
4285451
2643c6b
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,108 @@ | ||
| from django.core.management import BaseCommand | ||
| from django.db import transaction | ||
| from django.db.models import Exists, F, OuterRef, Q | ||
|
|
||
| from core.common.constants import HEAD | ||
| from core.mappings.models import Mapping | ||
| from core.sources.models import Source | ||
|
|
||
| # to first: replaced targets are told apart by comparing to_source_url with the not yet repaired from_source_url | ||
| SIDES = ( | ||
| ('to_source', 'to_source_url', 'to_source_version'), | ||
| ('from_source', 'from_source_url', 'from_source_version'), | ||
| ) | ||
|
|
||
|
|
||
| class Command(BaseCommand): | ||
| help = ( | ||
| "Points each HEAD mapping's from_source_url/to_source_url at its from/to source when the URL is missing or " | ||
| "is another source's URI or canonical URL (as $clone left them), and clears the source version that came " | ||
| "with the old URL. URLs that match the mapping's own source or no known source are left alone. " | ||
| "Only HEAD mappings (the versioned objects) are repaired. Mapping versions are left as they are: released " | ||
| "source versions and their exports, a mapping's versions list and version detail, and collection " | ||
| "expansions holding a version keep the old URL. The next edit of a mapping creates its new version from " | ||
| "the repaired HEAD. " | ||
| "A mapping whose to_concept was moved into its own source while its to_source_url names a source other " | ||
| "than the one it was cloned from (from_source_url, matched by URI or canonical URL) had its target in " | ||
| "another code system replaced by $clone. Its to_source_url is the only record of that code system, so it is " | ||
| "listed with both URLs to be fixed by hand, and its to_source_url is not repaired. Each source is repaired " | ||
| "in its own transaction." | ||
| ) | ||
|
|
||
| def add_arguments(self, parser): | ||
| parser.add_argument('--dry-run', action='store_true') | ||
|
|
||
| def handle(self, *args, **options): | ||
| dry_run = options['dry_run'] | ||
| verb = 'would be repaired' if dry_run else 'repaired' | ||
| for source_field, url_field, version_field in SIDES: | ||
| counts, skipped = self.repair(source_field, url_field, version_field, dry_run) | ||
| self.stdout.write(f'{sum(counts.values())} HEAD mapping(s) {verb} on {url_field}') | ||
| for source_uri, count in counts.items(): | ||
| self.stdout.write(f' {source_uri}: {count}') | ||
| if skipped: | ||
| self.stdout.write( | ||
| f'{len(skipped)} HEAD mapping(s) skipped on {url_field}: target in another code system was ' | ||
| f'replaced by a concept in the same source, fix by hand') | ||
| for uri, from_url, to_url in skipped: | ||
| self.stdout.write(f' {uri} (from_source_url {from_url}, to_source_url {to_url})') | ||
|
|
||
| @staticmethod | ||
| def get_head_mappings(): | ||
|
snyaggarwal marked this conversation as resolved.
Resolved
|
||
| return Mapping.objects.filter(id=F('versioned_object_id')) | ||
|
|
||
| @staticmethod | ||
| def url_missing(url_field): | ||
| return Q(**{f'{url_field}__isnull': True}) | Q(**{url_field: ''}) | ||
|
|
||
| @staticmethod | ||
| def source_url_is(url_field): | ||
| return Q(uri=OuterRef(url_field)) | (Q(canonical_url=OuterRef(url_field)) & ~Q(canonical_url='')) | ||
|
|
||
| def get_replaced_targets(self, rows): | ||
| same_source = Source.objects.filter(version=HEAD).filter( | ||
| self.source_url_is('to_source_url')).filter(self.source_url_is('from_source_url')) | ||
| return rows.filter(to_source_id=F('parent_id')).exclude(Exists(same_source)) | ||
|
|
||
| @staticmethod | ||
| def source_url_in(url_field, sources): | ||
| return Q(**{f'{url_field}__in': sources.values('uri')}) | Q( | ||
| **{f'{url_field}__in': sources.exclude(canonical_url__isnull=True).exclude( | ||
| canonical_url='').values('canonical_url')}) | ||
|
|
||
| def get_source_ids(self, source_field, url_field): | ||
| candidates = self.url_missing(url_field) | self.source_url_in(url_field, Source.objects.filter(version=HEAD)) | ||
| return list( | ||
| self.get_head_mappings().filter(**{f'{source_field}__isnull': False}).filter(candidates).exclude( | ||
| **{url_field: F(f'{source_field}__uri')} | ||
| ).exclude( | ||
| Q(**{url_field: F(f'{source_field}__canonical_url')}) & ~Q(**{f'{source_field}__canonical_url': ''}) | ||
| ).order_by().values_list(f'{source_field}_id', flat=True).distinct() | ||
| ) | ||
|
|
||
| def repair(self, source_field, url_field, version_field, dry_run=False): | ||
| counts, skipped = {}, [] | ||
| for source in Source.objects.filter(id__in=self.get_source_ids(source_field, url_field)).order_by('uri'): | ||
| count, source_skipped = self.repair_source(source, source_field, url_field, version_field, dry_run) | ||
| if count: | ||
| counts[source.uri] = count | ||
| skipped += source_skipped | ||
| return counts, skipped | ||
|
|
||
| def repair_source(self, source, source_field, url_field, version_field, dry_run=False): # pylint: disable=too-many-arguments | ||
| rows = self.get_head_mappings().filter(**{f'{source_field}_id': source.id}).exclude(**{url_field: source.uri}) | ||
| if source.canonical_url: | ||
| rows = rows.exclude(**{url_field: source.canonical_url}) | ||
| missing = rows.filter(self.url_missing(url_field)) | ||
| elsewhere = rows.filter( | ||
| self.source_url_in(url_field, Source.objects.filter(version=HEAD).exclude(id=source.id))) | ||
| skipped = [] | ||
| if source_field == 'to_source': | ||
| replaced_target = self.get_replaced_targets(elsewhere) | ||
| skipped = list(replaced_target.order_by('uri').values_list('uri', 'from_source_url', 'to_source_url')) | ||
| elsewhere = elsewhere.exclude(id__in=replaced_target.values('id')) | ||
| if dry_run: | ||
| return missing.count() + elsewhere.count(), skipped | ||
| with transaction.atomic(): | ||
| count = missing.update(**{url_field: source.uri}) | ||
| return count + elsewhere.update(**{url_field: source.uri, version_field: None}), skipped | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -10,6 +10,7 @@ | |
| from django.core.cache import cache | ||
| from django.core.files.uploadedfile import SimpleUploadedFile | ||
| from django.db import transaction | ||
| from django.db.models import F | ||
| from django.test import override_settings, tag | ||
| from mock import patch, Mock, ANY, PropertyMock | ||
| from mock.mock import call | ||
|
|
@@ -2717,6 +2718,40 @@ def test_post_success(self, bundle_clone_mock): | |
| resource_budget=100, **parameters # no group: the preview per-call budget (ocl_online#230) | ||
| ) | ||
|
|
||
| def test_post_retargets_answers_to_equivalents_and_keeps_targets_in_other_code_systems(self): # pylint: disable=too-many-locals | ||
| def named_concept(parent): | ||
| return ConceptFactory(parent=parent, names=[ConceptNameFactory.build(locale_preferred=True)]) | ||
|
|
||
| source = OrganizationSourceFactory() | ||
| external = OrganizationSourceFactory() | ||
| target = self.clone_to_source | ||
| question, answer, external_concept = named_concept(source), named_concept(source), named_concept(external) | ||
| existing_answer, existing_external = named_concept(target), named_concept(target) | ||
| for existing, original in ((existing_answer, answer), (existing_external, external_concept)): | ||
| MappingFactory(parent=target, from_concept=existing, to_concept=original, map_type='SAME-AS') | ||
| MappingFactory( | ||
| parent=source, from_concept=question, to_concept=answer, map_type='Q-AND-A', | ||
| from_source=source, to_source=source, from_source_url=source.uri, to_source_url=source.uri) | ||
| MappingFactory( | ||
| parent=source, from_concept=question, to_concept=external_concept, map_type='SAME-AS', | ||
| from_source=source, to_source=external, from_source_url=source.uri, to_source_url=external.uri) | ||
|
|
||
| response = self.post_clone( | ||
| [question.uri], user=UserProfileFactory(is_staff=True), | ||
| parameters={'mapTypes': 'Q-AND-A,SAME-AS', 'equivalencyMapType': 'SAME-AS'}) | ||
|
|
||
| self.assertEqual(response.data[question.uri]['status'], 200) | ||
| copies = Mapping.objects.filter(parent=target, from_source_url=target.uri, id=F('versioned_object_id')) | ||
| answer_copy = copies.get(map_type='Q-AND-A') | ||
| self.assertEqual(answer_copy.to_concept_id, existing_answer.id) | ||
| self.assertEqual((answer_copy.to_source_id, answer_copy.to_source_url), (target.id, target.uri)) | ||
| external_copy = copies.get(map_type='SAME-AS', to_source_id=external.id) | ||
| self.assertEqual(external_copy.to_concept_id, external_concept.id) | ||
| self.assertEqual(external_copy.to_concept_code, external_concept.mnemonic) | ||
| self.assertEqual(external_copy.to_source_url, external.uri) | ||
| self.assertFalse(copies.filter(to_concept=existing_external).exists()) | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Style (nit, non-blocking): add a second blank line after this test. There's only one blank line between the end of this test and
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Fixed in 2643c6b: two blank lines before |
||
|
|
||
|
|
||
| class SourceVersionsChangelogOutputViewTest(OCLAPITestCase): | ||
| def _build_changelog_output_fixture(self): # pylint: disable=too-many-locals | ||
| source = OrganizationSourceFactory() | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.