Skip to content
Open
108 changes: 108 additions & 0 deletions core/common/management/commands/repair_mapping_source_urls.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
from django.core.management import BaseCommand
from django.db import transaction
from django.db.models import Exists, F, OuterRef, Q

from core.common.constants import HEAD
from core.mappings.models import Mapping
from core.sources.models import Source

# to first: replaced targets are told apart by comparing to_source_url with the not yet repaired from_source_url
SIDES = (
Comment thread
snyaggarwal marked this conversation as resolved.
Resolved
('to_source', 'to_source_url', 'to_source_version'),
('from_source', 'from_source_url', 'from_source_version'),
)


class Command(BaseCommand):
help = (
"Points each HEAD mapping's from_source_url/to_source_url at its from/to source when the URL is missing or "
"is another source's URI or canonical URL (as $clone left them), and clears the source version that came "
"with the old URL. URLs that match the mapping's own source or no known source are left alone. "
"Only HEAD mappings (the versioned objects) are repaired. Mapping versions are left as they are: released "
"source versions and their exports, a mapping's versions list and version detail, and collection "
"expansions holding a version keep the old URL. The next edit of a mapping creates its new version from "
"the repaired HEAD. "
"A mapping whose to_concept was moved into its own source while its to_source_url names a source other "
"than the one it was cloned from (from_source_url, matched by URI or canonical URL) had its target in "
"another code system replaced by $clone. Its to_source_url is the only record of that code system, so it is "
"listed with both URLs to be fixed by hand, and its to_source_url is not repaired. Each source is repaired "
"in its own transaction."
)

def add_arguments(self, parser):
parser.add_argument('--dry-run', action='store_true')

def handle(self, *args, **options):
dry_run = options['dry_run']
verb = 'would be repaired' if dry_run else 'repaired'
for source_field, url_field, version_field in SIDES:
counts, skipped = self.repair(source_field, url_field, version_field, dry_run)
self.stdout.write(f'{sum(counts.values())} HEAD mapping(s) {verb} on {url_field}')
for source_uri, count in counts.items():
self.stdout.write(f' {source_uri}: {count}')
if skipped:
self.stdout.write(
f'{len(skipped)} HEAD mapping(s) skipped on {url_field}: target in another code system was '
f'replaced by a concept in the same source, fix by hand')
for uri, from_url, to_url in skipped:
self.stdout.write(f' {uri} (from_source_url {from_url}, to_source_url {to_url})')

@staticmethod
def get_head_mappings():
Comment thread
snyaggarwal marked this conversation as resolved.
Resolved
return Mapping.objects.filter(id=F('versioned_object_id'))

@staticmethod
def url_missing(url_field):
return Q(**{f'{url_field}__isnull': True}) | Q(**{url_field: ''})

@staticmethod
def source_url_is(url_field):
return Q(uri=OuterRef(url_field)) | (Q(canonical_url=OuterRef(url_field)) & ~Q(canonical_url=''))

def get_replaced_targets(self, rows):
same_source = Source.objects.filter(version=HEAD).filter(
self.source_url_is('to_source_url')).filter(self.source_url_is('from_source_url'))
return rows.filter(to_source_id=F('parent_id')).exclude(Exists(same_source))

@staticmethod
def source_url_in(url_field, sources):
return Q(**{f'{url_field}__in': sources.values('uri')}) | Q(
**{f'{url_field}__in': sources.exclude(canonical_url__isnull=True).exclude(
canonical_url='').values('canonical_url')})

def get_source_ids(self, source_field, url_field):
candidates = self.url_missing(url_field) | self.source_url_in(url_field, Source.objects.filter(version=HEAD))
return list(
self.get_head_mappings().filter(**{f'{source_field}__isnull': False}).filter(candidates).exclude(
**{url_field: F(f'{source_field}__uri')}
).exclude(
Q(**{url_field: F(f'{source_field}__canonical_url')}) & ~Q(**{f'{source_field}__canonical_url': ''})
).order_by().values_list(f'{source_field}_id', flat=True).distinct()
)

def repair(self, source_field, url_field, version_field, dry_run=False):
counts, skipped = {}, []
for source in Source.objects.filter(id__in=self.get_source_ids(source_field, url_field)).order_by('uri'):
count, source_skipped = self.repair_source(source, source_field, url_field, version_field, dry_run)
if count:
counts[source.uri] = count
skipped += source_skipped
return counts, skipped

def repair_source(self, source, source_field, url_field, version_field, dry_run=False): # pylint: disable=too-many-arguments
rows = self.get_head_mappings().filter(**{f'{source_field}_id': source.id}).exclude(**{url_field: source.uri})
if source.canonical_url:
rows = rows.exclude(**{url_field: source.canonical_url})
missing = rows.filter(self.url_missing(url_field))
elsewhere = rows.filter(
self.source_url_in(url_field, Source.objects.filter(version=HEAD).exclude(id=source.id)))
skipped = []
if source_field == 'to_source':
replaced_target = self.get_replaced_targets(elsewhere)
skipped = list(replaced_target.order_by('uri').values_list('uri', 'from_source_url', 'to_source_url'))
elsewhere = elsewhere.exclude(id__in=replaced_target.values('id'))
if dry_run:
return missing.count() + elsewhere.count(), skipped
with transaction.atomic():
count = missing.update(**{url_field: source.uri})
return count + elsewhere.update(**{url_field: source.uri, version_field: None}), skipped
13 changes: 7 additions & 6 deletions core/importers/tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -1413,20 +1413,21 @@ def test_mapping_import_with_id_moved_to_another_from_concept_refreshes_the_prev
{old_from.versioned_object_id, new_from.versioned_object_id})

def cloned_mapping_in_local_source(self):
"""A mapping cloned into a local source: linked to local concepts, with its target URL kept on the origin."""
"""A mapping cloned into a local source before clones fixed source URLs: linked to local concepts, with its
target URL still on the origin."""
organization = OrganizationFactory(mnemonic='CloneOrg')
origin = OrganizationSourceFactory(organization=organization, mnemonic='Origin')
local = OrganizationSourceFactory(organization=organization, mnemonic='Local')
for source in (origin, local):
ConceptFactory(parent=source, mnemonic='From')
ConceptFactory(parent=origin, mnemonic='From')
local_from = ConceptFactory(parent=local, mnemonic='From')
origin_to = ConceptFactory(parent=origin, mnemonic='To')
local_to = ConceptFactory(parent=local, mnemonic='To')
user = origin_to.created_by
mapping = MappingFactory.build(
from_concept=ConceptFactory.build(mnemonic='From'), to_concept=ConceptFactory.build(mnemonic='To'),
parent=local, mnemonic='cloned-map', map_type='Same As', to_source_url=origin.uri,
from_source_url=origin.uri)
from_concept=local_from, to_concept=local_to, parent=local, mnemonic='cloned-map', map_type='Same As')
local.clone_mappings([mapping], user)
Mapping.objects.filter(parent=local, from_concept_code='From').update(
from_source_url=origin.uri, to_source_url=origin.uri)
mapping = Mapping.objects.get(parent=local, from_concept_code='From', id=F('versioned_object_id'))
self.assertEqual((mapping.to_concept_id, mapping.to_source_url), (local_to.id, origin.uri))
return origin, local, mapping
Expand Down
7 changes: 4 additions & 3 deletions core/integration_tests/tests_mappings.py
Original file line number Diff line number Diff line change
Expand Up @@ -707,15 +707,16 @@ def test_put_200_sort_weight(self):
self.assertEqual(self.mapping.sort_weight, 1.67983)

def test_put_200_keeps_the_local_link_of_a_cloned_mapping(self):
"""A cloned mapping keeps its target URL on the origin; sending that URL and code back keeps the local link."""
"""A mapping cloned before clones fixed source URLs keeps its target URL on the origin; sending that URL and
code back keeps the local link."""
origin = OrganizationSourceFactory(mnemonic='CloneOrigin')
ConceptFactory(parent=origin, mnemonic='To')
local_from = ConceptFactory(parent=self.source, mnemonic='From')
local_to = ConceptFactory(parent=self.source, mnemonic='To')
cloned = MappingFactory.build(
from_concept=ConceptFactory.build(mnemonic='From'), to_concept=ConceptFactory.build(mnemonic='To'),
parent=self.source, map_type=SAME_AS, to_source_url=origin.uri)
from_concept=local_from, to_concept=local_to, parent=self.source, map_type=SAME_AS)
self.source.clone_mappings([cloned], self.user)
Mapping.objects.filter(parent=self.source, from_concept_code='From').update(to_source_url=origin.uri)
cloned = Mapping.objects.get(parent=self.source, from_concept_code='From', id=F('versioned_object_id'))
self.assertEqual((cloned.from_concept_id, cloned.to_concept_id), (local_from.id, local_to.id))

Expand Down
35 changes: 35 additions & 0 deletions core/integration_tests/tests_sources.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
from django.core.cache import cache
from django.core.files.uploadedfile import SimpleUploadedFile
from django.db import transaction
from django.db.models import F
from django.test import override_settings, tag
from mock import patch, Mock, ANY, PropertyMock
from mock.mock import call
Expand Down Expand Up @@ -2717,6 +2718,40 @@ def test_post_success(self, bundle_clone_mock):
resource_budget=100, **parameters # no group: the preview per-call budget (ocl_online#230)
)

def test_post_retargets_answers_to_equivalents_and_keeps_targets_in_other_code_systems(self): # pylint: disable=too-many-locals
def named_concept(parent):
return ConceptFactory(parent=parent, names=[ConceptNameFactory.build(locale_preferred=True)])

source = OrganizationSourceFactory()
external = OrganizationSourceFactory()
target = self.clone_to_source
question, answer, external_concept = named_concept(source), named_concept(source), named_concept(external)
existing_answer, existing_external = named_concept(target), named_concept(target)
for existing, original in ((existing_answer, answer), (existing_external, external_concept)):
MappingFactory(parent=target, from_concept=existing, to_concept=original, map_type='SAME-AS')
MappingFactory(
parent=source, from_concept=question, to_concept=answer, map_type='Q-AND-A',
from_source=source, to_source=source, from_source_url=source.uri, to_source_url=source.uri)
MappingFactory(
parent=source, from_concept=question, to_concept=external_concept, map_type='SAME-AS',
from_source=source, to_source=external, from_source_url=source.uri, to_source_url=external.uri)

response = self.post_clone(
[question.uri], user=UserProfileFactory(is_staff=True),
parameters={'mapTypes': 'Q-AND-A,SAME-AS', 'equivalencyMapType': 'SAME-AS'})

self.assertEqual(response.data[question.uri]['status'], 200)
copies = Mapping.objects.filter(parent=target, from_source_url=target.uri, id=F('versioned_object_id'))
answer_copy = copies.get(map_type='Q-AND-A')
self.assertEqual(answer_copy.to_concept_id, existing_answer.id)
self.assertEqual((answer_copy.to_source_id, answer_copy.to_source_url), (target.id, target.uri))
external_copy = copies.get(map_type='SAME-AS', to_source_id=external.id)
self.assertEqual(external_copy.to_concept_id, external_concept.id)
self.assertEqual(external_copy.to_concept_code, external_concept.mnemonic)
self.assertEqual(external_copy.to_source_url, external.uri)
self.assertFalse(copies.filter(to_concept=existing_external).exists())

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Style (nit, non-blocking): add a second blank line after this test.

There's only one blank line between the end of this test and class SourceVersionsChangelogOutputViewTest, while PEP 8 (E302) and the rest of the file use two. Pylint, the only CI linter, doesn't flag it, so CI passes. But flake8 or an editor's formatter would, and a later formatting pass would add an unrelated hunk to this file.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in 2643c6b: two blank lines before class SourceVersionsChangelogOutputViewTest.



class SourceVersionsChangelogOutputViewTest(OCLAPITestCase):
def _build_changelog_output_fixture(self): # pylint: disable=too-many-locals
source = OrganizationSourceFactory()
Expand Down
25 changes: 25 additions & 0 deletions core/mappings/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -296,13 +296,38 @@ def clone(self, user=None, from_concept=None, to_concept=None):
)
if to_concept:
mapping.to_concept = to_concept
mapping.set_to_source_from_concept(
to_concept, get(to_concept, 'parent_id') != get(self, 'to_concept.parent_id'))
Comment thread
snyaggarwal marked this conversation as resolved.
Resolved
if from_concept:
mapping.from_concept = from_concept
mapping.set_from_source_from_concept(
from_concept, get(from_concept, 'parent_id') != get(self, 'from_concept.parent_id'))
if user:
mapping.created_by = mapping.updated_by = user

return mapping

def set_from_source_from_concept(self, concept, source_changed=False):
self.from_source_id, self.from_source_url, self.from_source_version = self.get_source_fields_from_concept(
concept, (self.from_source_id, self.from_source_url, self.from_source_version), source_changed)

def set_to_source_from_concept(self, concept, source_changed=False):
Comment thread
snyaggarwal marked this conversation as resolved.
Resolved
self.to_source_id, self.to_source_url, self.to_source_version = self.get_source_fields_from_concept(
concept, (self.to_source_id, self.to_source_url, self.to_source_version), source_changed)

@staticmethod
def get_source_fields_from_concept(concept, source_fields, source_changed=False):
"""(source id, URL, version) for concept's source. Unless the source changed, a recorded URL and version
are kept."""
source_id, url, version = source_fields
if not get(concept, 'parent_id'):
return source_id, url, version
if source_changed or not url:
url = concept.parent.uri
Comment thread
snyaggarwal marked this conversation as resolved.
Resolved
if source_changed:
version = None
return concept.parent_id, url, version

@classmethod
def build(cls, map_type, from_concept, to_concept, **kwargs):
return cls(
Expand Down
73 changes: 73 additions & 0 deletions core/mappings/tests/tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -324,6 +324,79 @@ def test_save_as_new_version(self):
persisted_mapping.version_url, persisted_mapping.uri
)

def test_repair_mapping_source_urls(self): # pylint: disable=too-many-locals,too-many-statements
origin = OrganizationSourceFactory(version=HEAD, canonical_url='https://example.org/origin')
source = OrganizationSourceFactory(version=HEAD, canonical_url='https://example.org/source')
other = OrganizationSourceFactory(version=HEAD, canonical_url='https://example.org/other')
concept = ConceptFactory(parent=source)
answer = ConceptFactory(parent=source)
other_concept = ConceptFactory(parent=other)
blank = OrganizationSourceFactory(version=HEAD, canonical_url='')
blank_concept = ConceptFactory(parent=blank)
blank.refresh_from_db()
self.assertEqual(blank.canonical_url, '')

def make(to_concept=other_concept, **urls):
mapping = MappingFactory(
parent=source, from_concept=concept, to_concept=to_concept, from_source=source,
to_source=to_concept.parent, from_source_url=source.uri, to_source_url=to_concept.parent.uri)
Mapping.objects.filter(id=mapping.id).update(**urls)
return mapping

def urls(mapping):
mapping.refresh_from_db()
return (mapping.from_source_url, mapping.from_source_version,
mapping.to_source_url, mapping.to_source_version)

stale_uri = make(
from_source_url=other.uri, from_source_version='v1', to_source_url=source.uri, to_source_version='v2')
stale_canonical = make(from_source_url=other.canonical_url, from_source_version='v1')
missing = make(from_source_url='', to_source_url=None)
Comment thread
snyaggarwal marked this conversation as resolved.
Resolved
own_canonical = make(from_source_url=source.canonical_url, from_source_version='v1')
unknown = make(from_source_url='/orgs/foo/sources/bar/')
retargeted_answer = make(to_concept=answer, from_source_url=origin.uri, to_source_url=origin.uri)
canonical_answer = make(to_concept=answer, from_source_url=origin.uri, to_source_url=origin.canonical_url)
replaced_target = make(to_concept=answer, from_source_url=origin.uri, to_source_url=other.uri)
blank_canonical = make(to_concept=blank_concept, to_source_url='')
self.assertEqual(stale_uri.versioned_object_id, stale_uri.id)
stale_version = Mapping.create_initial_version(stale_uri)
Mapping.objects.filter(id=stale_version.id).update(from_source_url=other.uri, from_source_version='v1')
repaired = (
stale_uri, stale_canonical, missing, retargeted_answer, canonical_answer, replaced_target, blank_canonical)
replaced_line = f' {replaced_target.uri} (from_source_url {origin.uri}, to_source_url {other.uri})\n'

before = {mapping.id: urls(mapping) for mapping in repaired}
out = StringIO()
call_command('repair_mapping_source_urls', '--dry-run', stdout=out)
to_output, from_output = out.getvalue().split('on from_source_url')
self.assertIn(f' {other.uri}: 2\n', to_output)
self.assertIn(f' {source.uri}: 2\n', to_output)
self.assertIn(replaced_line, to_output)
self.assertIn(f' {blank.uri}: 1\n', to_output)
self.assertIn(f' {source.uri}: 6\n', from_output)
self.assertEqual(before, {mapping.id: urls(mapping) for mapping in repaired})

call_command('repair_mapping_source_urls', stdout=StringIO())

self.assertEqual(urls(stale_uri), (source.uri, None, other.uri, None))
self.assertEqual(urls(stale_canonical), (source.uri, None, other.uri, None))
self.assertEqual(urls(missing), (source.uri, None, other.uri, None))
self.assertEqual(urls(own_canonical), (source.canonical_url, 'v1', other.uri, None))
self.assertEqual(urls(unknown), ('/orgs/foo/sources/bar/', None, other.uri, None))
self.assertEqual(urls(retargeted_answer), (source.uri, None, source.uri, None))
self.assertEqual(urls(canonical_answer), (source.uri, None, source.uri, None))
self.assertEqual(urls(blank_canonical), (source.uri, None, blank.uri, None))
self.assertEqual(urls(replaced_target), (source.uri, None, other.uri, None))
self.assertEqual(urls(stale_version)[:2], (other.uri, 'v1'))

out = StringIO()
call_command('repair_mapping_source_urls', stdout=out)
self.assertNotIn(f' {source.uri}:', out.getvalue())
self.assertNotIn(f' {other.uri}:', out.getvalue())
self.assertIn(f' {replaced_target.uri} (from_source_url {source.uri}, to_source_url {other.uri})\n',
out.getvalue())
self.assertEqual(urls(replaced_target), (source.uri, None, other.uri, None))

def test_save_as_new_version_and_repair_demote_every_version_flagged_latest(self):
source = OrganizationSourceFactory(version=HEAD)
mapping = MappingFactory(parent=source)
Expand Down
Loading
Loading