diff --git a/core/common/mixins.py b/core/common/mixins.py index 0ba3221df..ee2c4b930 100644 --- a/core/common/mixins.py +++ b/core/common/mixins.py @@ -373,6 +373,8 @@ def get_csv(self, request, queryset=None): if queryset is None: queryset = self.get_queryset()[0:CSV_DEFAULT_LIMIT] data = self.get_csv_rows(queryset) if hasattr(self, 'get_csv_rows') else queryset.values() + if hasattr(self, 'get_csv_field_serializers'): + kwargs['field_serializer_map'] = self.get_csv_field_serializers() requester = request.user.id if request.user.is_authenticated else 'anonymous' url = write_csv_to_s3(data, requester, **kwargs) diff --git a/core/common/utils.py b/core/common/utils.py index 791155fda..80fe8bfa1 100644 --- a/core/common/utils.py +++ b/core/common/utils.py @@ -1039,3 +1039,25 @@ def get_export_key_pattern(prefix): (1.8.24.1 under 1.8.24.) don't match. """ return re.escape(prefix) + EXPORT_KEY_TIMESTAMP_PATTERN + + +def drop_key_deep(value, key): + """A copy of a JSON value with `key` removed from every dict in it, at any depth.""" + if isinstance(value, dict): + return {name: drop_key_deep(val, key) for name, val in value.items() if name != key} + if isinstance(value, list): + return [drop_key_deep(item, key) for item in value] + return value + + +def get_url_origin(url): + """(scheme, host[:port]) of a URL with a host, lowercased, default port left out; None for a path or bad URL.""" + try: + parsed = parse.urlparse(url) + host, port = parsed.hostname, parsed.port + except (ValueError, TypeError, AttributeError): + return None + if not host: + return None + scheme = parsed.scheme.lower() + return scheme, host if port in (None, {'http': 80, 'https': 443}.get(scheme)) else f'{host}:{port}' diff --git a/core/map_projects/migrations/0041_mapproject_credentials.py b/core/map_projects/migrations/0041_mapproject_credentials.py new file mode 100644 index 000000000..64f32d562 --- /dev/null +++ b/core/map_projects/migrations/0041_mapproject_credentials.py @@ -0,0 +1,116 @@ +# Generated by Django 5.2.17 on 2026-11-12 02:42 + +from django.db import migrations, models + + +def drop_key_deep(value, key): + if isinstance(value, dict): + return {name: drop_key_deep(val, key) for name, val in value.items() if name != key} + if isinstance(value, list): + return [drop_key_deep(item, key) for item in value] + return value + + +def find_values(value, key): + if isinstance(value, dict): + return [val for name, val in value.items() if name == key] + find_values(list(value.values()), key) + if isinstance(value, list): + return [hit for item in value for hit in find_values(item, key)] + return [] + + +def _move_token(entry, tokens): + if not isinstance(entry, dict): + return entry + token = entry.get('token') + if isinstance(token, str) and token.strip(): + url = entry.get('url') if isinstance(entry.get('url'), str) else '' + tokens[url] = token + return drop_key_deep(entry, 'token') + + +def find_token_conflicts(project_model): + """(project id, URL) pairs where algorithms on one URL hold different tokens; only one could be kept.""" + conflicts = [] + for project in project_model.objects.only('id', 'algorithms').iterator(chunk_size=500): + tokens, urls = {}, [] + for entry in project.algorithms or []: + url, token = (entry.get('url'), entry.get('token')) if isinstance(entry, dict) else (None, None) + if isinstance(url, str) and url and isinstance(token, str) and token.strip() and \ + tokens.setdefault(url, token) != token and url not in urls: + urls.append(url) + conflicts += [(project.id, url) for url in urls] + return conflicts + + +# Every stored token moves to credentials, OCL API lookup tokens and ones found only in logs too, so they can +# still be found to revoke. Logs and run snapshots lose their tokens. +def move_tokens_to_credentials(apps, schema_editor): # pylint: disable=unused-argument + MapProject = apps.get_model('map_projects', 'MapProject') + conflicts = find_token_conflicts(MapProject) + if conflicts: + raise RuntimeError( + 'Map project algorithms on the same URL have different tokens, so only one could be kept: ' + + ', '.join(f'project {project_id} ({url})' for project_id, url in conflicts) + + '. Leave one token per URL (or remove the extra algorithm) on these projects, then migrate again.' + ) + for project in MapProject.objects.only('id', 'algorithms', 'lookup_config', 'logs').iterator(chunk_size=500): + algorithm_tokens, lookup_tokens = {}, {} + algorithms = [_move_token(entry, algorithm_tokens) for entry in project.algorithms or []] + lookup_config = _move_token(project.lookup_config, lookup_tokens) + algorithm_tokens.pop('', None) + kept = set(algorithm_tokens.values()) | set(lookup_tokens.values()) + logged_tokens = sorted({ + token for token in find_values(project.logs, 'token') + if isinstance(token, str) and token.strip() and token not in kept + }) + logs = drop_key_deep(project.logs, 'token') + if algorithm_tokens or lookup_tokens or logged_tokens or algorithms != (project.algorithms or []) or \ + lookup_config != project.lookup_config or logs != project.logs: + credentials = { + key: tokens for key, tokens in ( + ('algorithms', algorithm_tokens), ('lookup', lookup_tokens), ('logs', logged_tokens) + ) if tokens + } + MapProject.objects.filter(id=project.id).update( + algorithms=algorithms, lookup_config=lookup_config, logs=logs, credentials=credentials) + + AutomatchRun = apps.get_model('map_projects', 'AutomatchRun') + for run in AutomatchRun.objects.only('id', 'config_snapshot').iterator(chunk_size=500): + snapshot = drop_key_deep(run.config_snapshot, 'token') + if snapshot != run.config_snapshot: + AutomatchRun.objects.filter(id=run.id).update(config_snapshot=snapshot) + + +# Tokens go back on their entries; tokens removed from logs and run snapshots are not restored. +def move_tokens_back(apps, schema_editor): # pylint: disable=unused-argument + MapProject = apps.get_model('map_projects', 'MapProject') + for project in MapProject.objects.exclude(credentials={}).exclude(credentials__isnull=True).only( + 'id', 'algorithms', 'lookup_config', 'credentials').iterator(chunk_size=500): + algorithm_tokens = project.credentials.get('algorithms') or {} + lookup_tokens = project.credentials.get('lookup') or {} + algorithms = [ + {**entry, 'token': algorithm_tokens[entry['url']]} + if isinstance(entry, dict) and entry.get('url') in algorithm_tokens else entry + for entry in project.algorithms or [] + ] + lookup_config = project.lookup_config + if isinstance(lookup_config, dict) and (lookup_config.get('url') or '') in lookup_tokens: + lookup_config = {**lookup_config, 'token': lookup_tokens[lookup_config.get('url') or '']} + MapProject.objects.filter(id=project.id).update(algorithms=algorithms, lookup_config=lookup_config) + + +class Migration(migrations.Migration): + + dependencies = [ + ('map_projects', '0040_retire_public_edit_access'), + ] + + operations = [ + migrations.AddField( + model_name='mapproject', + name='credentials', + field=models.JSONField(blank=True, default=dict, null=True), + ), + migrations.RunPython(move_tokens_to_credentials, move_tokens_back), + ] diff --git a/core/map_projects/models.py b/core/map_projects/models.py index 84ad34eb5..c22bb1583 100644 --- a/core/map_projects/models.py +++ b/core/map_projects/models.py @@ -1,17 +1,22 @@ import json +import re from collections import Counter from django.conf import settings from django.contrib.postgres.fields import ArrayField from django.core.exceptions import ValidationError -from django.db import models, IntegrityError +from django.db import models, IntegrityError, transaction from django.db.models import Sum from django.utils import timezone from pydash import get from core.common.constants import PERSIST_NEW_ERROR_MESSAGE, ACCESS_TYPE_CHOICES, ACCESS_TYPE_NONE from core.common.models import BaseModel -from core.common.utils import get_export_service, generate_temp_version +from core.common.utils import get_export_service, generate_temp_version, drop_key_deep, get_url_origin, \ + get_api_base_url + + +PERSIST_ERROR_MESSAGE = 'The map project could not be saved.' def default_score_configuration(): @@ -54,6 +59,8 @@ class MapProject(BaseModel): # No further migration needed at that point. input_locales = ArrayField(models.CharField(max_length=10), null=True, blank=True, default=list) use_lexical_variants = models.BooleanField(default=False) + # Endpoint tokens split out of algorithms and lookup_config; never serialized. + credentials = models.JSONField(default=dict, null=True, blank=True) # Fields that define how a project matches — # excluding identity, results, logs, and audit metadata. @@ -64,9 +71,23 @@ class MapProject(BaseModel): 'prompt_output_locale', 'input_locales', 'use_lexical_variants' ] + # Never returned by the API, in any response or export. + WRITE_ONLY_FIELDS = ['credentials'] + + TOKEN_KEY = 'token' + HAS_TOKEN_KEY = 'has_token' + ALGORITHMS_CREDENTIALS_KEY = 'algorithms' + LOOKUP_CREDENTIALS_KEY = 'lookup' + # Tokens 0041 found only in logs, kept so they can be revoked; never used or flagged. + LOGGED_CREDENTIALS_KEY = 'logs' + class Meta: db_table = 'map_projects' + @classmethod + def get_readable_field_names(cls): + return [field.attname for field in cls._meta.concrete_fields if field.name not in cls.WRITE_ONLY_FIELDS] + @property def mnemonic(self): return self.id @@ -136,7 +157,13 @@ def update_input_file(self, input_file): result = service.upload(key=key, file_content=input_file) if result == 204: self.input_file_name = file_name - self.save() + self.save(update_fields=['input_file_name']) + + def save(self, *args, **kwargs): + # Tokens never stay in logs, whichever path saves them. + if self.logs: + self.logs = drop_key_deep(self.logs, self.TOKEN_KEY) + super().save(*args, **kwargs) @classmethod def persist_new(cls, instance, user, **kwargs): @@ -153,8 +180,9 @@ def persist_new(cls, instance, user, **kwargs): instance.save() persisted = True instance.update_input_file(input_file) - except IntegrityError as ex: - errors.update({'__all__': ex.args}) + except IntegrityError: + # The database's message can quote the failing row, credentials included. + errors.update({'__all__': [PERSIST_ERROR_MESSAGE]}) except ValidationError as ex: errors = get(ex, 'message_dict', {}) or get(ex, 'error_dict', {}) finally: @@ -170,14 +198,21 @@ def persist_changes(cls, instance, user, **kwargs): if input_file: # without a new file, keep the stored one instance.input_file_name = input_file.name instance.updated_by = user - instance.full_clean() - instance.save(**kwargs) + with transaction.atomic(): + # Stored tokens are read under a row lock so a concurrent save can't drop a newly stored one. + instance.credentials = cls.objects.select_for_update().values_list( + 'credentials', flat=True).get(id=instance.id) + instance.full_clean() + instance.save(**kwargs) if input_file: instance.update_input_file(input_file) except ValidationError as ex: errors.update(get(ex, 'message_dict', {}) or get(ex, 'error_dict', {})) - except IntegrityError as ex: - errors.update({'__all__': ex.args}) + except IntegrityError: + # The database's message can quote the failing row, credentials included. + errors.update({'__all__': [PERSIST_ERROR_MESSAGE]}) + except cls.DoesNotExist: + errors.update({'__all__': ['This map project no longer exists.']}) return errors @@ -243,6 +278,109 @@ def clean(self): self.clean_include_retired() self.clean_encoder_model() self.clean_matches() + self.clean_tokens() + + def clean_tokens(self): + if self.lookup_config and not isinstance(self.lookup_config, dict): + raise ValidationError({'lookup_config': ['Must be an object.']}) + if any(not isinstance(entry, dict) for entry in self.algorithms or []): + raise ValidationError({'algorithms': ['Each algorithm must be an object.']}) + self.algorithms, self.lookup_config, self.credentials = self.split_tokens( + self.algorithms, self.lookup_config, self.credentials) + + @staticmethod + def is_ocl_api_url(url): + """ + A lookup URL the Mapper sends to this API: none (the target repo), a path, or a URL on its origin. + As in the Mapper, only an http(s):// URL, in any case, is absolute; anything else is a path on this API. + """ + if not url: + return True + if not isinstance(url, str): + return False + if not re.match(r'https?://', url, re.IGNORECASE): + return True + return get_url_origin(url) == get_url_origin(get_api_base_url()) + + @staticmethod + def get_endpoint_url(entry): + url = entry.get('url') + return url if isinstance(url, str) else '' + + @classmethod + def without_token(cls, entry): + entry = drop_key_deep(entry, cls.TOKEN_KEY) + entry.pop(cls.HAS_TOKEN_KEY, None) + return entry + + @classmethod + def take_tokens(cls, entries, stored, field, accept_new=True): + """ + Tokens of the entries by URL. A missing token keeps the stored one for the same URL, so changing + the URL drops it; an empty one clears it. Entries sharing a URL share one token. + """ + new_tokens, kept_urls, cleared_urls = {}, set(), set() + for entry in entries: + url = cls.get_endpoint_url(entry) + token = entry.get(cls.TOKEN_KEY) + if cls.TOKEN_KEY not in entry or (token and not accept_new): + kept_urls.add(url) + elif not (isinstance(token, str) and token.strip()): + cleared_urls.add(url) + elif new_tokens.setdefault(url, token) != token: + raise ValidationError({field: ['Entries with the same URL must use the same token.']}) + tokens = {url: stored[url] for url in kept_urls - cleared_urls if stored.get(url)} + return {**tokens, **new_tokens} + + @classmethod + def split_tokens(cls, algorithms, lookup_config, credentials): + """ + Returns (algorithms, lookup_config, credentials) with tokens moved into credentials. No token is kept + for an algorithm without a URL, nor a new one for an OCL API lookup, which gets the user's own. + """ + credentials = credentials if isinstance(credentials, dict) else {} + algorithm_tokens, lookup_tokens = {}, {} + if isinstance(algorithms, list): + algorithm_tokens = cls.take_tokens( + [entry for entry in algorithms if isinstance(entry, dict) and cls.get_endpoint_url(entry)], + credentials.get(cls.ALGORITHMS_CREDENTIALS_KEY) or {}, 'algorithms' + ) + algorithms = [cls.without_token(entry) if isinstance(entry, dict) else entry for entry in algorithms] + if isinstance(lookup_config, dict): + lookup_tokens = cls.take_tokens( + [lookup_config], credentials.get(cls.LOOKUP_CREDENTIALS_KEY) or {}, 'lookup_config', + accept_new=not cls.is_ocl_api_url(cls.get_endpoint_url(lookup_config)) + ) + lookup_config = cls.without_token(lookup_config) + return algorithms, lookup_config, { + key: tokens for key, tokens in ( + (cls.ALGORITHMS_CREDENTIALS_KEY, algorithm_tokens), (cls.LOOKUP_CREDENTIALS_KEY, lookup_tokens), + (cls.LOGGED_CREDENTIALS_KEY, credentials.get(cls.LOGGED_CREDENTIALS_KEY)), + ) if tokens + } + + def flag_token(self, entry, credentials_key): + """The entry without tokens; has_token where one is stored. OCL API lookups never show one.""" + entry = self.without_token(entry) + url = self.get_endpoint_url(entry) + tokens = (self.credentials or {}).get(credentials_key) or {} + is_usable = bool(url) if credentials_key == self.ALGORITHMS_CREDENTIALS_KEY else not self.is_ocl_api_url(url) + return {**entry, self.HAS_TOKEN_KEY: True} if url in tokens and is_usable else entry + + def get_algorithms_without_tokens(self, algorithms): + """The API's view of algorithms: no tokens, and has_token where one is stored.""" + if not isinstance(algorithms, list): + return drop_key_deep(algorithms, self.TOKEN_KEY) + return [ + self.flag_token(entry, self.ALGORITHMS_CREDENTIALS_KEY) if isinstance(entry, dict) else + drop_key_deep(entry, self.TOKEN_KEY) for entry in algorithms + ] + + def get_lookup_config_without_token(self, lookup_config): + """The API's view of lookup_config: no token, and has_token where one is stored.""" + if not isinstance(lookup_config, dict): + return drop_key_deep(lookup_config, self.TOKEN_KEY) + return self.flag_token(lookup_config, self.LOOKUP_CREDENTIALS_KEY) def clean_matches(self): if self.matches: diff --git a/core/map_projects/serializers.py b/core/map_projects/serializers.py index 6d8860e5b..e06a164d0 100644 --- a/core/map_projects/serializers.py +++ b/core/map_projects/serializers.py @@ -6,11 +6,24 @@ from core.common.constants import ACCESS_TYPE_NONE, INCLUDE_SUMMARY, INCLUDE_LOGS from core.common.serializers import PublicAccessField -from core.common.utils import get_truthy_values +from core.common.utils import get_truthy_values, drop_key_deep from core.map_projects.models import MapProject, AutomatchRun -class MapProjectCreateUpdateSerializer(serializers.ModelSerializer): +class TokenFlagsMixin: + """Endpoint tokens are write-only: a response shows has_token in their place.""" + def to_representation(self, instance): + data = super().to_representation(instance) + if 'algorithms' in data: + data['algorithms'] = instance.get_algorithms_without_tokens(data['algorithms']) + if 'lookup_config' in data: + data['lookup_config'] = instance.get_lookup_config_without_token(data['lookup_config']) + if 'logs' in data: + data['logs'] = drop_key_deep(data['logs'], MapProject.TOKEN_KEY) + return data + + +class MapProjectCreateUpdateSerializer(TokenFlagsMixin, serializers.ModelSerializer): id = IntegerField(read_only=True) file = FileField(write_only=True, required=False) user_id = IntegerField(write_only=True, required=False) @@ -121,7 +134,7 @@ class Meta: ] -class MapProjectConfigurationsSerializer(serializers.ModelSerializer): +class MapProjectConfigurationsSerializer(TokenFlagsMixin, serializers.ModelSerializer): class Meta: model = MapProject fields = [ @@ -173,8 +186,15 @@ class Meta: model = MapProject fields = MapProjectSerializer.Meta.fields + ['file_url', 'matches', 'columns'] + def update(self, instance, validated_data): + """A PATCH saves as a PUT does, so tokens are split out and model validation runs.""" + for attr, value in validated_data.items(): + setattr(instance, attr, value) + self._errors.update(MapProject.persist_changes(instance, self.context['request'].user)) + return instance + -class MapProjectLogsSerializer(serializers.ModelSerializer): +class MapProjectLogsSerializer(TokenFlagsMixin, serializers.ModelSerializer): class Meta: model = MapProject fields = ['id', 'logs', 'url'] @@ -208,6 +228,13 @@ class Meta: 'config_snapshot', 'client_user_agent', 'client_ip', 'created_at', 'updated_at', ] + def to_representation(self, instance): + # Snapshots are stripped on save; this also covers any written outside the API. + data = super().to_representation(instance) + if 'config_snapshot' in data: + data['config_snapshot'] = drop_key_deep(data['config_snapshot'], MapProject.TOKEN_KEY) + return data + class AutomatchRunCreateSerializer(serializers.ModelSerializer): id = IntegerField(read_only=True) @@ -223,6 +250,10 @@ class Meta: # later via PATCH, so completion_status is intentionally not settable here. fields = ['id', 'url', 'intended_rows', 'config_snapshot', 'trigger_source', 'parent_run'] + @staticmethod + def validate_config_snapshot(value): + return drop_key_deep(value, MapProject.TOKEN_KEY) + def validate(self, attrs): # A retry must link to a run within the SAME project — both a data-integrity # rule and an authorization guard (a caller must not chain a run onto another diff --git a/core/map_projects/tests/test_tokens.py b/core/map_projects/tests/test_tokens.py new file mode 100644 index 000000000..6f9d8e385 --- /dev/null +++ b/core/map_projects/tests/test_tokens.py @@ -0,0 +1,474 @@ +import importlib + +from mock import patch, Mock + +from django.apps import apps +from django.core.files.uploadedfile import SimpleUploadedFile +from django.db import IntegrityError +from django.test import override_settings, SimpleTestCase + +from core.common.utils import drop_key_deep, get_url_origin +from core.map_projects.models import MapProject, AutomatchRun +from core.map_projects.tests.factories import MapProjectFactory, AutomatchRunFactory +from core.map_projects.tests.tests import MapProjectAbstractViewTest + +CUSTOM_URL = 'https://match.example.org/match/' +LOOKUP_URL = 'https://terms.example.org/orgs/X/sources/Y/' + + +def find_token_keys(value, path=''): + if isinstance(value, dict): + return [f'{path}.{key}' for key in value if key == 'token'] + [ + hit for key, val in value.items() for hit in find_token_keys(val, f'{path}.{key}')] + if isinstance(value, list): + return [hit for index, item in enumerate(value) for hit in find_token_keys(item, f'{path}[{index}]')] + return [] + + +@override_settings(API_BASE_URL='https://api.example.org') +class MapProjectTokensTest(MapProjectAbstractViewTest): + def setUp(self): + super().setUp() + from django.contrib.auth.models import Permission + self.user.user_permissions.add(Permission.objects.get(codename='mapper_custom_algorithms')) + self.project = MapProjectFactory(organization=self.org, name='Tokens') + self.url = f'/orgs/CIEL/map-projects/{self.project.id}/' + self.auth = {'HTTP_AUTHORIZATION': 'Token ' + self.user.get_token()} + + def put(self, data): + return self.client.put(self.url, {'name': 'Tokens', **data}, format='json', **self.auth) + + def put_tokens(self): + return self.put({ + 'name': 'Tokens', + 'algorithms': [ + {'id': 'custom-1', 'type': 'custom', 'url': CUSTOM_URL, 'token': 'algo-key'}, + {'id': 'ocl-semantic', 'type': 'ocl-semantic'}, + ], + 'lookup_config': {'url': LOOKUP_URL, 'token': 'lookup-key'}, + }) + + def test_put_stores_tokens_and_returns_has_token(self): + response = self.put_tokens() + + self.assertEqual(response.status_code, 200) + self.assertEqual(find_token_keys(response.data), []) + self.assertEqual( + response.data['algorithms'], + [{'id': 'custom-1', 'type': 'custom', 'url': CUSTOM_URL, 'has_token': True}, + {'id': 'ocl-semantic', 'type': 'ocl-semantic'}] + ) + self.assertEqual(response.data['lookup_config'], {'url': LOOKUP_URL, 'has_token': True}) + self.project.refresh_from_db() + self.assertEqual(find_token_keys(self.project.algorithms), []) + self.assertEqual(find_token_keys(self.project.lookup_config), []) + self.assertEqual( + self.project.credentials, {'algorithms': {CUSTOM_URL: 'algo-key'}, 'lookup': {LOOKUP_URL: 'lookup-key'}}) + + def test_no_response_returns_tokens(self): + self.put_tokens() + AutomatchRunFactory( + map_project=self.project, config_snapshot={'algorithms': [{'url': CUSTOM_URL, 'token': 'algo-key'}]}) + run = self.project.auto_match_runs.first() + + for url in [ + self.url, + f'{self.url}configurations/', + '/orgs/CIEL/map-projects/?verbose=true', + '/orgs/CIEL/map-projects/', + f'/users/{self.user.username}/orgs/map-projects/?verbose=true', + f'/auto-match-runs/{run.id}/', + ]: + response = self.client.get(url, **self.auth) + self.assertEqual(response.status_code, 200, url) + self.assertEqual(find_token_keys(response.data), [], url) + self.assertNotIn('algo-key', str(response.data), url) + self.assertNotIn('lookup-key', str(response.data), url) + self.assertNotIn('credentials', str(response.data), url) + + def test_put_without_token_keeps_it(self): + self.put_tokens() + + response = self.put({ + 'algorithms': [{'id': 'custom-1', 'type': 'custom', 'url': CUSTOM_URL, 'has_token': True}], + 'lookup_config': {'url': LOOKUP_URL, 'has_token': True}, + }) + + self.assertEqual(response.status_code, 200) + self.project.refresh_from_db() + self.assertEqual( + self.project.credentials, {'algorithms': {CUSTOM_URL: 'algo-key'}, 'lookup': {LOOKUP_URL: 'lookup-key'}}) + + def test_put_without_algorithms_keeps_tokens(self): + self.put_tokens() + + self.assertEqual(self.put({'name': 'Renamed'}).status_code, 200) + + self.project.refresh_from_db() + self.assertEqual(self.project.credentials['algorithms'], {CUSTOM_URL: 'algo-key'}) + + def test_put_empty_token_clears_it(self): + self.put_tokens() + + self.put({ + 'algorithms': [{'id': 'custom-1', 'type': 'custom', 'url': CUSTOM_URL, 'token': ''}], + 'lookup_config': {'url': LOOKUP_URL, 'token': None}, + }) + + self.project.refresh_from_db() + self.assertEqual(self.project.credentials, {}) + + def test_changed_url_drops_token(self): + self.put_tokens() + + response = self.put({ + 'algorithms': [{'id': 'custom-1', 'type': 'custom', 'url': 'https://elsewhere.example.com/'}], + 'lookup_config': {'url': 'https://elsewhere.example.com/orgs/X/'}, + }) + + self.assertNotIn('has_token', response.data['algorithms'][0]) + self.assertNotIn('has_token', response.data['lookup_config']) + self.project.refresh_from_db() + self.assertEqual(self.project.credentials, {}) + + def test_removed_algorithm_drops_token(self): + self.put_tokens() + + self.put({'algorithms': [{'id': 'ocl-semantic', 'type': 'ocl-semantic'}]}) + + self.project.refresh_from_db() + self.assertNotIn('algorithms', self.project.credentials) + + def test_ocl_api_lookup_token_is_not_stored(self): + for lookup_url in ['/orgs/CIEL/sources/CIEL/', 'https://api.example.org/orgs/CIEL/', '']: + response = self.put({'lookup_config': {'url': lookup_url, 'token': 'ocl-token'}}) + + self.assertEqual(response.status_code, 200, lookup_url) + self.assertNotIn('has_token', response.data['lookup_config'], lookup_url) + self.project.refresh_from_db() + self.assertNotIn('lookup', self.project.credentials, lookup_url) + self.assertEqual(find_token_keys(self.project.lookup_config), [], lookup_url) + + def test_algorithm_without_url_keeps_no_token(self): + self.put({'algorithms': [{'id': 'ocl-semantic', 'token': 'stray'}]}) + + self.project.refresh_from_db() + self.assertEqual(self.project.algorithms, [{'id': 'ocl-semantic'}]) + self.assertEqual(self.project.credentials, {}) + + def test_nested_tokens_are_not_stored(self): + self.put({ + 'algorithms': [{'id': 'custom-1', 'type': 'custom', 'url': CUSTOM_URL, 'auth': {'token': 'nested'}}], + 'lookup_config': {'url': LOOKUP_URL, 'headers': [{'token': 'nested'}]}, + }) + + self.project.refresh_from_db() + self.assertEqual(self.project.algorithms, [{'id': 'custom-1', 'type': 'custom', 'url': CUSTOM_URL, 'auth': {}}]) + self.assertEqual(self.project.lookup_config, {'url': LOOKUP_URL, 'headers': [{}]}) + self.assertEqual(self.project.credentials, {}) + + def test_migrated_ocl_lookup_token_is_kept_but_not_shown(self): + MapProject.objects.filter(id=self.project.id).update( + lookup_config={'url': '/orgs/CIEL/'}, credentials={'lookup': {'/orgs/CIEL/': 'old-ocl-token'}}) + + self.assertNotIn('has_token', self.client.get(self.url, **self.auth).data['lookup_config']) + response = self.put({'lookup_config': {'url': '/orgs/CIEL/', 'token': 'new-ocl-token'}}) + + self.assertNotIn('has_token', response.data['lookup_config']) + self.project.refresh_from_db() + self.assertEqual(self.project.credentials, {'lookup': {'/orgs/CIEL/': 'old-ocl-token'}}) + + def test_algorithms_sharing_a_url_share_one_token(self): + self.put_tokens() + + response = self.put({'algorithms': [ + {'id': 'custom-1', 'url': CUSTOM_URL, 'token': 'k1'}, + {'id': 'custom-2', 'url': CUSTOM_URL, 'token': 'k2'}, + ]}) + self.assertEqual(response.status_code, 400) + self.assertIn('algorithms', response.data) + + response = self.put({'algorithms': [ + {'id': 'custom-1', 'url': CUSTOM_URL, 'has_token': True}, + {'id': 'custom-2', 'url': CUSTOM_URL, 'token': 'k2'}, + ]}) + self.assertEqual(response.status_code, 200) + self.assertEqual([algo.get('has_token') for algo in response.data['algorithms']], [True, True]) + self.project.refresh_from_db() + self.assertEqual(self.project.credentials['algorithms'], {CUSTOM_URL: 'k2'}) + + def test_stale_save_keeps_newer_token(self): + self.put_tokens() + stale = MapProject.objects.get(id=self.project.id) + MapProject.objects.filter(id=self.project.id).update( + credentials={'algorithms': {CUSTOM_URL: 'newer'}, 'lookup': {LOOKUP_URL: 'lookup-key'}}) + stale.name = 'Renamed' + + self.assertEqual(MapProject.persist_changes(stale, self.user), {}) + + self.project.refresh_from_db() + self.assertEqual(self.project.name, 'Renamed') + self.assertEqual(self.project.credentials['algorithms'], {CUSTOM_URL: 'newer'}) + + def test_save_of_deleted_project_is_an_error(self): + stale = MapProject.objects.get(id=self.project.id) + MapProject.objects.filter(id=self.project.id).delete() + + self.assertEqual( + MapProject.persist_changes(stale, self.user), {'__all__': ['This map project no longer exists.']}) + + @patch('core.common.mixins.write_csv_to_s3', return_value='https://csv-url') + def test_patch_splits_tokens(self, write_csv_mock): + response = self.client.patch(self.url, { + 'algorithms': [{'id': 'custom-1', 'type': 'custom', 'url': CUSTOM_URL, 'token': 'algo-key'}], + 'lookup_config': {'url': LOOKUP_URL, 'token': 'lookup-key'}, + }, format='json', **self.auth) + + self.assertEqual(response.status_code, 200) + self.assertNotIn('algo-key', str(response.data)) + self.project.refresh_from_db() + self.assertEqual(find_token_keys(self.project.algorithms), []) + self.assertEqual(find_token_keys(self.project.lookup_config), []) + self.assertEqual( + self.project.credentials, {'algorithms': {CUSTOM_URL: 'algo-key'}, 'lookup': {LOOKUP_URL: 'lookup-key'}}) + self.client.get('/orgs/CIEL/map-projects/?csv=true&verbose=true', **self.auth) + rows = str(list(write_csv_mock.call_args.args[0])) + self.assertNotIn('algo-key', rows) + self.assertNotIn('lookup-key', rows) + + def test_logs_never_hold_tokens(self): + logs = {'project_logs': [{'action': 'Created', 'extras': {'project': { + 'algorithms': [{'url': CUSTOM_URL, 'token': 'algo-key'}], 'lookup_config': {'token': 'lookup-key'}}}}]} + + self.client.post(f'{self.url}logs/', {'logs': logs}, format='json', **self.auth) + + self.project.refresh_from_db() + self.assertEqual(find_token_keys(self.project.logs), []) + self.assertEqual(self.project.logs['project_logs'][0]['extras']['project']['algorithms'], [{'url': CUSTOM_URL}]) + + MapProject.objects.filter(id=self.project.id).update(logs=logs) + for url in [f'{self.url}logs/', f'{self.url}?includeSummary=true']: + response = self.client.get(url, **self.auth) + self.assertEqual(response.status_code, 200, url) + self.assertIn('logs', response.data, url) + self.assertEqual(find_token_keys(response.data), [], url) + + def test_logged_tokens_are_kept_on_save(self): + MapProject.objects.filter(id=self.project.id).update(credentials={'logs': ['old-key']}) + + self.put_tokens() + + self.project.refresh_from_db() + self.assertEqual(self.project.credentials['logs'], ['old-key']) + self.assertNotIn('old-key', str(self.client.get(f'{self.url}?includeSummary=true', **self.auth).data)) + + def test_logs_save_writes_only_log_fields(self): + original_save = MapProject.save + with patch.object(MapProject, 'save', autospec=True, side_effect=original_save) as save_mock: + self.client.post(f'{self.url}logs/', {'logs': {'project_logs': [{'action': 'Opened'}]}}, format='json', + **self.auth) + + self.assertEqual(save_mock.call_args.kwargs, {'update_fields': ['logs', 'updated_by', 'updated_at']}) + self.project.refresh_from_db() + self.assertEqual(self.project.logs, {'project_logs': [{'action': 'Opened'}]}) + + @patch('core.map_projects.models.get_export_service') + def test_input_file_save_keeps_newer_tokens(self, get_export_service_mock): + get_export_service_mock.return_value.upload.return_value = 204 + self.put_tokens() + stale = MapProject.objects.get(id=self.project.id) + MapProject.objects.filter(id=self.project.id).update(credentials={'algorithms': {CUSTOM_URL: 'newer'}}) + + stale.update_input_file(SimpleUploadedFile('new.csv', b'a', 'text/csv')) + + self.project.refresh_from_db() + self.assertEqual(self.project.input_file_name, 'new.csv') + self.assertEqual(self.project.credentials, {'algorithms': {CUSTOM_URL: 'newer'}}) + + def test_settings_must_be_objects(self): + response = self.put({'lookup_config': [{'url': LOOKUP_URL, 'token': 'lookup-key'}]}) + self.assertEqual(response.status_code, 400) + self.assertIn('lookup_config', response.data) + + response = self.put({'algorithms': [[{'url': CUSTOM_URL, 'token': 'algo-key'}]]}) + self.assertEqual(response.status_code, 400) + self.assertIn('algorithms', response.data) + + self.project.refresh_from_db() + self.assertNotIn('lookup-key', str(self.project.lookup_config)) + self.assertNotIn('algo-key', str(self.project.algorithms)) + + @patch('core.common.mixins.write_csv_to_s3', return_value='https://csv-url') + def test_stored_non_object_settings_are_stripped_on_read(self, write_csv_mock): + MapProject.objects.filter(id=self.project.id).update( + lookup_config=[{'url': LOOKUP_URL, 'token': 'lookup-key'}], + algorithms=[[{'url': CUSTOM_URL, 'token': 'algo-key'}]], + logs={'project_logs': [{'token': 'logged-key'}]}, + ) + + for url in [self.url, f'{self.url}configurations/', '/orgs/CIEL/map-projects/?verbose=true']: + response = self.client.get(url, **self.auth) + self.assertEqual(response.status_code, 200, url) + self.assertEqual(find_token_keys(response.data), [], url) + + self.client.get('/orgs/CIEL/map-projects/?csv=true&verbose=true', **self.auth) + rows = list(write_csv_mock.call_args.args[0]) + serializers = write_csv_mock.call_args.kwargs['field_serializer_map'] + written = str([{key: serializers.get(key, str)(val) for key, val in row.items()} for row in rows]) + self.assertIn(CUSTOM_URL, written) + for token in ['algo-key', 'lookup-key', 'logged-key']: + self.assertNotIn(token, written) + + def test_integrity_error_is_generic(self): + project = MapProject.objects.get(id=self.project.id) + project.full_clean = Mock() + project.save = Mock(side_effect=IntegrityError('DETAIL: Failing row contains (..., {"algorithms": "k"})')) + + self.assertEqual( + MapProject.persist_changes(project, self.user), {'__all__': ['The map project could not be saved.']}) + + def test_credentials_cannot_be_written_directly(self): + self.put({'credentials': {'algorithms': {CUSTOM_URL: 'injected'}}}) + + self.project.refresh_from_db() + self.assertEqual(self.project.credentials, {}) + + @patch('core.common.mixins.write_csv_to_s3', return_value='https://csv-url') + def test_csv_exports_no_tokens(self, write_csv_mock): + self.put_tokens() + + for url in [ + '/orgs/CIEL/map-projects/?csv=true', + '/orgs/CIEL/map-projects/?csv=true&verbose=true', + f'/users/{self.user.username}/orgs/map-projects/?csv=true&verbose=true', + ]: + write_csv_mock.reset_mock() + response = self.client.get(url, **self.auth) + + self.assertEqual(response.status_code, 200, url) + write_csv_mock.assert_called_once() + rows = list(write_csv_mock.call_args.args[0]) + self.assertEqual([row['id'] for row in rows], [self.project.id], url) + self.assertNotIn('credentials', rows[0], url) + self.assertNotIn('algo-key', str(rows), url) + self.assertNotIn('lookup-key', str(rows), url) + + def test_run_snapshot_drops_tokens(self): + response = self.client.post( + f'{self.url}auto-match-runs/', + data={ + 'intended_rows': 1, 'trigger_source': 'ui-auto-match', + 'config_snapshot': { + 'algorithms': [{'url': CUSTOM_URL, 'token': 'algo-key'}], + 'lookup_config': {'url': LOOKUP_URL, 'token': 'lookup-key'}, + }, + }, + format='json', **self.auth, + ) + + self.assertEqual(response.status_code, 201) + self.assertEqual(find_token_keys(response.data), []) + self.assertEqual(find_token_keys(AutomatchRun.objects.get(id=response.data['id']).config_snapshot), []) + + +class MapProjectTokensMigrationTest(MapProjectAbstractViewTest): + def setUp(self): + super().setUp() + self.migration = importlib.import_module('core.map_projects.migrations.0041_mapproject_credentials') + + def test_moves_tokens_to_credentials(self): + project = MapProjectFactory(organization=self.org) + MapProject.objects.filter(id=project.id).update( + algorithms=[ + {'type': 'custom', 'url': CUSTOM_URL, 'token': 'algo-key', 'auth': {'token': 'nested'}}, + {'type': 'ocl-semantic', 'token': 'stray'}, + ], + lookup_config={'url': '/orgs/CIEL/', 'token': 'ocl-token'}, + logs={'project_logs': [{'extras': {'project': {'algorithms': [ + {'url': CUSTOM_URL, 'token': 'algo-key'}, {'url': CUSTOM_URL, 'token': 'logged-only'}]}}}]}, + ) + run = AutomatchRunFactory(map_project=project, config_snapshot={'a': [{'token': 'algo-key', 'x': 1}]}) + + self.migration.move_tokens_to_credentials(apps, None) + + project.refresh_from_db() + run.refresh_from_db() + self.assertEqual( + project.algorithms, [{'type': 'custom', 'url': CUSTOM_URL, 'auth': {}}, {'type': 'ocl-semantic'}]) + self.assertEqual(project.lookup_config, {'url': '/orgs/CIEL/'}) + self.assertEqual(project.credentials, { + 'algorithms': {CUSTOM_URL: 'algo-key'}, 'lookup': {'/orgs/CIEL/': 'ocl-token'}, 'logs': ['logged-only']}) + self.assertEqual(find_token_keys(project.logs), []) + self.assertEqual(run.config_snapshot, {'a': [{'x': 1}]}) + + self.migration.move_tokens_back(apps, None) + + project.refresh_from_db() + self.assertEqual(project.algorithms[0]['token'], 'algo-key') + self.assertEqual(project.lookup_config['token'], 'ocl-token') + + def test_stops_on_different_tokens_for_one_url(self): + project = MapProjectFactory(organization=self.org) + algorithms = [ + {'id': 'a', 'url': CUSTOM_URL, 'token': 'k1'}, + {'id': 'b', 'url': CUSTOM_URL, 'token': 'k2'}, + {'id': 'c', 'url': CUSTOM_URL, 'token': 'k2'}, + ] + MapProject.objects.filter(id=project.id).update(algorithms=algorithms) + + with self.assertRaisesMessage(RuntimeError, f'project {project.id} ({CUSTOM_URL}).'): + self.migration.move_tokens_to_credentials(apps, None) + + project.refresh_from_db() + self.assertEqual(project.algorithms, algorithms) + self.assertEqual(project.credentials, {}) + + def test_same_token_on_one_url_migrates(self): + project = MapProjectFactory(organization=self.org) + MapProject.objects.filter(id=project.id).update(algorithms=[ + {'id': 'a', 'url': CUSTOM_URL, 'token': 'k1'}, {'id': 'b', 'url': CUSTOM_URL, 'token': 'k1'}]) + + self.migration.move_tokens_to_credentials(apps, None) + + project.refresh_from_db() + self.assertEqual(project.credentials, {'algorithms': {CUSTOM_URL: 'k1'}}) + + def test_leaves_projects_without_tokens_alone(self): + project = MapProjectFactory(organization=self.org, algorithms=[{'type': 'ocl-semantic'}]) + updated_at = MapProject.objects.get(id=project.id).updated_at + + self.migration.move_tokens_to_credentials(apps, None) + + project.refresh_from_db() + self.assertEqual(project.updated_at, updated_at) + self.assertEqual(project.credentials, {}) + + +class DropKeyDeepTest(SimpleTestCase): + def test_drops_key_at_any_depth(self): + self.assertEqual( + drop_key_deep({'token': 1, 'a': [{'token': 2, 'b': {'token': 3, 'c': 4}}]}, 'token'), + {'a': [{'b': {'c': 4}}]} + ) + self.assertEqual(drop_key_deep('token', 'token'), 'token') + + +class GetUrlOriginTest(SimpleTestCase): + def test_origin(self): + self.assertEqual(get_url_origin('HTTPS://API.Example.org:443/orgs/'), ('https', 'api.example.org')) + self.assertEqual(get_url_origin('http://api.example.org:8000/'), ('http', 'api.example.org:8000')) + self.assertEqual(get_url_origin('//api.example.org/orgs/'), ('', 'api.example.org')) + self.assertIsNone(get_url_origin('/orgs/CIEL/')) + self.assertIsNone(get_url_origin('http://[bad')) + self.assertIsNone(get_url_origin(None)) + + +@override_settings(API_BASE_URL='https://api.example.org') +class IsOCLAPIURLTest(SimpleTestCase): + def test_is_ocl_api_url(self): + for url in ['', None, '/orgs/CIEL/', '//terms.example.org/', 'https://api.example.org/orgs/', + 'HTTPS://api.example.org:443/orgs/']: + self.assertTrue(MapProject.is_ocl_api_url(url), url) + for url in [ + LOOKUP_URL, 'HTTPS://terms.example.org/', 'http://api.example.org/', 'https://api.example.org:8443/']: + self.assertFalse(MapProject.is_ocl_api_url(url), url) diff --git a/core/map_projects/views.py b/core/map_projects/views.py index 0f2e6686b..f8ccd3c50 100644 --- a/core/map_projects/views.py +++ b/core/map_projects/views.py @@ -15,6 +15,7 @@ from core.common.mixins import ListWithHeadersMixin, ConceptDictionaryCreateMixin from core.common.permissions import CanEditConceptDictionary, CanCreateOrgMapProjects, \ CanUseCustomMapperAlgorithms, HasMapProjectParentOwnership, HasMapProjectCapacity +from core.common.utils import drop_key_deep from core.common.views import BaseAPIView from core.map_projects.models import MapProject, AutomatchRun from core.map_projects.serializers import MapProjectCreateUpdateSerializer, \ @@ -71,7 +72,7 @@ def get_queryset(self): if self.request.method == 'GET' and not self.is_verbose(): queryset = queryset.defer( 'matches', 'columns', 'candidates', 'analysis', 'logs', 'extras', 'algorithms', 'filters', - 'lookup_config', 'input_locales' + 'lookup_config', 'input_locales', 'credentials' ) return self.filter_queryset_by_public_access(self.filter_queryset_by_owner(queryset)) @@ -82,11 +83,19 @@ def filter_queryset_by_public_access(self, queryset): Q(name__icontains=search_query) | Q(id_text__icontains=search_query)) return super().filter_queryset_by_public_access(queryset) + @staticmethod + def csv_value_without_tokens(value): + return str(drop_key_deep(value, MapProject.TOKEN_KEY)) + + def get_csv_field_serializers(self): + """Verbose rows are raw columns, so settings stored in any shape lose their tokens as they're written.""" + return {field: self.csv_value_without_tokens for field in ('algorithms', 'lookup_config', 'logs')} + def get_csv_rows(self, queryset): - """A row per listed project, read again by id as the list may be a union; verbose keeps every column.""" + """A row per listed project, read again by id as the list may be a union; verbose has every readable column.""" rows = MapProject.objects.filter(id__in=[project.id for project in queryset]).order_by('id') if self.is_verbose(): - return rows.values() + return rows.values(*MapProject.get_readable_field_names()) return rows.values( 'id', 'name', 'public_access', 'is_active', 'created_at', 'updated_at', 'created_by__username', 'updated_by__username', 'organization__mnemonic', 'user__username') @@ -193,7 +202,8 @@ def create(self, request, *args, **kwargs): if new_logs: map_project.logs = new_logs map_project.updated_by = request.user - map_project.save() + # Only the log fields, so an older copy of the project's settings and tokens isn't written back. + map_project.save(update_fields=['logs', 'updated_by', 'updated_at']) return Response(status.HTTP_204_NO_CONTENT)