Skip to content
Merged
2 changes: 2 additions & 0 deletions core/common/mixins.py
Original file line number Diff line number Diff line change
Expand Up @@ -373,6 +373,8 @@ def get_csv(self, request, queryset=None):
if queryset is None:
queryset = self.get_queryset()[0:CSV_DEFAULT_LIMIT]
data = self.get_csv_rows(queryset) if hasattr(self, 'get_csv_rows') else queryset.values()
if hasattr(self, 'get_csv_field_serializers'):
kwargs['field_serializer_map'] = self.get_csv_field_serializers()
requester = request.user.id if request.user.is_authenticated else 'anonymous'
url = write_csv_to_s3(data, requester, **kwargs)

Expand Down
22 changes: 22 additions & 0 deletions core/common/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -1039,3 +1039,25 @@ def get_export_key_pattern(prefix):
(1.8.24.1 under 1.8.24.) don't match.
"""
return re.escape(prefix) + EXPORT_KEY_TIMESTAMP_PATTERN


def drop_key_deep(value, key):
"""A copy of a JSON value with `key` removed from every dict in it, at any depth."""
if isinstance(value, dict):
return {name: drop_key_deep(val, key) for name, val in value.items() if name != key}
if isinstance(value, list):
return [drop_key_deep(item, key) for item in value]
return value


def get_url_origin(url):
"""(scheme, host[:port]) of a URL with a host, lowercased, default port left out; None for a path or bad URL."""
try:
parsed = parse.urlparse(url)
host, port = parsed.hostname, parsed.port
except (ValueError, TypeError, AttributeError):
return None
if not host:
return None
scheme = parsed.scheme.lower()
return scheme, host if port in (None, {'http': 80, 'https': 443}.get(scheme)) else f'{host}:{port}'
116 changes: 116 additions & 0 deletions core/map_projects/migrations/0041_mapproject_credentials.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
# Generated by Django 5.2.17 on 2026-11-12 02:42

from django.db import migrations, models


def drop_key_deep(value, key):
if isinstance(value, dict):
return {name: drop_key_deep(val, key) for name, val in value.items() if name != key}
if isinstance(value, list):
return [drop_key_deep(item, key) for item in value]
return value


def find_values(value, key):
if isinstance(value, dict):
return [val for name, val in value.items() if name == key] + find_values(list(value.values()), key)
if isinstance(value, list):
return [hit for item in value for hit in find_values(item, key)]
return []


def _move_token(entry, tokens):
if not isinstance(entry, dict):
return entry
token = entry.get('token')
if isinstance(token, str) and token.strip():
url = entry.get('url') if isinstance(entry.get('url'), str) else ''
tokens[url] = token
Comment thread
snyaggarwal marked this conversation as resolved.
Resolved
return drop_key_deep(entry, 'token')


def find_token_conflicts(project_model):
"""(project id, URL) pairs where algorithms on one URL hold different tokens; only one could be kept."""
conflicts = []
for project in project_model.objects.only('id', 'algorithms').iterator(chunk_size=500):
tokens, urls = {}, []
for entry in project.algorithms or []:
url, token = (entry.get('url'), entry.get('token')) if isinstance(entry, dict) else (None, None)
if isinstance(url, str) and url and isinstance(token, str) and token.strip() and \
tokens.setdefault(url, token) != token and url not in urls:
urls.append(url)
conflicts += [(project.id, url) for url in urls]
return conflicts


# Every stored token moves to credentials, OCL API lookup tokens and ones found only in logs too, so they can
# still be found to revoke. Logs and run snapshots lose their tokens.
def move_tokens_to_credentials(apps, schema_editor): # pylint: disable=unused-argument
MapProject = apps.get_model('map_projects', 'MapProject')
conflicts = find_token_conflicts(MapProject)
if conflicts:
raise RuntimeError(
'Map project algorithms on the same URL have different tokens, so only one could be kept: ' +
', '.join(f'project {project_id} ({url})' for project_id, url in conflicts) +
'. Leave one token per URL (or remove the extra algorithm) on these projects, then migrate again.'
)
for project in MapProject.objects.only('id', 'algorithms', 'lookup_config', 'logs').iterator(chunk_size=500):
algorithm_tokens, lookup_tokens = {}, {}
algorithms = [_move_token(entry, algorithm_tokens) for entry in project.algorithms or []]
lookup_config = _move_token(project.lookup_config, lookup_tokens)
algorithm_tokens.pop('', None)
kept = set(algorithm_tokens.values()) | set(lookup_tokens.values())
logged_tokens = sorted({
token for token in find_values(project.logs, 'token')
if isinstance(token, str) and token.strip() and token not in kept
})
logs = drop_key_deep(project.logs, 'token')
if algorithm_tokens or lookup_tokens or logged_tokens or algorithms != (project.algorithms or []) or \
lookup_config != project.lookup_config or logs != project.logs:
credentials = {
key: tokens for key, tokens in (
('algorithms', algorithm_tokens), ('lookup', lookup_tokens), ('logs', logged_tokens)
) if tokens
}
MapProject.objects.filter(id=project.id).update(
algorithms=algorithms, lookup_config=lookup_config, logs=logs, credentials=credentials)

AutomatchRun = apps.get_model('map_projects', 'AutomatchRun')
for run in AutomatchRun.objects.only('id', 'config_snapshot').iterator(chunk_size=500):
snapshot = drop_key_deep(run.config_snapshot, 'token')
if snapshot != run.config_snapshot:
AutomatchRun.objects.filter(id=run.id).update(config_snapshot=snapshot)


# Tokens go back on their entries; tokens removed from logs and run snapshots are not restored.
def move_tokens_back(apps, schema_editor): # pylint: disable=unused-argument
MapProject = apps.get_model('map_projects', 'MapProject')
for project in MapProject.objects.exclude(credentials={}).exclude(credentials__isnull=True).only(
'id', 'algorithms', 'lookup_config', 'credentials').iterator(chunk_size=500):
algorithm_tokens = project.credentials.get('algorithms') or {}
lookup_tokens = project.credentials.get('lookup') or {}
algorithms = [
{**entry, 'token': algorithm_tokens[entry['url']]}
if isinstance(entry, dict) and entry.get('url') in algorithm_tokens else entry
for entry in project.algorithms or []
]
lookup_config = project.lookup_config
if isinstance(lookup_config, dict) and (lookup_config.get('url') or '') in lookup_tokens:
lookup_config = {**lookup_config, 'token': lookup_tokens[lookup_config.get('url') or '']}
MapProject.objects.filter(id=project.id).update(algorithms=algorithms, lookup_config=lookup_config)


class Migration(migrations.Migration):

dependencies = [
('map_projects', '0040_retire_public_edit_access'),
]

operations = [
migrations.AddField(
model_name='mapproject',
name='credentials',
field=models.JSONField(blank=True, default=dict, null=True),
),
migrations.RunPython(move_tokens_to_credentials, move_tokens_back),
]
156 changes: 147 additions & 9 deletions core/map_projects/models.py
Original file line number Diff line number Diff line change
@@ -1,17 +1,22 @@
import json
import re
from collections import Counter

from django.conf import settings
from django.contrib.postgres.fields import ArrayField
from django.core.exceptions import ValidationError
from django.db import models, IntegrityError
from django.db import models, IntegrityError, transaction
from django.db.models import Sum
from django.utils import timezone
from pydash import get

from core.common.constants import PERSIST_NEW_ERROR_MESSAGE, ACCESS_TYPE_CHOICES, ACCESS_TYPE_NONE
from core.common.models import BaseModel
from core.common.utils import get_export_service, generate_temp_version
from core.common.utils import get_export_service, generate_temp_version, drop_key_deep, get_url_origin, \
get_api_base_url


PERSIST_ERROR_MESSAGE = 'The map project could not be saved.'


def default_score_configuration():
Expand Down Expand Up @@ -54,6 +59,8 @@ class MapProject(BaseModel):
# No further migration needed at that point.
input_locales = ArrayField(models.CharField(max_length=10), null=True, blank=True, default=list)
use_lexical_variants = models.BooleanField(default=False)
# Endpoint tokens split out of algorithms and lookup_config; never serialized.
credentials = models.JSONField(default=dict, null=True, blank=True)

# Fields that define how a project matches —
# excluding identity, results, logs, and audit metadata.
Expand All @@ -64,9 +71,23 @@ class MapProject(BaseModel):
'prompt_output_locale', 'input_locales', 'use_lexical_variants'
]

# Never returned by the API, in any response or export.
WRITE_ONLY_FIELDS = ['credentials']

TOKEN_KEY = 'token'
HAS_TOKEN_KEY = 'has_token'
ALGORITHMS_CREDENTIALS_KEY = 'algorithms'
LOOKUP_CREDENTIALS_KEY = 'lookup'
# Tokens 0041 found only in logs, kept so they can be revoked; never used or flagged.
LOGGED_CREDENTIALS_KEY = 'logs'

class Meta:
db_table = 'map_projects'

@classmethod
def get_readable_field_names(cls):
return [field.attname for field in cls._meta.concrete_fields if field.name not in cls.WRITE_ONLY_FIELDS]

@property
def mnemonic(self):
return self.id
Expand Down Expand Up @@ -136,7 +157,13 @@ def update_input_file(self, input_file):
result = service.upload(key=key, file_content=input_file)
if result == 204:
self.input_file_name = file_name
self.save()
self.save(update_fields=['input_file_name'])

def save(self, *args, **kwargs):
# Tokens never stay in logs, whichever path saves them.
if self.logs:
self.logs = drop_key_deep(self.logs, self.TOKEN_KEY)
super().save(*args, **kwargs)

@classmethod
def persist_new(cls, instance, user, **kwargs):
Expand All @@ -153,8 +180,9 @@ def persist_new(cls, instance, user, **kwargs):
instance.save()
persisted = True
instance.update_input_file(input_file)
except IntegrityError as ex:
errors.update({'__all__': ex.args})
except IntegrityError:
# The database's message can quote the failing row, credentials included.
errors.update({'__all__': [PERSIST_ERROR_MESSAGE]})
except ValidationError as ex:
errors = get(ex, 'message_dict', {}) or get(ex, 'error_dict', {})
finally:
Expand All @@ -170,14 +198,21 @@ def persist_changes(cls, instance, user, **kwargs):
if input_file: # without a new file, keep the stored one
instance.input_file_name = input_file.name
instance.updated_by = user
instance.full_clean()
instance.save(**kwargs)
with transaction.atomic():
# Stored tokens are read under a row lock so a concurrent save can't drop a newly stored one.
instance.credentials = cls.objects.select_for_update().values_list(
'credentials', flat=True).get(id=instance.id)
instance.full_clean()
instance.save(**kwargs)
if input_file:
instance.update_input_file(input_file)
except ValidationError as ex:
errors.update(get(ex, 'message_dict', {}) or get(ex, 'error_dict', {}))
except IntegrityError as ex:
errors.update({'__all__': ex.args})
except IntegrityError:
# The database's message can quote the failing row, credentials included.
errors.update({'__all__': [PERSIST_ERROR_MESSAGE]})
except cls.DoesNotExist:
errors.update({'__all__': ['This map project no longer exists.']})

return errors

Expand Down Expand Up @@ -243,6 +278,109 @@ def clean(self):
self.clean_include_retired()
self.clean_encoder_model()
self.clean_matches()
self.clean_tokens()

def clean_tokens(self):
Comment thread
snyaggarwal marked this conversation as resolved.
Resolved
if self.lookup_config and not isinstance(self.lookup_config, dict):
raise ValidationError({'lookup_config': ['Must be an object.']})
if any(not isinstance(entry, dict) for entry in self.algorithms or []):
raise ValidationError({'algorithms': ['Each algorithm must be an object.']})
self.algorithms, self.lookup_config, self.credentials = self.split_tokens(
self.algorithms, self.lookup_config, self.credentials)

@staticmethod
def is_ocl_api_url(url):
"""
A lookup URL the Mapper sends to this API: none (the target repo), a path, or a URL on its origin.
As in the Mapper, only an http(s):// URL, in any case, is absolute; anything else is a path on this API.
"""
if not url:
return True
if not isinstance(url, str):
return False
if not re.match(r'https?://', url, re.IGNORECASE):
return True
return get_url_origin(url) == get_url_origin(get_api_base_url())

@staticmethod
def get_endpoint_url(entry):
url = entry.get('url')
return url if isinstance(url, str) else ''

@classmethod
def without_token(cls, entry):
entry = drop_key_deep(entry, cls.TOKEN_KEY)
entry.pop(cls.HAS_TOKEN_KEY, None)
return entry

@classmethod
def take_tokens(cls, entries, stored, field, accept_new=True):
"""
Tokens of the entries by URL. A missing token keeps the stored one for the same URL, so changing
the URL drops it; an empty one clears it. Entries sharing a URL share one token.
"""
new_tokens, kept_urls, cleared_urls = {}, set(), set()
for entry in entries:
url = cls.get_endpoint_url(entry)
token = entry.get(cls.TOKEN_KEY)
if cls.TOKEN_KEY not in entry or (token and not accept_new):
kept_urls.add(url)
elif not (isinstance(token, str) and token.strip()):
cleared_urls.add(url)
elif new_tokens.setdefault(url, token) != token:
raise ValidationError({field: ['Entries with the same URL must use the same token.']})
tokens = {url: stored[url] for url in kept_urls - cleared_urls if stored.get(url)}
return {**tokens, **new_tokens}

@classmethod
def split_tokens(cls, algorithms, lookup_config, credentials):
Comment thread
snyaggarwal marked this conversation as resolved.
Resolved
"""
Returns (algorithms, lookup_config, credentials) with tokens moved into credentials. No token is kept
for an algorithm without a URL, nor a new one for an OCL API lookup, which gets the user's own.
"""
credentials = credentials if isinstance(credentials, dict) else {}
algorithm_tokens, lookup_tokens = {}, {}
if isinstance(algorithms, list):
algorithm_tokens = cls.take_tokens(
[entry for entry in algorithms if isinstance(entry, dict) and cls.get_endpoint_url(entry)],
credentials.get(cls.ALGORITHMS_CREDENTIALS_KEY) or {}, 'algorithms'
)
algorithms = [cls.without_token(entry) if isinstance(entry, dict) else entry for entry in algorithms]
if isinstance(lookup_config, dict):
lookup_tokens = cls.take_tokens(
[lookup_config], credentials.get(cls.LOOKUP_CREDENTIALS_KEY) or {}, 'lookup_config',
accept_new=not cls.is_ocl_api_url(cls.get_endpoint_url(lookup_config))
)
lookup_config = cls.without_token(lookup_config)
return algorithms, lookup_config, {
key: tokens for key, tokens in (
(cls.ALGORITHMS_CREDENTIALS_KEY, algorithm_tokens), (cls.LOOKUP_CREDENTIALS_KEY, lookup_tokens),
(cls.LOGGED_CREDENTIALS_KEY, credentials.get(cls.LOGGED_CREDENTIALS_KEY)),
) if tokens
}

def flag_token(self, entry, credentials_key):
"""The entry without tokens; has_token where one is stored. OCL API lookups never show one."""
entry = self.without_token(entry)
url = self.get_endpoint_url(entry)
tokens = (self.credentials or {}).get(credentials_key) or {}
is_usable = bool(url) if credentials_key == self.ALGORITHMS_CREDENTIALS_KEY else not self.is_ocl_api_url(url)
return {**entry, self.HAS_TOKEN_KEY: True} if url in tokens and is_usable else entry

def get_algorithms_without_tokens(self, algorithms):
"""The API's view of algorithms: no tokens, and has_token where one is stored."""
if not isinstance(algorithms, list):
return drop_key_deep(algorithms, self.TOKEN_KEY)
return [
self.flag_token(entry, self.ALGORITHMS_CREDENTIALS_KEY) if isinstance(entry, dict) else
drop_key_deep(entry, self.TOKEN_KEY) for entry in algorithms
]

def get_lookup_config_without_token(self, lookup_config):
"""The API's view of lookup_config: no token, and has_token where one is stored."""
if not isinstance(lookup_config, dict):
return drop_key_deep(lookup_config, self.TOKEN_KEY)
return self.flag_token(lookup_config, self.LOOKUP_CREDENTIALS_KEY)

def clean_matches(self):
if self.matches:
Expand Down
Loading
Loading