Repository navigation
OpenConceptLab/ocl_online#201 | Map project endpoint tokens are write-only #991
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
Show all changes
8 commits
Select commit
Hold shift + click to select a range
41a830b
OpenConceptLab/ocl_online#201 | Map project endpoint tokens are write…
snyaggarwal 6a4a6c9
OpenConceptLab/ocl_online#201 | Map project CSV exports leave out cre…
snyaggarwal a4767f2
OpenConceptLab/ocl_online#201 | Map project tokens: nested ones strip…
snyaggarwal 21488e0
OpenConceptLab/ocl_online#201 | Map project save of a deleted project…
snyaggarwal 0252372
OpenConceptLab/ocl_online#201 | Migration 0041 stops, naming each pro…
snyaggarwal bc4a4f5
OpenConceptLab/ocl_online#201 | Map project PATCH splits tokens; logs…
snyaggarwal 933cb39
OpenConceptLab/ocl_online#201 | Map project logs and input-file saves…
snyaggarwal 2e72f41
OpenConceptLab/ocl_online#201 | Verbose map project CSV strips tokens…
snyaggarwal File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
116 changes: 116 additions & 0 deletions
116
core/map_projects/migrations/0041_mapproject_credentials.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,116 @@ | ||
| # Generated by Django 5.2.17 on 2026-11-12 02:42 | ||
|
|
||
| from django.db import migrations, models | ||
|
|
||
|
|
||
| def drop_key_deep(value, key): | ||
| if isinstance(value, dict): | ||
| return {name: drop_key_deep(val, key) for name, val in value.items() if name != key} | ||
| if isinstance(value, list): | ||
| return [drop_key_deep(item, key) for item in value] | ||
| return value | ||
|
|
||
|
|
||
| def find_values(value, key): | ||
| if isinstance(value, dict): | ||
| return [val for name, val in value.items() if name == key] + find_values(list(value.values()), key) | ||
| if isinstance(value, list): | ||
| return [hit for item in value for hit in find_values(item, key)] | ||
| return [] | ||
|
|
||
|
|
||
| def _move_token(entry, tokens): | ||
| if not isinstance(entry, dict): | ||
| return entry | ||
| token = entry.get('token') | ||
| if isinstance(token, str) and token.strip(): | ||
| url = entry.get('url') if isinstance(entry.get('url'), str) else '' | ||
| tokens[url] = token | ||
| return drop_key_deep(entry, 'token') | ||
|
|
||
|
|
||
| def find_token_conflicts(project_model): | ||
| """(project id, URL) pairs where algorithms on one URL hold different tokens; only one could be kept.""" | ||
| conflicts = [] | ||
| for project in project_model.objects.only('id', 'algorithms').iterator(chunk_size=500): | ||
| tokens, urls = {}, [] | ||
| for entry in project.algorithms or []: | ||
| url, token = (entry.get('url'), entry.get('token')) if isinstance(entry, dict) else (None, None) | ||
| if isinstance(url, str) and url and isinstance(token, str) and token.strip() and \ | ||
| tokens.setdefault(url, token) != token and url not in urls: | ||
| urls.append(url) | ||
| conflicts += [(project.id, url) for url in urls] | ||
| return conflicts | ||
|
|
||
|
|
||
| # Every stored token moves to credentials, OCL API lookup tokens and ones found only in logs too, so they can | ||
| # still be found to revoke. Logs and run snapshots lose their tokens. | ||
| def move_tokens_to_credentials(apps, schema_editor): # pylint: disable=unused-argument | ||
| MapProject = apps.get_model('map_projects', 'MapProject') | ||
| conflicts = find_token_conflicts(MapProject) | ||
| if conflicts: | ||
| raise RuntimeError( | ||
| 'Map project algorithms on the same URL have different tokens, so only one could be kept: ' + | ||
| ', '.join(f'project {project_id} ({url})' for project_id, url in conflicts) + | ||
| '. Leave one token per URL (or remove the extra algorithm) on these projects, then migrate again.' | ||
| ) | ||
| for project in MapProject.objects.only('id', 'algorithms', 'lookup_config', 'logs').iterator(chunk_size=500): | ||
| algorithm_tokens, lookup_tokens = {}, {} | ||
| algorithms = [_move_token(entry, algorithm_tokens) for entry in project.algorithms or []] | ||
| lookup_config = _move_token(project.lookup_config, lookup_tokens) | ||
| algorithm_tokens.pop('', None) | ||
| kept = set(algorithm_tokens.values()) | set(lookup_tokens.values()) | ||
| logged_tokens = sorted({ | ||
| token for token in find_values(project.logs, 'token') | ||
| if isinstance(token, str) and token.strip() and token not in kept | ||
| }) | ||
| logs = drop_key_deep(project.logs, 'token') | ||
| if algorithm_tokens or lookup_tokens or logged_tokens or algorithms != (project.algorithms or []) or \ | ||
| lookup_config != project.lookup_config or logs != project.logs: | ||
| credentials = { | ||
| key: tokens for key, tokens in ( | ||
| ('algorithms', algorithm_tokens), ('lookup', lookup_tokens), ('logs', logged_tokens) | ||
| ) if tokens | ||
| } | ||
| MapProject.objects.filter(id=project.id).update( | ||
| algorithms=algorithms, lookup_config=lookup_config, logs=logs, credentials=credentials) | ||
|
|
||
| AutomatchRun = apps.get_model('map_projects', 'AutomatchRun') | ||
| for run in AutomatchRun.objects.only('id', 'config_snapshot').iterator(chunk_size=500): | ||
| snapshot = drop_key_deep(run.config_snapshot, 'token') | ||
| if snapshot != run.config_snapshot: | ||
| AutomatchRun.objects.filter(id=run.id).update(config_snapshot=snapshot) | ||
|
|
||
|
|
||
| # Tokens go back on their entries; tokens removed from logs and run snapshots are not restored. | ||
| def move_tokens_back(apps, schema_editor): # pylint: disable=unused-argument | ||
| MapProject = apps.get_model('map_projects', 'MapProject') | ||
| for project in MapProject.objects.exclude(credentials={}).exclude(credentials__isnull=True).only( | ||
| 'id', 'algorithms', 'lookup_config', 'credentials').iterator(chunk_size=500): | ||
| algorithm_tokens = project.credentials.get('algorithms') or {} | ||
| lookup_tokens = project.credentials.get('lookup') or {} | ||
| algorithms = [ | ||
| {**entry, 'token': algorithm_tokens[entry['url']]} | ||
| if isinstance(entry, dict) and entry.get('url') in algorithm_tokens else entry | ||
| for entry in project.algorithms or [] | ||
| ] | ||
| lookup_config = project.lookup_config | ||
| if isinstance(lookup_config, dict) and (lookup_config.get('url') or '') in lookup_tokens: | ||
| lookup_config = {**lookup_config, 'token': lookup_tokens[lookup_config.get('url') or '']} | ||
| MapProject.objects.filter(id=project.id).update(algorithms=algorithms, lookup_config=lookup_config) | ||
|
|
||
|
|
||
| class Migration(migrations.Migration): | ||
|
|
||
| dependencies = [ | ||
| ('map_projects', '0040_retire_public_edit_access'), | ||
| ] | ||
|
|
||
| operations = [ | ||
| migrations.AddField( | ||
| model_name='mapproject', | ||
| name='credentials', | ||
| field=models.JSONField(blank=True, default=dict, null=True), | ||
| ), | ||
| migrations.RunPython(move_tokens_to_credentials, move_tokens_back), | ||
| ] | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.