Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
112 changes: 50 additions & 62 deletions plugins/callback/dashboard.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
__metaclass__ = type

import atexit
import fcntl
import hashlib
import json
import os
import re
Expand Down Expand Up @@ -51,7 +51,7 @@
- Shows the run on one fixed screen that is updated in place instead of scrolling, at least once a second. At the top the playbook, the number of hosts and the elapsed time, the current role and task, a progress bar of the play with the estimated remaining time, then counters of the results of the current task and of the whole run. The ok counter of the task is followed by the number of hosts still at work on the task and the number of hosts, for example C(ok 13/22/35).
- Below the title, where LFOps comes from - its version, or for a git checkout the branch, the commit and the number of uncommitted files, and the source path, shortened to the initials of its parent directories, when the checkout is mounted into an execution environment - and the ansible-core version. An C(LFOPS_EE_VERSION) environment variable is shown as the EE version.
- The progress bar counts the tasks of the play that are known when it starts, after C(--tags) and C(--skip-tags) are applied. A task included with C(include_tasks) or C(include_role) is one step, the bar stands still while the tasks it includes run. Tasks in C(rescue) sections and handlers are not counted. The bar counts tasks, not time, and starts over with every batch of C(serial).
- The estimated remaining time (ETA) adds up how long each step left took the last time it ran, an include with all its tasks, until the slowest host was done. The durations are kept in O(eta_cache_dir) for the next runs. A step that never ran before is assumed to take as long as the steps of this play so far did on average.
- The estimated remaining time (ETA) adds up how long each step left took the last time it ran, an include with all its tasks, until the slowest host was done. The durations are kept in O(eta_cache_dir) for the next runs, one file per step, so that runs in parallel, like several ansible-navigator sessions, do not wait for each other. A step that never ran before is assumed to take as long as the steps of this play so far did on average.
- Below, the hosts are listed, sorted by their most severe result (fatal, warning, changed, ignored, ok), then by name, with their counts of fatal (F), warning (W), changed (C) and ok (O) results and their last notable result (LAST WORST), or the last ok result for a host without one. Hosts with only skipped results are counted, not listed.
- At the end of the run, the screen is left and the details follow in the format of C(linuxfabrik.lfops.compact), but only for the tasks with changes, warnings or failures. Then the recap and the report of the distinct messages by severity.
- Without a terminal (log file, pipe, CI) or with C(--verbose), the plugin behaves exactly like C(linuxfabrik.lfops.compact).
Expand Down Expand Up @@ -115,10 +115,7 @@
# collection root: plugins/callback/dashboard.py -> ../../..
_COLLECTION_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))

# durations of steps that did not run for so long are dropped from the cache, e.g.
# those of a task whose line moved
_DURATIONS_MAX_AGE = 90 * 24 * 3600
_DURATIONS_FILE = 'durations.json'
_DURATION_SUFFIX = '.json'


def _git(root, *args):
Expand Down Expand Up @@ -297,71 +294,62 @@ def _private_dir(directory):
return True


def _read_durations(path):
"""The cache file: key -> {"seconds": float, "time": epoch}, {} if unusable."""
def _duration_file(directory, key):
"""The file of a step: the key hashed, since it contains slashes and spaces."""
name = hashlib.sha256(key.encode('utf-8')).hexdigest()
return os.path.join(directory, name + _DURATION_SUFFIX)


def _read_duration(path, key):
"""The seconds in the file of the step `key`, None if unusable."""
try:
with open(path, encoding='utf-8') as f:
data = json.load(f)
except (OSError, ValueError):
return {}
if not isinstance(data, dict):
return {}
return {
key: value
for key, value in data.items()
if isinstance(value, dict)
and isinstance(value.get('seconds'), (int, float))
and isinstance(value.get('time'), (int, float))
}
return None
if not isinstance(data, dict) or data.get('key') != key:
return None
seconds = data.get('seconds')
return seconds if isinstance(seconds, (int, float)) else None


def load_durations(directory):
"""The durations of the steps of earlier runs, key -> seconds."""
if not directory or not _private_dir(directory):
def load_durations(directory, keys):
"""The durations of the steps `keys` from earlier runs, key -> seconds, for
those that are known.
"""
if not directory or not keys or not _private_dir(directory):
return {}
durations = _read_durations(os.path.join(directory, _DURATIONS_FILE))
return {key: value['seconds'] for key, value in durations.items()}


def save_durations(directory, measured, now=None):
"""Merge the `measured` durations (key -> seconds) into the cache file, the
last one wins. Entries not updated for _DURATIONS_MAX_AGE are dropped. Runs in
parallel take turns with a lock and see each other's results.
durations = {}
for key in keys:
seconds = _read_duration(_duration_file(directory, key), key)
if seconds is not None:
durations[key] = seconds
return durations


def save_durations(directory, measured):
"""Write the `measured` durations (key -> seconds), one file per step, so
that runs in parallel, e.g. several ansible-navigator sessions, need no lock:
each file is replaced as a whole, the last run wins.
"""
if not directory or not measured or not _private_dir(directory):
return
now = time.time() if now is None else now
path = os.path.join(directory, _DURATIONS_FILE)
try:
lock = os.open(
os.path.join(directory, 'durations.lock'),
os.O_WRONLY | os.O_CREAT | os.O_NOFOLLOW,
0o600,
)
except OSError:
return
try:
fcntl.flock(lock, fcntl.LOCK_EX)
durations = {
key: value
for key, value in _read_durations(path).items()
if now - value['time'] < _DURATIONS_MAX_AGE
}
for key, seconds in measured.items():
durations[key] = {'seconds': round(seconds, 1), 'time': int(now)}
for key, seconds in measured.items():
# a new file moved into place, so that a run that aborts while writing
# leaves the old one
fd, tmp = tempfile.mkstemp(dir=directory, prefix='.durations.')
# leaves the old one, and another run never reads half a file
try:
fd, tmp = tempfile.mkstemp(dir=directory, prefix='.tmp.')
except OSError:
return
try:
with os.fdopen(fd, 'w', encoding='utf-8') as f:
json.dump(durations, f, indent=0, sort_keys=True)
os.replace(tmp, path)
json.dump({'key': key, 'seconds': round(seconds, 1)}, f)
os.replace(tmp, _duration_file(directory, key))
except OSError:
os.unlink(tmp)
except OSError:
pass
finally:
os.close(lock)
try:
os.unlink(tmp)
except OSError:
pass


def progress_bar(step, total, width, color=True, remaining=None):
Expand Down Expand Up @@ -446,9 +434,9 @@ def __init__(self):
self._step = 0
self._play_start = time.monotonic()
self._step_start = self._play_start
# key -> seconds: the durations of the steps from earlier runs (None until
# the first play starts), and those measured in this run, see _learn()
self._durations = None
# key -> seconds: the durations of the steps of the running play from
# earlier runs, and those measured in this run, see _learn()
self._durations = {}
self._measured = {}
# where the plugin, and with it LFOps, comes from, see lfops_origin()
self._origin = None
Expand Down Expand Up @@ -951,12 +939,12 @@ def v2_playbook_on_play_start(self, play):
self._play_start = time.monotonic()
self._step_start = self._play_start
if self._tui:
if self._durations is None:
self._durations = load_durations(self._eta_cache_dir())
try:
self._steps, self._step_keys = play_steps(play)
except Exception: # only for the progress bar, which is then left out
pass
# per play, to see what runs in parallel learned meanwhile
self._durations = load_durations(self._eta_cache_dir(), self._step_keys)
# the play line is part of the screen header, not of the details, and was
# printed above the screen already
self._capture = []
Expand Down
74 changes: 50 additions & 24 deletions tests/unit/plugins/callback/test_dashboard.py
Original file line number Diff line number Diff line change
Expand Up @@ -294,37 +294,40 @@ def tearDown(self):
shutil.rmtree(self.tmp, ignore_errors=True)

def test_saved_and_loaded(self):
_module.save_durations(self.directory, {'roles/a/tasks/main.yml:3': 12.34})
key = 'roles/a/tasks/main.yml:3 [l.l.a > l.l.b]'
_module.save_durations(self.directory, {key: 12.34})
self.assertEqual(
_module.load_durations(self.directory),
{'roles/a/tasks/main.yml:3': 12.3},
_module.load_durations(self.directory, [key, 'unknown:1']), {key: 12.3}
)
# directory and parent private
# one file per step, directory and parent private
self.assertEqual(len(os.listdir(self.directory)), 1)
for path in (self.directory, os.path.dirname(self.directory)):
self.assertEqual(os.stat(path).st_mode & 0o777, 0o700)

def test_last_duration_wins_and_others_are_kept(self):
_module.save_durations(self.directory, {'a:1': 50.0, 'b:1': 5.0})
_module.save_durations(self.directory, {'a:1': 10.0})
self.assertEqual(
_module.load_durations(self.directory), {'a:1': 10.0, 'b:1': 5.0}
_module.load_durations(self.directory, ['a:1', 'b:1']),
{'a:1': 10.0, 'b:1': 5.0},
)

def test_old_entries_are_dropped(self):
now = 1_800_000_000
_module.save_durations(self.directory, {'old:1': 5.0}, now=now)
_module.save_durations(
self.directory, {'new:1': 7.0}, now=now + _module._DURATIONS_MAX_AGE + 1
)
self.assertEqual(_module.load_durations(self.directory), {'new:1': 7.0})
# no temporary files left
self.assertEqual(len(os.listdir(self.directory)), 2)

def test_broken_file_is_ignored_and_replaced(self):
os.makedirs(self.directory)
with open(os.path.join(self.directory, 'durations.json'), 'w') as f:
f.write('{"a:1": {"seconds": 1')
self.assertEqual(_module.load_durations(self.directory), {})
_module.save_durations(self.directory, {'b:1': 2.0})
self.assertEqual(_module.load_durations(self.directory), {'b:1': 2.0})
path = _module._duration_file(self.directory, 'a:1')
with open(path, 'w') as f:
f.write('{"key": "a:1", "seconds": 1')
self.assertEqual(_module.load_durations(self.directory, ['a:1']), {})
_module.save_durations(self.directory, {'a:1': 2.0})
self.assertEqual(_module.load_durations(self.directory, ['a:1']), {'a:1': 2.0})

def test_file_of_another_key_is_ignored(self):
os.makedirs(self.directory)
with open(_module._duration_file(self.directory, 'a:1'), 'w') as f:
f.write('{"key": "b:1", "seconds": 1}')
self.assertEqual(_module.load_durations(self.directory, ['a:1']), {})

def test_symlinked_directory_is_not_used(self):
target = os.path.join(self.tmp, 'elsewhere')
Expand All @@ -333,11 +336,32 @@ def test_symlinked_directory_is_not_used(self):
os.symlink(target, self.directory)
_module.save_durations(self.directory, {'a:1': 1.0})
self.assertEqual(os.listdir(target), [])
self.assertEqual(_module.load_durations(self.directory), {})
self.assertEqual(_module.load_durations(self.directory, ['a:1']), {})

def test_disabled_with_an_empty_value(self):
_module.save_durations('', {'a:1': 1.0})
self.assertEqual(_module.load_durations(''), {})
self.assertEqual(_module.load_durations('', ['a:1']), {})

def test_parallel_runs_keep_each_others_durations(self):
# many processes write at the same time, none loses its steps
keys = [[f'p{p}:{i}' for i in range(50)] + ['shared:1'] for p in range(8)]
pids = []
for p, own in enumerate(keys):
pid = os.fork()
if pid == 0:
try:
_module.save_durations(
self.directory, {key: float(p) for key in own}
)
finally:
os._exit(0)
pids.append(pid)
for pid in pids:
os.waitpid(pid, 0)
everything = sorted({key for own in keys for key in own})
loaded = _module.load_durations(self.directory, everything)
self.assertEqual(sorted(loaded), everything)
self.assertEqual(len(os.listdir(self.directory)), len(everything))

def test_task_key_relative_to_the_collection(self):
class Role:
Expand Down Expand Up @@ -417,16 +441,18 @@ def test_durations_learned_and_used_by_the_next_run(self):
workdir=tmp,
)
self.assertEqual(returncode, 0, raw)
with open(os.path.join(tmp, 'eta', 'durations.json')) as f:
durations = json.load(f)
durations = {}
for name in os.listdir(os.path.join(tmp, 'eta')):
with open(os.path.join(tmp, 'eta', name)) as f:
data = json.load(f)
durations[data['key']] = data['seconds']
finally:
shutil.rmtree(tmp, ignore_errors=True)
# the three steps by file and line of the playbook, outside the collection
self.assertEqual(len(durations), 3, durations)
self.assertTrue(all('playbook.yml:' in key for key in durations), durations)
# the include with its tasks, until the slowest host was done
slowest = max(durations.values(), key=lambda value: value['seconds'])
self.assertGreaterEqual(slowest['seconds'], 3, durations)
self.assertGreaterEqual(max(durations.values()), 3, durations)
# in the second run, the ETA is known from the first step on and counts
# the sleep of the include
screen = raw[raw.index(_SCREEN_ON) : raw.rindex(_SCREEN_OFF)]
Expand Down
Loading