diff --git a/src/main/python/systemds/scuro/drsearch/node_executor.py b/src/main/python/systemds/scuro/drsearch/node_executor.py index 7a35135b4f8..3e9f6cd00be 100644 --- a/src/main/python/systemds/scuro/drsearch/node_executor.py +++ b/src/main/python/systemds/scuro/drsearch/node_executor.py @@ -75,15 +75,40 @@ MemoryMeasurement, cleanup_gpu, cpu_memory_budget_bytes, - estimate_modality_bytes, is_cuda_oom, measure_memory_during, + split_modality_bytes, ) from systemds.scuro.utils.static_variables import DEBUG _MAX_NODE_RETRIES = int(os.environ.get("SCURO_MAX_NODE_RETRIES", "3")) _FUSED_CONTEXT_BYTES = 512 * 2**20 _FUSED_WAIT_S = float(os.environ.get("SCURO_FUSED_WAIT_S", "30")) +_DISPATCH_DIAG_DIR = os.environ.get("SCURO_DISPATCH_DIAG") + + +def _diag(kind: str, record: dict) -> None: + if not _DISPATCH_DIAG_DIR: + return + import json + + os.makedirs(_DISPATCH_DIAG_DIR, exist_ok=True) + record = {"t": time.perf_counter(), "pid": os.getpid(), **record} + path = os.path.join(_DISPATCH_DIAG_DIR, f"{kind}_{os.getpid()}.jsonl") + with open(path, "a") as f: + f.write(json.dumps(record, default=str) + "\n") + + +def _diag_width(data) -> int: + """Per-instance feature count of a task input, -1 if it is ragged.""" + try: + first = data[0] + shape = getattr(first, "shape", None) + if shape is None: + shape = (len(first),) if hasattr(first, "__len__") else () + return int(math.prod(shape)) if shape else 1 + except Exception: + return -1 def _materialize_lazy_leaves() -> bool: @@ -294,10 +319,11 @@ def _run_node_op(): gpu_peak_bytes = -1 measurement = None if _measure_memory(): - input_resident = sum(estimate_modality_bytes(m) for m in input_mods) + input_split = [split_modality_bytes(m) for m in input_mods] result, measurement = measure_memory_during( lambda: _run_gpu_op(_run_node_op, gpu_id, cleanup_after=False), - input_resident_bytes=input_resident, + input_resident_bytes=sum(p for p, _ in input_split), + input_shared_bytes=sum(s for _, s in input_split), sample_s=0.01, ) gpu_peak_bytes = ( @@ -309,6 +335,8 @@ def _run_node_op(): shm_name, resident_bytes, shm_bytes, actual_stats = _offload_to_shared_memory( result ) + if measurement is not None: + measurement.output_shm_bytes = int(shm_bytes or 0) end_time = time.perf_counter() return { "result": result, @@ -343,10 +371,12 @@ def _execute_task_batch_worker(task: Any, items: List[tuple], gpu_id: int): datas.append((node_id, task_input.data, task_input.transform_time)) except Exception as e: failed[node_id] = f"{type(e).__name__}: {e}" + input_done = time.perf_counter() allowed = os.sched_getaffinity(0) cores = sorted(allowed) - os.sched_setaffinity(0, {cores[os.getpid() % len(cores)]}) + pinned_core = cores[os.getpid() % len(cores)] + os.sched_setaffinity(0, {pinned_core}) try: scores = task.run_many( [data for _, data, _ in datas], @@ -354,9 +384,28 @@ def _execute_task_batch_worker(task: Any, items: List[tuple], gpu_id: int): [seconds for _, _, seconds in datas], ) finally: + fit_done = time.perf_counter() os.sched_setaffinity(0, allowed) cleanup_gpu(gpu_id) - share = (time.perf_counter() - start_time) / max(1, len(datas)) + end = time.perf_counter() + share = (end - start_time) / max(1, len(datas)) + _diag( + "worker", + { + "path": "batch", + "gpu": gpu_id, + "n": len(items), + "nodes": [node_id for node_id, _, _ in items], + "widths": [_diag_width(data) for _, data, _ in datas], + "start": start_time, + "input_s": input_done - start_time, + "fit_s": fit_done - input_done, + "cleanup_s": end - fit_done, + "pinned_core": pinned_core, + "n_allowed_cores": len(cores), + "timing": dict(getattr(task, "last_run_timing", {}) or {}), + }, + ) results = {} for i, ((node_id, _, _), result) in enumerate(zip(datas, scores)): if isinstance(result, Exception): @@ -396,16 +445,34 @@ def _execute_task_worker( def _run_task(): start = time.perf_counter() task_input = _task_input(modality, aggregation) + input_done = time.perf_counter() scores = task.run(task_input.data, task_input.transform_time) end = time.perf_counter() + _diag( + "worker", + { + "path": "single", + "gpu": gpu_id, + "n": 1, + "nodes": [task_node_id], + "widths": [_diag_width(task_input.data)], + "start": start, + "input_s": input_done - start, + "fit_s": end - input_done, + "n_allowed_cores": len(os.sched_getaffinity(0)), + "timing": dict(getattr(task, "last_run_timing", {}) or {}), + }, + ) return scores, end - start gpu_peak_bytes = -1 measurement = None if _measure_memory(): + input_private, input_shared = split_modality_bytes(modality) result, measurement = measure_memory_during( lambda: _run_gpu_op(_run_task, gpu_id), - input_resident_bytes=estimate_modality_bytes(modality), + input_resident_bytes=input_private, + input_shared_bytes=input_shared, sample_s=0.01, ) gpu_peak_bytes = ( @@ -590,8 +657,14 @@ def _dispatch_split_dags(payload, gpu_id): if gpu_id is not None: cores = sorted(allowed) os.sched_setaffinity(0, {cores[os.getpid() % len(cores)]}) + measurement = None try: - results = evaluate_fitted_dags(*payload) + if _measure_memory(): + results, measurement = measure_memory_during( + lambda: evaluate_fitted_dags(*payload), sample_s=0.05 + ) + else: + results = evaluate_fitted_dags(*payload) finally: os.sched_setaffinity(0, allowed) if gpu_id is not None: @@ -601,6 +674,7 @@ def _dispatch_split_dags(payload, gpu_id): "start_time": start_time, "end_time": time.perf_counter(), "pid": os.getpid(), + "memory": measurement, } @@ -677,6 +751,7 @@ def __init__( worker_pool: Optional[PersistentWorkerPool] = None, search_start: Optional[float] = None, split_jobs: Optional[List[tuple]] = None, + split_job_memory: Optional[List[int]] = None, ): self.enable_checkpointing = enable_checkpointing available_total_cpu = cpu_memory_budget_bytes( @@ -734,9 +809,12 @@ def __init__( self._leaf_load_seconds: Dict[int, float] = {} self._task_results: Dict[str, ResultEntry] = {} self._fused_shapes: Dict[tuple, tuple] = {} + self._fused_widths: Dict[str, int] = {} + self._last_dispatch_log = 0.0 self._fused_ready_since: Dict[str, float] = {} self._split_pending = list(enumerate(split_jobs or [])) self._split_results: List[Any] = [None] * len(self._split_pending) + self._split_job_memory = list(split_job_memory or []) self._owns_pool = worker_pool is None if worker_pool is None: @@ -1023,11 +1101,19 @@ def _fused_task_nodes(self, ready) -> List[str]: fused.append(node_id) return fused + def _fused_input_width(self, node) -> int: + width = self._fused_widths.get(node.node_id) + if width is None: + stats = self.scheduler.task_input_stats( + node.node_id, self.scheduler.node_stats.get(node.inputs[0]) + ) + width = int(math.prod(getattr(stats, "output_shape", None) or (1,))) + self._fused_widths[node.node_id] = width + return width + def _fused_batch_shape(self, node) -> tuple: task_idx = int(node.parameters.get("_task_idx", 0)) - stats = self.scheduler.node_stats.get(node.inputs[0]) - width = math.prod(getattr(stats, "output_shape", None) or (1,)) - key = (task_idx, int(width)) + key = (task_idx, self._fused_input_width(node)) if key not in self._fused_shapes: task = self._tasks[task_idx] rows = len(set(task.train_indices) | set(task.test_indices or [])) @@ -1046,12 +1132,72 @@ def _submit_fused_task_batches(self, node_ids: List[str]) -> None: self._fused_ready_since.setdefault(node_id, now) shape = self._fused_batch_shape(self.scheduler.mapping[node_id]) groups.setdefault(shape, []).append(node_id) - for ((task_idx, _), per_candidate, slots), members in sorted(groups.items()): + diag = ( + {"submitted": [], "held": 0, "stop": None} if _DISPATCH_DIAG_DIR else None + ) + try: + self._submit_fused_groups(groups, idle, now, diag) + finally: + if diag is not None: + self._log_dispatch(node_ids, groups, idle, diag) + + def _log_dispatch(self, node_ids, groups, idle, diag) -> None: + now = time.perf_counter() + if not diag["submitted"] and now - self._last_dispatch_log < 0.5: + return + self._last_dispatch_log = now + sched = self.scheduler + roots = sched.roots + units = {} + for unit in self._job_units.values(): + units[type(unit).__name__] = units.get(type(unit).__name__, 0) + 1 + _diag( + "dispatch", + { + "fused_ready": len(node_ids), + "groups": len(groups), + "group_sizes": sorted((len(m) for m in groups.values()), reverse=True), + "idle_flag": idle, + "submitted": diag["submitted"], + "held_for_siblings": diag["held"], + "stop": diag["stop"], + "tasks_awaiting_admission": sum( + 1 for n in sched._candidates if n in roots + ), + "tasks_admitted": sum( + 1 + for n in sched.ready_nodes + if not isinstance(n, list) and n in roots + ), + "tasks_running": sum(1 for n in sched.running_nodes if n in roots), + "job_units": units, + "idle_cpu_workers": len(self._pool._idle_pids), + "gpus": { + g: { + "slots_used": sched.gpu_slots_in_use.get(g, 0), + "slots": sched.gpu_slots_per_device, + "free_gb": sched._free_gpu_bytes(g, {}) / 2**30, + "in_use_gb": sched.memory_stats["gpu_in_use"].get(g, 0) / 2**30, + "retained_gb": sched.memory_stats["gpu_retained"].get(g, 0) + / 2**30, + "idle_workers": len(self._pool._idle_gpu_pids.get(g, [])), + } + for g in sched.gpu_devices + }, + }, + ) + + def _submit_fused_groups(self, groups, idle, now, diag) -> None: + for ((task_idx, width), per_candidate, slots), members in sorted( + groups.items() + ): task = self._tasks[task_idx] gpu_bytes = slots * per_candidate + _FUSED_CONTEXT_BYTES waited = now - min(self._fused_ready_since[n] for n in members) for start in range(0, len(members), slots): if len(members) - start < slots and not idle and waited < _FUSED_WAIT_S: + if diag is not None: + diag["held"] += len(members) - start break gpu_id = self.scheduler.reserve_gpu( gpu_bytes, @@ -1059,7 +1205,24 @@ def _submit_fused_task_batches(self, node_ids: List[str]) -> None: force=not self._job_units, ) if gpu_id is None: + if diag is not None: + diag["stop"] = { + "reason": "reserve_failed", + "width": width, + "need_gb": gpu_bytes / 2**30, + "waiting_in_group": len(members) - start, + } return + if diag is not None: + diag["submitted"].append( + { + "gpu": gpu_id, + "width": width, + "n": min(slots, len(members) - start), + "slots": slots, + "reserved_gb": gpu_bytes / 2**30, + } + ) batch = members[start : start + slots] items, retained = [], [] for node_id in batch: @@ -1086,6 +1249,9 @@ def _submit_fused_task_batches(self, node_ids: List[str]) -> None: def _submit_split_jobs(self) -> None: while self._split_pending and self._pool.has_idle_worker: index, (payload, uses_gpu) = self._split_pending[0] + cpu_bytes = self._split_cpu_bytes(index) + if not self.scheduler.can_reserve_external_cpu(cpu_bytes): + return gpu_id = None if uses_gpu and self.scheduler.gpu_devices: gpu_id = self.scheduler.reserve_gpu( @@ -1098,11 +1264,17 @@ def _submit_split_jobs(self) -> None: if gpu_id is None and not self._pool.has_idle_worker_for(None, True): return self._split_pending.pop(0) + self.scheduler.reserve_external_cpu(f"split_{index}", cpu_bytes) job_id = self._pool.submit( "split_dags", payload, gpu_id=gpu_id, allow_gpu_worker_for_cpu=True ) self._job_units[job_id] = _SplitUnit(index, gpu_id) + def _split_cpu_bytes(self, index: int) -> int: + if index < len(self._split_job_memory): + return int(self._split_job_memory[index] or 0) + return 0 + def _fill_pipeline(self) -> None: if not self._pool.has_idle_worker: return @@ -1126,7 +1298,15 @@ def _gpu_id(entry): gpu_id, allow_gpu_worker_for_cpu=allow_gpu_worker_for_cpu, ): - continue + if gpu_id is None or isinstance(entry, list): + continue + if ( + self.scheduler.rebind_gpu( + entry, lambda g: self._pool.has_idle_worker_for(g) + ) + is None + ): + continue if isinstance(entry, list): self._submit_leaf_batch( entry, @@ -1184,16 +1364,32 @@ def _process_result(self, jr) -> None: if isinstance(unit, _TaskBatchUnit): self.scheduler.release_gpu(unit.gpu_id, unit.gpu_bytes) if isinstance(unit, _SplitUnit): + split_key = f"split_{unit.index}" + self.scheduler.release_external_cpu(split_key) if unit.gpu_id is not None: self.scheduler.release_gpu(unit.gpu_id, _FUSED_CONTEXT_BYTES) if jr.ok: value = jr.value self._record_stats( - f"split_{unit.index}", + split_key, value["pid"], value["start_time"], value["end_time"], ) + measurement = value.get("memory") + if self.enable_checkpointing: + self._checkpoint_memory_usage( + split_key, + measurement.increment_bytes if measurement else -1, + -1, + "split_dags", + None, + measurement=measurement, + estimated_resources=( + self._split_cpu_bytes(unit.index), + 0, + ), + ) self._split_results[unit.index] = value["results"] else: self._split_results[unit.index] = jr.error or "worker failure" @@ -1397,7 +1593,10 @@ def _checkpoint_memory_usage( actual_stats: Optional[RepresentationStats], measurement: Optional[MemoryMeasurement] = None, estimated_stats: Optional[RepresentationStats] = None, + estimated_resources: Optional[tuple] = None, ): + if estimated_resources is None: + estimated_resources = self.scheduler.node_resources[node_id] if self.enable_checkpointing: self._memory_usage_checkpoint.increment(node_id) if estimated_stats is None: @@ -1409,7 +1608,7 @@ def _checkpoint_memory_usage( shape = None if DEBUG: shape = self._print_node_stats(node_id, actual_stats, operation_name) - est_cpu, est_gpu = self.scheduler.node_resources[node_id] + est_cpu, est_gpu = estimated_resources if peak_bytes is not None and peak_bytes >= 0: if peak_bytes > est_cpu: print( @@ -1437,8 +1636,8 @@ def _checkpoint_memory_usage( "cpu_peak_bytes": peak_bytes if peak_bytes is not None else -1, "gpu_peak_bytes": gpu_peak_bytes if gpu_peak_bytes is not None else -1, "operation_name": operation_name, - "estimated_cpu_bytes": self.scheduler.node_resources[node_id][0], - "estimated_gpu_bytes": self.scheduler.node_resources[node_id][1], + "estimated_cpu_bytes": estimated_resources[0], + "estimated_gpu_bytes": estimated_resources[1], "shape": shape, "cpu_increment_bytes": ( measurement.increment_bytes if measurement else -1 @@ -1453,6 +1652,27 @@ def _checkpoint_memory_usage( measurement.traced_peak_bytes if measurement else -1 ), "rss_delta_bytes": measurement.rss_delta_bytes if measurement else -1, + "input_shared_bytes": ( + measurement.input_shared_bytes if measurement else -1 + ), + "shared_rss_delta_bytes": ( + measurement.shared_rss_delta_bytes if measurement else -1 + ), + "output_shm_bytes": ( + measurement.output_shm_bytes if measurement else -1 + ), + "pid": measurement.pid if measurement else -1, + "t_start_unix": measurement.t_start_unix if measurement else -1.0, + "t_end_unix": measurement.t_end_unix if measurement else -1.0, + "peak_abs_rss_bytes": ( + measurement.peak_abs_rss_bytes if measurement else -1 + ), + "baseline_private_rss_bytes": ( + measurement.baseline_private_rss_bytes if measurement else -1 + ), + "peak_private_rss_bytes": ( + measurement.peak_private_rss_bytes if measurement else -1 + ), "num_instances": getattr(actual_stats, "num_instances", None), "output_shape": getattr(actual_stats, "output_shape", None), "estimated_output_shape": getattr( @@ -1607,6 +1827,7 @@ def _reuse_report(self) -> Dict[str, Any]: "cpu_fallback_nodes": len( getattr(self.scheduler, "cpu_fallback_nodes", []) or [] ), + "gpu_rebinds": getattr(self.scheduler, "gpu_rebinds", 0), "planned_task_nodes": len(getattr(self.scheduler, "roots", ()) or ()), "completed_task_nodes": self._eval_counter, "exclusive_memory_nodes": len( diff --git a/src/main/python/systemds/scuro/drsearch/node_scheduler.py b/src/main/python/systemds/scuro/drsearch/node_scheduler.py index 77ddc9f9557..39cfa2809f1 100644 --- a/src/main/python/systemds/scuro/drsearch/node_scheduler.py +++ b/src/main/python/systemds/scuro/drsearch/node_scheduler.py @@ -38,7 +38,11 @@ from systemds.scuro.utils.memory_utility import gpu_memory_info from systemds.scuro.utils.static_variables import DEBUG, deterministic_enabled -_MAX_GPU_SCHEDULE_ATTEMPTS = int(os.environ.get("SCURO_MAX_GPU_SCHEDULE_ATTEMPTS", "3")) +_MAX_GPU_SCHEDULE_ATTEMPTS = ( + int(os.environ["SCURO_MAX_GPU_SCHEDULE_ATTEMPTS"]) + if os.environ.get("SCURO_MAX_GPU_SCHEDULE_ATTEMPTS") + else None +) def _exclusive_over_budget_enabled() -> bool: @@ -85,6 +89,7 @@ def __init__( self.gpu_slots_per_device = default_gpu_slots_per_device() self.gpu_slots_in_use = {gpu_id: 0 for gpu_id in self.gpu_devices} self._gpu_slot_nodes: Dict[str, int] = {} + self.gpu_rebinds = 0 self.success = False self.deadlock = False self.ready_nodes = [] @@ -120,6 +125,8 @@ def __init__( } self.release_requests: Set[int] = set() self._cpu_reserved_nodes: Dict[str, int] = {} + # Reservations of work that is not a DAG node (split jobs). + self._external_cpu_keys: Set[str] = set() self.exclusive_memory_nodes: Dict[str, str] = {} self._initialized = False @@ -144,6 +151,46 @@ def can_start_now(self, node_id: str) -> bool: return True return resources[0] <= self.memory_budget["cpu"] - self._total_cpu_in_use() + def can_reserve_external_cpu(self, cpu_bytes: int) -> bool: + if cpu_bytes <= 0: + return True + free = ( + self.memory_budget["cpu"] + - self._total_cpu_in_use() + - self._pending_admitted_cpu_bytes() + ) + if cpu_bytes <= free: + return True + return ( + not self.running_nodes + and not self._ready_set + and self.memory_stats["cpu_in_flight"] == 0 + ) + + def reserve_external_cpu(self, key: str, cpu_bytes: int) -> None: + if cpu_bytes <= 0 or key in self._cpu_reserved_nodes: + return + if cpu_bytes > ( + self.memory_budget["cpu"] + - self._total_cpu_in_use() + - self._pending_admitted_cpu_bytes() + ): + self.exclusive_memory_nodes[key] = ( + f"estimated {cpu_bytes / 1024**3:.2f} GB, more than the free CPU " + f"budget; run alone" + ) + self._cpu_reserved_nodes[key] = int(cpu_bytes) + self.__dict__.setdefault("_external_cpu_keys", set()).add(key) + self.memory_stats["cpu_in_flight"] += int(cpu_bytes) + + def release_external_cpu(self, key: str) -> None: + getattr(self, "_external_cpu_keys", set()).discard(key) + reserved = self._cpu_reserved_nodes.pop(key, 0) + if reserved: + self.memory_stats["cpu_in_flight"] = max( + 0, self.memory_stats["cpu_in_flight"] - reserved + ) + def update_cpu_memory_in_use(self, delta_bytes: int): self.memory_stats["cpu_cached"] += delta_bytes @@ -273,6 +320,7 @@ def _estimate_node_costs(self) -> Dict[str, float]: ) task_idx = self.mapping[node_id].parameters.get("_task_idx", 0) try: + input_stats = self.task_input_stats(node_id, input_stats) costs[node_id] = max( 1.0, float(self.tasks[task_idx].estimate_relative_cost(input_stats)), @@ -331,6 +379,23 @@ def reserve_gpu(self, memory_needed: int, usable, force: bool = False): return gpu_id return None + def rebind_gpu(self, node_id: str, usable) -> Optional[int]: + node = self.mapping[node_id] + gpu_mem = self.node_resources[node_id][1] + for gpu_id in sorted( + self.gpu_devices, key=lambda g: -self._free_gpu_bytes(g, {}) + ): + if gpu_id == node.gpu_id or not usable(gpu_id): + continue + if self.gpu_slots_in_use.get(gpu_id, 0) >= self.gpu_slots_per_device: + continue + if self._free_gpu_bytes(gpu_id, {}) < gpu_mem: + continue + node.gpu_id = gpu_id + self.gpu_rebinds += 1 + return gpu_id + return None + def release_gpu(self, gpu_id: int, memory_needed: int) -> None: self.gpu_slots_in_use[gpu_id] = max(0, self.gpu_slots_in_use[gpu_id] - 1) self.memory_stats["gpu_in_use"][gpu_id] = max( @@ -456,7 +521,9 @@ def update_node_stats_and_reestimate_descendants( else: task_node = self.mapping[desc_id] task = self.tasks[task_node.parameters["_task_idx"]] - peak_memory = task.estimate_peak_memory_bytes(input_stats) + peak_memory = task.estimate_peak_memory_bytes( + self.task_input_stats(desc_id, input_stats) + ) cpu_increment = max( int(peak_memory["cpu_peak_bytes"]), @@ -468,6 +535,19 @@ def update_node_stats_and_reestimate_descendants( ) self.node_stats[desc_id] = task.get_output_stats(input_stats) + def task_input_stats(self, node_id: str, input_stats: Any) -> Any: + if os.environ.get("SCURO_TASK_WIDTH", "aggregated") == "parent": + return input_stats + aggregation = getattr(self.mapping[node_id], "aggregation", None) + if aggregation is None or input_stats is None or isinstance(input_stats, list): + return input_stats + try: + return aggregation.operation( + params=aggregation.parameters + ).get_output_stats(input_stats) + except Exception: + return input_stats + def _get_descendants_in_topo_order(self, node_id: str) -> List[str]: reachable = set() queue = deque(self.children.get(node_id, set())) @@ -502,7 +582,11 @@ def _is_deadlock(self) -> bool: return blocked def not_enough_memory(self) -> bool: - if self.running_nodes or self.ready_nodes: + if ( + self.running_nodes + or self.ready_nodes + or getattr(self, "_external_cpu_keys", None) + ): return False for node_id in self._candidates: cpu_mem, gpu_mem = self.node_resources[node_id] @@ -521,8 +605,9 @@ def not_enough_memory(self) -> bool: if gpu_id is None: if self._gpu_freed_by_release(gpu_mem) is not None: continue - if ( - self.gpu_wait_attempts.get(node_id, 0) + if self._fits_some_gpu(gpu_mem) and ( + _MAX_GPU_SCHEDULE_ATTEMPTS is None + or self.gpu_wait_attempts.get(node_id, 0) <= _MAX_GPU_SCHEDULE_ATTEMPTS ): continue @@ -583,26 +668,29 @@ def _check_memory_constraints( if release_gpu is not None: self.release_requests.add(release_gpu) return False, None - if deterministic_enabled() and any( - gpu_mem <= self.memory_budget["gpu"][gpu] - for gpu in self.gpu_devices - ): - return False, None - attempts = self.gpu_wait_attempts.get(node_id, 0) + 1 - self.gpu_wait_attempts[node_id] = attempts - if attempts > _MAX_GPU_SCHEDULE_ATTEMPTS: + if self._fits_some_gpu(gpu_mem): + if _MAX_GPU_SCHEDULE_ATTEMPTS is None or deterministic_enabled(): + if DEBUG: + print(f"Node {node_id} waits for a GPU") + return False, None + attempts = self.gpu_wait_attempts.get(node_id, 0) + 1 + self.gpu_wait_attempts[node_id] = attempts + if attempts <= _MAX_GPU_SCHEDULE_ATTEMPTS: + return False, None reason = ( f"no GPU had {gpu_mem / 1024**3:.2f} GB free after " f"{attempts} scheduling attempts; running on CPU instead" ) - if DEBUG: - print(f"Node {node_id}: {reason}") - self.cpu_fallback_nodes.append(node_id) - self.cpu_fallback_reasons[node_id] = reason - return True, None + else: + reason = ( + f"needs {gpu_mem / 1024**3:.2f} GB of GPU memory, more " + f"than any GPU's budget; running on CPU instead" + ) if DEBUG: - print(f"Node {node_id} has no available GPU") - return False, None + print(f"Node {node_id}: {reason}") + self.cpu_fallback_nodes.append(node_id) + self.cpu_fallback_reasons[node_id] = reason + return True, None return True, gpu_id @@ -653,6 +741,12 @@ def _gpu_freed_by_release( candidates = [c for c in candidates if c[0] >= memory_needed] return max(candidates)[1] if candidates else None + def _fits_some_gpu(self, memory_needed: int) -> bool: + return any( + memory_needed <= self.memory_budget["gpu"][gpu_id] + for gpu_id in self.gpu_devices + ) + def _waiting_for_gpu_slot( self, memory_needed: int, @@ -836,7 +930,9 @@ def _estimate_node_resources(self): else: task_node = self.mapping[node] task = self.tasks[task_node.parameters["_task_idx"]] - peak_memory = task.estimate_peak_memory_bytes(input_stats) + peak_memory = task.estimate_peak_memory_bytes( + self.task_input_stats(node, input_stats) + ) cpu_increment = max( int(peak_memory["cpu_peak_bytes"]), diff --git a/src/main/python/systemds/scuro/drsearch/task.py b/src/main/python/systemds/scuro/drsearch/task.py index fbc9b9e4bba..014354b41a3 100644 --- a/src/main/python/systemds/scuro/drsearch/task.py +++ b/src/main/python/systemds/scuro/drsearch/task.py @@ -25,8 +25,10 @@ from systemds.scuro.utils.static_variables import get_global_seed, get_train_seed from systemds.scuro.models.model import Model from systemds.scuro.models.fused_mlp import ( + candidate_bytes, candidate_slots, fit_folds, + fused_gpu_bytes, fused_training_enabled, supports_fused_training, ) @@ -157,6 +159,15 @@ def get_output_stats(self, input_stats): # TODO: Implement a default estimate of the output stats for the task return RepresentationStats(0, (0,)) + def trains_fused(self, feature_dim: int) -> bool: + if not (fused_training_enabled() and supports_fused_training(self.model)): + return False + if feature_dim <= 0: + return True + rows = len(set(self.train_indices) | set(self.test_indices or [])) + per_candidate = candidate_bytes(self.model, feature_dim, rows, self.kfold) + return per_candidate <= fused_gpu_bytes() + def estimate_peak_memory_bytes(self, input_stats): labels_array = np.asarray(self.labels) n_instances = int(input_stats.num_instances) @@ -223,7 +234,7 @@ def estimate_peak_memory_bytes(self, input_stats): uses_gpu = getattr(self.model, "uses_gpu", None) if uses_gpu is None: uses_gpu = model_peak_memory_gpu > 0 - if fused_training_enabled() and supports_fused_training(self.model): + if self.trains_fused(feature_dim): uses_gpu = False gpu_peak_bytes = int(model_peak_memory_gpu * 1.4) if uses_gpu else 0 diff --git a/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py b/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py index 64624ee10e6..9073ed45b85 100644 --- a/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py +++ b/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py @@ -37,6 +37,8 @@ ResultEntry, decoded_lazy_leaves, ) +from systemds.scuro.drsearch.node_scheduler import MemoryAwareNodeScheduler +from systemds.scuro.utils.memory_utility import gpu_memory_info, release_freed_heap from systemds.scuro.drsearch.split_aware_evaluator import ( SplitAwareDAGEvaluator, dag_requires_fitting, @@ -105,6 +107,14 @@ def _task_uses_gpu(task) -> bool: return str(getattr(device, "type", device) or "").startswith("cuda") +def _trains_representation(dags) -> bool: + return any( + _fit_requirement(node) is FitRequirement.SUPERVISED + for dag in dags + for node in dag.nodes + ) + + def evaluate_fitted_dags(dags, modality, tasks) -> list: budget_gb = float(os.environ.get("SCURO_FITTED_DAG_CACHE_GB", "4")) store = TrialResultCache(int(budget_gb * 1024**3), min_compute_seconds=0) @@ -113,6 +123,7 @@ def evaluate_fitted_dags(dags, modality, tasks) -> list: eval_dag, eval_modality, encode_time = UnimodalOptimizer._encode_below_lstm( dag, modality ) + release_freed_heap() supervised = any( _fit_requirement(node) is FitRequirement.SUPERVISED for node in eval_dag.nodes @@ -141,6 +152,153 @@ def evaluate_fitted_dags(dags, modality, tasks) -> list: return results +_FIT_PHASE_FACTOR = 6 +_ENCODE_PHASE_MARGIN = 1.2 +_GPU_USABLE_FRACTION = float(os.environ.get("SCURO_GPU_USABLE_FRACTION", "0.9")) + + +def _fitted_dag_cache_bytes() -> int: + return int(float(os.environ.get("SCURO_FITTED_DAG_CACHE_GB", "4")) * 1024**3) + + +def _encodes_below_lstm(dag: RepresentationDag) -> bool: + root = dag.get_node_by_id(dag.root_node_id) + if root.operation is not LSTM: + return False + return not dag_requires_fitting(RepresentationDag(dag.nodes, root.inputs[0])) + + +def estimate_fitted_dag_memory( + dag: RepresentationDag, + modality, + tasks, + task_dags: List[RepresentationDag], +) -> Dict[str, int]: + scheduler = MemoryAwareNodeScheduler(task_dags, [modality], tasks, float("inf")) + resources = scheduler.node_resources + leaves = scheduler.leaves + tasks_ids = scheduler.roots + leaf_bytes = sum(int(resources[leaf][0]) for leaf in leaves) + + def output_bytes(node_id: str) -> int: + if node_id in leaves: + return 0 + return MemoryAwareNodeScheduler._stats_to_bytes( + scheduler.node_stats.get(node_id) + ) + + task_peak = max((int(resources[n][0]) for n in tasks_ids), default=0) + violation = _fitted_input_violation(scheduler) + gpu_peak = max( + (int(resources[n][1]) for n in scheduler.mapping if n not in leaves), + default=0, + ) + + root = dag.get_node_by_id(dag.root_node_id) + if _encodes_below_lstm(dag): + + def chain_peak(node_id: str) -> tuple: + node = scheduler.mapping[node_id] + if not node.inputs: + return 0, 0 + held, peak = 0, 0 + for input_id in node.inputs: + input_peak, input_out = chain_peak(input_id) + peak = max(peak, held + input_peak) + held += input_out + out = output_bytes(node_id) + peak = max(peak, int(resources[node_id][0]), held + out) + return peak, out + + encode_peak, encoded = chain_peak(root.inputs[0]) + encode_peak = int(_ENCODE_PHASE_MARGIN * encode_peak) + lstm_own = max(0, int(resources[root.node_id][0]) - encoded) + fit_peak = _FIT_PHASE_FACTOR * encoded + lstm_own + task_peak + cpu_peak = leaf_bytes + max(encode_peak, fit_peak) + return { + "cpu_peak_bytes": int(cpu_peak), + "gpu_peak_bytes": int(gpu_peak), + "encode_peak_bytes": int(encode_peak), + "fit_peak_bytes": int(fit_peak), + "input_violation": violation, + } + else: + representation_ids = [ + n for n in scheduler.mapping if n not in leaves and n not in tasks_ids + ] + live = sum(output_bytes(n) for n in representation_ids) + transient = max( + (int(resources[n][0]) for n in scheduler.mapping if n not in leaves), + default=0, + ) + fit_input = 0 + for node_id in representation_ids: + node = scheduler.mapping[node_id] + if _fit_requirement(node) is FitRequirement.NONE: + continue + fit_input += 2 * sum( + output_bytes(p) if p not in leaves else int(resources[p][0]) + for p in node.inputs + ) + cpu_peak = leaf_bytes + live + transient + fit_input + return { + "cpu_peak_bytes": int(cpu_peak), + "gpu_peak_bytes": int(gpu_peak), + "input_violation": violation, + } + + +def _fitted_input_violation(scheduler) -> Optional[str]: + for node_id in scheduler.topo_order: + node = scheduler.mapping[node_id] + if not node.inputs or _fit_requirement(node) is FitRequirement.NONE: + continue + operation = node.operation(params=node.parameters) + check = getattr(operation, "input_size_violation", None) + if not callable(check): + continue + stats = [scheduler.node_stats.get(input_id) for input_id in node.inputs] + if any(s is None for s in stats): + continue + reason = check(stats[0] if len(stats) == 1 else stats) + if reason: + return reason + return None + + +def _drop_infeasible_fitted_dags(dags, memory, modality) -> list: + feasible = [] + for dag in dags: + reason = memory[dag.dag_id].get("input_violation") + if reason: + print( + f"[unimodal] WARNING: modality {modality.modality_id} skips fitted " + f"DAG {dag.dag_id}: {reason}", + flush=True, + ) + continue + feasible.append(dag) + dags = feasible + largest = max((int(info["total_b"]) for info in gpu_memory_info()), default=0) + if largest <= 0: + return list(dags) + usable = int(largest * _GPU_USABLE_FRACTION) + kept = [] + for dag in dags: + need = memory[dag.dag_id]["gpu_peak_bytes"] + if need > usable: + print( + f"[unimodal] WARNING: modality {modality.modality_id} skips fitted " + f"DAG {dag.dag_id}: its model needs about {need / 1024**3:.1f} GB " + f"of GPU memory per training batch, more than the usable " + f"{usable / 1024**3:.1f} GB of the largest GPU", + flush=True, + ) + continue + kept.append(dag) + return kept + + class UnimodalOptimizer: def __init__( self, @@ -464,14 +622,30 @@ def _process_modality(self, modality, skip_remaining: int = 0, scheduler=None): task_results = [] statistics = {"worker_stats": {}, "node_stats": {}, "reuse": {}} split_jobs = [] + split_job_memory = [] if fitted_dags: SplitAwareDAGEvaluator.materialize(modality) + memory = { + dag.dag_id: estimate_fitted_dag_memory( + dag, modality, self.tasks, self._expand_dags_with_task_roots([dag]) + ) + for dag in fitted_dags + } groups = {} - for dag in fitted_dags: + for dag in _drop_infeasible_fitted_dags(fitted_dags, memory, modality): groups.setdefault(_lowest_fitted_key(dag), []).append(dag) - uses_gpu = any(_task_uses_gpu(task) for task in self.tasks) + tasks_use_gpu = any(_task_uses_gpu(task) for task in self.tasks) split_jobs = [ - ((group, modality, self.tasks), uses_gpu) for group in groups.values() + ( + (group, modality, self.tasks), + tasks_use_gpu or _trains_representation(group), + ) + for group in groups.values() + ] + split_job_memory = [ + max(memory[dag.dag_id]["cpu_peak_bytes"] for dag in group) + + _fitted_dag_cache_bytes() + for group in groups.values() ] split_results = [] @@ -486,6 +660,7 @@ def _process_modality(self, modality, skip_remaining: int = 0, scheduler=None): worker_pool=self._persistent_worker_pool, search_start=self._search_start, split_jobs=split_jobs, + split_job_memory=split_job_memory, ) if self._reuse_worker_pool and self._persistent_worker_pool is None: self._persistent_worker_pool = node_executor.detach_worker_pool() @@ -580,6 +755,8 @@ def _window_input_stats(self, modality: Modality, window_length: int): def _effective_window_length(context_operator, window_size, num_window, signal_len): if context_operator.granularity_kind == "count": return max(1, int(math.ceil(signal_len / max(1, int(num_window))))) + if context_operator.granularity_kind == "fixed": + return max(1, int(signal_len)) return max(1, int(window_size)) def _build_execution_dags_for_modality( @@ -982,7 +1159,12 @@ def temporal_context_operators(self, modality, builder, leaf_id): return [] dags = [] for context_operator in context_operators: - for window_size, num_window in zip(window_lengths, num_windows): + granularities = ( + [(None, None)] + if getattr(context_operator, "granularity_kind", None) == "fixed" + else list(zip(window_lengths, num_windows)) + ) + for window_size, num_window in granularities: window_node_ids = [] for agg in aggregators: aggregation_instance = agg() @@ -1043,6 +1225,8 @@ def temporal_context_operators(self, modality, builder, leaf_id): def _apply_granularity(context_operator_instance, window_size, num_window): parameter = getattr(context_operator_instance, "granularity_parameter", None) kind = getattr(context_operator_instance, "granularity_kind", None) + if kind == "fixed": + return if parameter is None or kind not in ("length", "count"): raise ValueError( f"{type(context_operator_instance).__name__} is registered as a " diff --git a/src/main/python/systemds/scuro/representations/concatenation.py b/src/main/python/systemds/scuro/representations/concatenation.py index f599aba0142..3074e2bf035 100644 --- a/src/main/python/systemds/scuro/representations/concatenation.py +++ b/src/main/python/systemds/scuro/representations/concatenation.py @@ -27,7 +27,10 @@ from systemds.scuro.representations.utils import pad_sequences from systemds.scuro.representations.fusion import Fusion -from systemds.scuro.representations.representation import RepresentationStats +from systemds.scuro.representations.representation import ( + RepresentationStats, + stats_itemsize, +) from systemds.scuro.drsearch.operator_registry import register_fusion_operator @@ -165,6 +168,25 @@ def estimate_peak_memory_bytes(self, input_stats) -> dict: input_bytes = sum(self._stats_bytes(s) for s in stats_list) output_bytes = self._stats_bytes(self.get_output_stats(input_stats)) + if self.preserve_leading_axis: + num_windows = self.get_output_stats(input_stats).output_shape[0] + padded_bytes = 0 + for stats in stats_list: + windows, features = self._flatten_feature_shape( + tuple(stats.output_shape) + ) + if windows < num_windows: + padded_bytes += ( + int(stats.num_instances) + * int(num_windows) + * int(features) + * stats_itemsize(stats) + ) + cpu_peak = ( + int((input_bytes + padded_bytes + output_bytes) * 1.1) + 8 * 1024 * 1024 + ) + return {"cpu_peak_bytes": cpu_peak, "gpu_peak_bytes": 0} + raw_bytes = self._raw_input_bytes(input_stats) cpu_peak = ( int((raw_bytes + 2 * input_bytes + output_bytes) * 1.1) + 8 * 1024 * 1024 diff --git a/src/main/python/systemds/scuro/utils/memory_utility.py b/src/main/python/systemds/scuro/utils/memory_utility.py index 521c8d40237..5c75fa3e1a7 100644 --- a/src/main/python/systemds/scuro/utils/memory_utility.py +++ b/src/main/python/systemds/scuro/utils/memory_utility.py @@ -18,6 +18,7 @@ # under the License. # # ------------------------------------------------------------- +import ctypes import os import re import resource @@ -221,9 +222,47 @@ def log_memory(operation_name: str, device: torch.device): ) +def _shared_container_bytes(data) -> Optional[int]: + if getattr(data, "shm_name", None) is None: + return None + payload = getattr(data, "payload_nbytes", None) # SharedStringList + if payload is not None: + return int(payload) + dtype = getattr(data, "_dtype", None) + if dtype is None: + return 0 + total_elems = getattr(data, "total_elems", None) # Shared(Grouped)ArrayList + if total_elems is None: + total_elems = int(np.prod(getattr(data, "shape", ()))) # SharedNDArray + return int(total_elems) * int(np.dtype(dtype).itemsize) + + +def split_numpy_like_bytes(data) -> Tuple[int, int]: + if data is None: + return 0, 0 + shared = _shared_container_bytes(data) + if shared is not None: + return 0, shared + if isinstance(data, dict): + values = data.values() + elif isinstance(data, (list, tuple)): + values = data + else: + return estimate_numpy_like_bytes(data), 0 + private = shared = 0 + for v in values: + p, s = split_numpy_like_bytes(v) + private += p + shared += s + return int(private), int(shared) + + def estimate_numpy_like_bytes(data) -> int: if data is None: return 0 + shared = _shared_container_bytes(data) + if shared is not None: + return shared if isinstance(data, np.ndarray): return int(data.nbytes) if isinstance(data, torch.Tensor): @@ -248,6 +287,14 @@ def estimate_modality_bytes(modality) -> int: return int(data_bytes + metadata_bytes) +def split_modality_bytes(modality) -> Tuple[int, int]: + if modality is None: + return 0, 0 + data_p, data_s = split_numpy_like_bytes(getattr(modality, "data", None)) + meta_p, meta_s = split_numpy_like_bytes(getattr(modality, "metadata", None)) + return int(data_p + meta_p), int(data_s + meta_s) + + def is_cuda_oom(exc: BaseException) -> bool: if isinstance(exc, torch.cuda.OutOfMemoryError): return True @@ -270,6 +317,14 @@ def cuda_oom_peak_bytes(exc: BaseException) -> int: return int(peak) +def release_freed_heap() -> None: + gc.collect() + try: + ctypes.CDLL("libc.so.6").malloc_trim(0) + except (OSError, AttributeError): + pass + + def cleanup_gpu(gpu_id: Optional[int]) -> None: if gpu_id is None or not torch.cuda.is_available(): return @@ -288,6 +343,14 @@ class MemoryMeasurement: traced_peak_bytes: int rss_delta_bytes: int peak_abs_rss_bytes: int + input_shared_bytes: int = 0 + shared_rss_delta_bytes: int = 0 + output_shm_bytes: int = 0 + pid: int = -1 + t_start_unix: float = -1.0 + t_end_unix: float = -1.0 + baseline_private_rss_bytes: int = -1 + peak_private_rss_bytes: int = -1 def merge_memory_measurements( @@ -303,23 +366,44 @@ def merge_memory_measurements( traced_peak_bytes=max(m.traced_peak_bytes for m in present), rss_delta_bytes=max(m.rss_delta_bytes for m in present), peak_abs_rss_bytes=max(m.peak_abs_rss_bytes for m in present), + input_shared_bytes=max(m.input_shared_bytes for m in present), + shared_rss_delta_bytes=max(m.shared_rss_delta_bytes for m in present), + output_shm_bytes=max(m.output_shm_bytes for m in present), + pid=present[0].pid, + t_start_unix=min(m.t_start_unix for m in present), + t_end_unix=max(m.t_end_unix for m in present), + baseline_private_rss_bytes=min(m.baseline_private_rss_bytes for m in present), + peak_private_rss_bytes=max(m.peak_private_rss_bytes for m in present), ) def measure_memory_during( - fn, *args, input_resident_bytes: int = 0, sample_s: float = 0.01, **kwargs + fn, + *args, + input_resident_bytes: int = 0, + input_shared_bytes: int = 0, + sample_s: float = 0.01, + **kwargs, ): proc = psutil.Process(os.getpid()) - baseline_rss = proc.memory_info().rss - peak_rss = baseline_rss + + def _sample(): + mi = proc.memory_info() + shared = getattr(mi, "shared", 0) + return int(mi.rss), int(mi.rss - shared), int(shared) + + t_start_unix = time.time() + baseline_rss, baseline_priv, baseline_shared = _sample() + peak_rss, peak_priv, peak_shared = baseline_rss, baseline_priv, baseline_shared stop = threading.Event() def sampler(): - nonlocal peak_rss + nonlocal peak_rss, peak_priv, peak_shared while not stop.is_set(): - rss = proc.memory_info().rss - if rss > peak_rss: - peak_rss = rss + rss, priv, shared = _sample() + peak_rss = max(peak_rss, rss) + peak_priv = max(peak_priv, priv) + peak_shared = max(peak_shared, shared) time.sleep(sample_s) owns_tracing = not tracemalloc.is_tracing() @@ -341,7 +425,7 @@ def sampler(): tracemalloc.stop() traced_increment = max(int(traced_peak) - int(traced_baseline), 0) - rss_delta = max(int(peak_rss) - int(baseline_rss), 0) + rss_delta = max(int(peak_priv) - int(baseline_priv), 0) increment = max(traced_increment, rss_delta) return out, MemoryMeasurement( @@ -351,4 +435,11 @@ def sampler(): traced_peak_bytes=traced_increment, rss_delta_bytes=rss_delta, peak_abs_rss_bytes=int(peak_rss), + input_shared_bytes=int(input_shared_bytes), + shared_rss_delta_bytes=max(int(peak_shared) - int(baseline_shared), 0), + pid=os.getpid(), + t_start_unix=t_start_unix, + t_end_unix=time.time(), + baseline_private_rss_bytes=int(baseline_priv), + peak_private_rss_bytes=int(peak_priv), ) diff --git a/src/main/python/tests/scuro/test_hp_tuner.py b/src/main/python/tests/scuro/test_hp_tuner.py index 7fd20872329..952e75452bb 100644 --- a/src/main/python/tests/scuro/test_hp_tuner.py +++ b/src/main/python/tests/scuro/test_hp_tuner.py @@ -52,6 +52,9 @@ ) from systemds.scuro.modality.type import ModalityType +from systemds.scuro.drsearch.hyperparameter_tuner import HyperparameterTuner +from systemds.scuro.drsearch.objectives import INFERENCE_TIME, dominates +from systemds.scuro.drsearch.representation_dag import CSEAwareDAGBuilder from systemds.scuro.drsearch.hyperparameter_tuner import ( HyperparamResult, HyperparamResults,