144 lines
4.5 KiB
Bash
Executable File
144 lines
4.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Batch Parallel Execution Utilities
|
|
# Provides functions for executing tasks in parallel batches
|
|
|
|
# Execute tasks in parallel batches
|
|
# Usage: batch_parallel_execute "function_name" "array_of_args" batch_size max_parallel
|
|
# Example: batch_parallel_execute "deploy_node" node_args_array 8 10
|
|
batch_parallel_execute() {
|
|
local func="$1"
|
|
local -n args_array="$2" # Name reference to array
|
|
local batch_size="${3:-8}"
|
|
local max_parallel="${4:-10}"
|
|
local total="${#args_array[@]}"
|
|
local pids=()
|
|
local completed=0
|
|
local failed=0
|
|
|
|
log_info "Executing $total tasks in batches of $batch_size (max $max_parallel parallel)..."
|
|
|
|
local i=0
|
|
while [[ $i -lt $total ]]; do
|
|
# Wait for slot if we've reached max parallel
|
|
while [[ ${#pids[@]} -ge $max_parallel ]]; do
|
|
for pid_idx in "${!pids[@]}"; do
|
|
local pid="${pids[$pid_idx]}"
|
|
if ! kill -0 "$pid" 2>/dev/null; then
|
|
wait "$pid"
|
|
local exit_code=$?
|
|
completed=$((completed + 1))
|
|
if [[ $exit_code -ne 0 ]]; then
|
|
log_error "Task failed with exit code $exit_code"
|
|
failed=$((failed + 1))
|
|
fi
|
|
unset pids[$pid_idx]
|
|
fi
|
|
done
|
|
pids=("${pids[@]}") # Rebuild array
|
|
sleep 0.2
|
|
done
|
|
|
|
# Process batch
|
|
local batch_end=$((i + batch_size))
|
|
[[ $batch_end -gt $total ]] && batch_end=$total
|
|
|
|
for ((j=i; j<batch_end; j++)); do
|
|
local args="${args_array[$j]}"
|
|
log_info "[$((j + 1))/$total] Starting: $args"
|
|
$func $args &
|
|
local pid=$!
|
|
pids+=("$pid")
|
|
done
|
|
|
|
i=$batch_end
|
|
done
|
|
|
|
# Wait for remaining processes
|
|
for pid in "${pids[@]}"; do
|
|
wait "$pid"
|
|
local exit_code=$?
|
|
completed=$((completed + 1))
|
|
if [[ $exit_code -ne 0 ]]; then
|
|
failed=$((failed + 1))
|
|
fi
|
|
done
|
|
|
|
if [[ $failed -gt 0 ]]; then
|
|
log_error "$failed task(s) failed out of $total"
|
|
return 1
|
|
fi
|
|
|
|
log_success "All $total tasks completed successfully"
|
|
return 0
|
|
}
|
|
|
|
# Execute tasks in parallel with progress tracking
|
|
# Usage: parallel_execute_with_progress "function_name" "array_of_args" max_parallel
|
|
parallel_execute_with_progress() {
|
|
local func="$1"
|
|
local -n args_array="$2"
|
|
local max_parallel="${3:-10}"
|
|
local total="${#args_array[@]}"
|
|
local pids=()
|
|
local completed=0
|
|
local failed=0
|
|
local start_time=$(date +%s)
|
|
|
|
log_info "Progress: [0%] [0/$total] Starting parallel execution (max $max_parallel)..."
|
|
|
|
local i=0
|
|
while [[ $i -lt $total ]]; do
|
|
# Wait for slot
|
|
while [[ ${#pids[@]} -ge $max_parallel ]]; do
|
|
for pid_idx in "${!pids[@]}"; do
|
|
local pid="${pids[$pid_idx]}"
|
|
if ! kill -0 "$pid" 2>/dev/null; then
|
|
wait "$pid"
|
|
local exit_code=$?
|
|
completed=$((completed + 1))
|
|
local percent=$((completed * 100 / total))
|
|
local elapsed=$(($(date +%s) - start_time))
|
|
local eta=0
|
|
[[ $completed -gt 0 ]] && eta=$((elapsed * (total - completed) / completed))
|
|
log_info "Progress: [$percent%] [$completed/$total] ETA: ${eta}s"
|
|
if [[ $exit_code -ne 0 ]]; then
|
|
failed=$((failed + 1))
|
|
fi
|
|
unset pids[$pid_idx]
|
|
fi
|
|
done
|
|
pids=("${pids[@]}")
|
|
sleep 0.2
|
|
done
|
|
|
|
# Start new task
|
|
local args="${args_array[$i]}"
|
|
$func $args &
|
|
local pid=$!
|
|
pids+=("$pid")
|
|
i=$((i + 1))
|
|
done
|
|
|
|
# Wait for remaining
|
|
for pid in "${pids[@]}"; do
|
|
wait "$pid"
|
|
local exit_code=$?
|
|
completed=$((completed + 1))
|
|
local percent=$((completed * 100 / total))
|
|
log_info "Progress: [$percent%] [$completed/$total]"
|
|
if [[ $exit_code -ne 0 ]]; then
|
|
failed=$((failed + 1))
|
|
fi
|
|
done
|
|
|
|
local elapsed=$(($(date +%s) - start_time))
|
|
if [[ $failed -gt 0 ]]; then
|
|
log_error "$failed task(s) failed out of $total (elapsed: ${elapsed}s)"
|
|
return 1
|
|
fi
|
|
|
|
log_success "All $total tasks completed successfully (elapsed: ${elapsed}s)"
|
|
return 0
|
|
}
|
|
|