generalParallel

#!/usr/bin/env bash
#########################################################################
# File Name: generalParallel
# Author: C.J. Liu
# Mail: samliu@hust.edu.cn
# Created Time: Tue 06 Dec 2016 04:39:49 PM CST
# Description: Run shell commands in parallel from a command file
#########################################################################

# Default values
DEFAULT_THREADS=20
MAX_RECOMMENDED_THREADS=50

# Parse arguments
command_file="$1"
num_threads="${2:-$DEFAULT_THREADS}"

function show_usage() {
	cat <<EOF
Usage: generalParallel <command_file> [num_threads]

Description:
    This script runs shell commands in parallel by reading them from a file.
    Each line in the command file should contain one shell command.
    Empty lines and lines starting with '#' are ignored.

Arguments:
    command_file    File containing commands to execute (one per line)
    num_threads     Number of parallel threads (default: $DEFAULT_THREADS)

Examples:
    generalParallel commands.txt
    generalParallel my_tasks.sh 10
    generalParallel job_list.txt 30
EOF
}

function validate_arguments() {
	# Check number of arguments
	if [ $# -lt 1 ] || [ $# -gt 2 ]; then
		echo "Error: Invalid number of arguments"
		show_usage
		exit 1
	fi

	# Check if command file exists
	if [ ! -f "$command_file" ]; then
		echo "Error: Command file '$command_file' not found"
		exit 1
	fi

	# Check if command file is readable
	if [ ! -r "$command_file" ]; then
		echo "Error: Command file '$command_file' is not readable"
		exit 1
	fi

	# Validate thread count
	if ! [[ "$num_threads" =~ ^[0-9]+$ ]] || [ "$num_threads" -lt 1 ]; then
		echo "Error: Number of threads must be a positive integer"
		exit 1
	fi

	# Warning for high thread count
	if [ "$num_threads" -gt "$MAX_RECOMMENDED_THREADS" ]; then
		echo "Warning: Thread count ($num_threads) is quite high. Consider using fewer threads to avoid system overload."
	fi
}

function setup_job_control() {
	# Create temporary directory and FIFO for job control
	local tmp_dir="${HOME}/tmp"
	mkdir -p "$tmp_dir"

	job_pipe="${tmp_dir}/parallel_$$.fifo"
	mkfifo "$job_pipe"
	exec 6<>"$job_pipe"
	rm -f "$job_pipe"

	# Initialize job slots (semaphore pattern)
	for ((i = 0; i < num_threads; i++)); do
		echo "slot" >&6
	done
}

function cleanup() {
	exec 6>&- 2>/dev/null
	rm -f "$job_pipe" 2>/dev/null
}

function run_parallel_jobs() {
	local job_count=0
	local line_number=0

	echo "Processing commands from: $command_file"
	echo "Using $num_threads parallel threads"
	echo "Started at: $(date)"
	echo "----------------------------------------"

	while IFS= read -r command || [ -n "$command" ]; do
		((line_number++))

		# Skip empty lines and comments
		if [[ -z "$command" || "$command" =~ ^[[:space:]]*# ]]; then
			continue
		fi

		# Wait for an available job slot
		read -u6

		# Run command in background
		{
			# Remove trailing & if present
			command="${command%&}"
			command="${command## }" # Remove leading spaces

			echo "[$(date +%T)] Line $line_number: Starting: $command"

			if eval "$command"; then
				echo "[$(date +%T)] Line $line_number: ✓ Completed: $command"
			else
				echo "[$(date +%T)] Line $line_number: ✗ Failed: $command"
			fi

			# Release job slot
			echo "slot" >&6
		} &

		((job_count++))
	done <"$command_file"

	# Wait for all background jobs to complete
	wait

	echo "----------------------------------------"
	echo "All jobs completed at: $(date)"
	echo "Total commands processed: $job_count"
}

# Main execution
trap cleanup EXIT

# Validate input
validate_arguments "$@"

# Setup parallel job control
setup_job_control

# Run the parallel jobs
run_parallel_jobs

echo "Parallel execution finished successfully"