kindman.org

# Launch Scheduler
$ cd scheduler/
$ mkfifo jobs nodes
$ ./sched.sh
 
# Add nodes to the node pipe
$ echo "10.0.0.1" > nodes &
$ echo "10.0.0.2" > nodes &
 
# Add jobs to the job pipe
$ echo "process /folder/data.txt" > jobs &
$ echo "make -j9" > jobs &
 
# Watch the scheduler :-)

#!/bin/bash
 
# dispatch.sh
#
# This is a job dispatcher to network nodes.
#
# (C) Copyright 2010, Mauricio Faria de Oliveira
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2
# as published by the Free Software Foundation;
 
# Variables
node_pipe="nodes"
 
# Parameters
node="$1"
shift
job="$@"
 
# Checks
#	1. Parameters.
if [ "$node" == "" || "$job" == "" ]; then
    echo "Usage: $0 node job"
    exit 1;
fi
 
# Log
echo ".. (dispatch) node '$node': '$job'"
 
# Dispatch job to node via ssh
ssh "$node" "$job" > /dev/null
 
# Put node back into nodes named pipe
echo "$node" > "$node_pipe"

#!/bin/bash
 
# sched.sh
#
# This is a 2-fifo-blocking job-scheduler to network nodes.
#
# (C) Copyright 2010, Mauricio Faria de Oliveira
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2
# as published by the Free Software Foundation;
 
# Variables
job_pipe="jobs"
node_pipe="nodes"
dispatcher="./dispatch.sh"
 
# Checks
#	1. Check if named pipes exist.
 
if [ ! -p "$job_pipe" -o \
     ! -p "$node_pipe" ];
then
    echo "Check for the fifos 'jobs' and 'nodes' (mkfifo jobs; mkfifo nodes)"
    exit 1;
fi
 
# Algorithm
#
#	1. For each available job ('jobs' named pipe, blocking),
#	2. For an available node ('nodes' named pipe, blocking),
#	3. Dispatch job to node (asynchronously).
#
# Important
#
#	1. The named pipes are NOT guaranteed to be 'fifo'.
#		Jobs and nodes are added/return asynchronously.
#		Only the blocking behavior is guaranteed.
#
#	2. There is no way to remove a single line from a named pipe, [is it?]
#		Then:
#		1. All lines are removed,
#		2. First line is used,
#		3. Other lines are put back (asynchronously).
#
#	3. If the 'cat <> jobs' hangs for even a little while
#		jobs in this little while are lost. [why?]
#		So things are asynchronous inside this loop.
#
#	4. Never use >> (append) asynchronously with named pipes.
#		It _seems_ append on pipes works as follows:
#		1. Buffer contents
#		2. Add new stuff to buffer
#		3. Overwrite contents with buffer
#
#		If something is removed (i.e. 'cat') between steps 1 and 3,
#		it will be inserted back on step 3, due to content buffering.
#
###############################################################################
# Code                                                                        #
###############################################################################
 
#	1.	For each available job ('jobs' named pipe, blocking),
cat <> "$job_pipe" | while read job; do
 
    echo
    echo "(job): $job"
 
    # 2. For an available node ('nodes' named pipe, blocking),
 
    # 2.1. All lines are removed,
    linecount=0
    cat "$node_pipe" | while read node; do
        echo -n ".. (node): $node (#${linecount}) "
 
        # 2.2. First line is used,
        if [ "$linecount" == "0" ]; then
 
            # 3. Dispatch job to node (asynchronously).
            echo "(dispatch)"
            $dispatcher $node $job &
 
        # 2.3. Other lines are put back.
        else
            echo "(put back)"
            echo "$node" > nodes &
        fi
 
        linecount=$(($linecount + 1))
 
    done
 
    echo
 
done