#!/bin/bash
#                    forkxds          Version 5-2017, 7-2017
#
# enables  multi-tasking by splitting the COLSPOT and INTEGRATE
# steps of xds into independent jobs. Each job is carried out by 
# a Fortran main program (mcolspot, mcolspot_par, mintegrate, or
# mintegrate_par). The jobs are distributed among the processor 
# nodes of the NFS cluster network.
#
# 'forkxds' is called by xds or xds_par by the Fortran instruction
# CALL SYSTEM('forkxds ntask maxcpu main rhosts'),
#    ntask  ::total number of independent jobs (tasks)
#   maxcpu  ::maximum number of processors used by each job
#    main   ::name of the main program to be executed; could be
#             mcolspot | mcolspot_par | mintegrate | mintegrate_par
#   rhosts  ::names of CPU cluster nodes in the NFS network 
#
# NOTE, that this can only work if the main program and this shell
# script are correctly installed in the search path for executables.
#
# NOTE: No blanks allowed adjacent to the = signs !!!
#
# W.Kabsch & K.Rohm    original version Februar 2005
# K.Diederichs 9-2005  modification for execution on remote hosts
# W.Kabsch 5-2017      simplified version replacing the old scripts
#                      (forkcolspot,forkintegrate); more flexibility
#                      by using names of the cluster nodes as input
#                      arguments of this shell script
#         10-2017      sync added after $amain to complete pending 
#                      disk writes on each node (suggested by M. Fodje)

ntask=$1  #total number of jobs
maxcpu=$2 #maximum number of processors used by each job
main=$3   #name of the main program to be executed
#extract names of cluster nodes from the script command line
shift 3
nhosts=0                       #changed from nhosts=1 (2020/02/11)
while (( "$#" )); do
   rhosts[$nhosts]=$1          #name of next processor node
   nhosts=$(( nhosts + 1 ))    #number of cluster processor nodes
   shift
done
#echo "$ntask $maxcpu $main $nhosts"
#echo ${rhosts[@]}
amain=`which $main`
pids=""                        #list of background process ID's
itask=1
while test $itask -le $ntask
do
   if [ $nhosts -gt 1 ]        #distribute jobs among the cluster nodes
   then
      j=$(( (itask -1) % nhosts )) #changed from % nhosts + 1 to nhosts
      echo "$itask" | ssh -x ${rhosts[$j]} "cd $PWD && $amain && sync" &
   else
      echo "$itask" | $amain && sync &  #submit all jobs to the peer node
   fi
   pids="$pids $!"             #append id of the new background process
   itask=`expr $itask + 1`
   # NOTE: sync after $amain complete pending disk writes on each node
done

trap "kill -15 $pids" 2 15     # 2:Control-C; 15:kill
wait                           #wait for all background processes issued
