openfoam.slurm

#!/bin/bash

#SBATCH -t 24:00:00

module load openfoam/last

homedir=$PWD

scratchdir=/scratch/$USER/$SLURM_JOB_ID

nodefile=nodes_$SLURM_JOB_ID.txt

srun hostname | sort -u > $nodefile

# creating scratch folders on the nodes
for node in `cat $nodefile`
do
    ssh $node mkdir -p $scratchdir
    rsync -a --exclude='slurm-${SLURM_JOB_ID}.out' $homedir/ $node:$scratchdir/
done

cd $scratchdir

export SCRATCH=$scratchdir

################# case prep ###################

diam=2

# getting viscosity nu from transportProperties file, otherwise just define it here
# visc=0.01

visc=`cat template/constant/transportProperties | grep nu | cut -d']' -f2 | sed 's/ //g' | sed 's/;//g'`

# read Re as a parameter to openfoam.slurm, e.g. when submitted as "sbatch .... openfoam.slurm 100"
Re=$1

echo Re=$Re visc=$visc diam=$diam

# calculate velocity from Re number
velo=`echo $Re/$diam*$visc | bc`

echo velo=$velo

cp -r template Re_$Re

cat template/U1 > Re_$Re/0/U
echo "        value           uniform ($velo 0 0);" >> Re_$Re/0/U
cat template/U2 >> Re_$Re/0/U

cd Re_$Re

###### Finally run the case ###########
. ${WM_PROJECT_DIR:?}/bin/tools/RunFunctions
solver=$(getApplication)

#foamGet Allclean
#chmod +x Allclean
#./Allclean
blockMesh
decomposePar

cpus=$SLURM_NTASKS

mpirun -np $cpus $solver -parallel

./reconstruct_parallel.sh

##########################

rm -fr processor*
touch Re=$Re.foam

cd ..

##### compressing results ##########
tar cvf Re_$Re.tar Re_$Re

pbzip2 -z Re_$Re.tar

rm -fr Re_$Re


######## copying results archives' back to home and clearing scratch folders ##############
cd $homedir

for node in `cat $nodefile`
do
    rsync -a --exclude='slurm-${SLURM_JOB_ID}.out' $node:/$scratchdir/*.tar.* $homedir/
    ssh $node "rm -fr $scratchdir"
done