#!/bin/bash

# if [ "$#" -ne 1 ]; then
#     echo "Usage: ${0} <xml input file>"
# fi

module load cudatoolkit/12.4
module load gcc-native/13.2

f=${1}
fname=$(basename -- ${f})
ngpus=${2}
nvecs=${3}

batchsize=16
prefetch_batchsize=32
if [ "$#" -gt 3 ]; then
    batchsize="$4"
    prefetch_batchsize=$(expr $batchsize \* 2)
fi  

basedir=/global/homes/r/roguzsel/gcp
codedir=/global/homes/r/roguzsel/gcp/code-v01
datadir=/global/homes/r/roguzsel/gcp/data

# all redstar params
# NPT_EDGE_COST_METHOD
# NPT_PREF_MULTI_EDGE_CONTRACT
# NPT_EDGE_METHOD
# NPT_GR_NUM
# NPT_BATCH_SIZE
# NPT_BATCH_PREFETCH_SIZE
# NPT_USE_METANODES
# NPT_EXP_SCHEDULER
# NPT_DAG_PART_SIZE
# NPT_FAST_CONTRACTALL
# NPT_PREFETCH_ELEMENTALS

# settings for perlmutter cpu nodes
export OMP_NUM_THREADS=64
export KMP_AFFINITY=verbose,scatter,granularity=fine
echo "#gpus ${ngpus}"
if [ "$batchsize" -eq 1 ]; then
    echo "NPT_BATCH_SIZE 1"
    export NPT_BATCH_SIZE=1
else
    echo "NPT_BATCH_SIZE ${batchsize}"
    export NPT_BATCH_SIZE=${batchsize}
    export NPT_BATCH_PREFETCH_SIZE=${prefetch_batchsize}
fi

if [ "${ngpus}" -eq 1 ]; then
    echo "Setting visible devices to 0"
    export CUDA_VISIBLE_DEVICES=0
elif [ "${ngpus}" -eq 2 ]; then
    echo "Setting visible devices to 0,1"
    export CUDA_VISIBLE_DEVICES=0,1
elif [ "${ngpus}" -eq 4 ]; then
    echo "Setting visible devices to 0,1,2,3"
    export CUDA_VISIBLE_DEVICES=0,1,2,3
fi

echo "num_vecs ${nvecs}"
sed -i "s/<num_vecs>.*<\/num_vecs>/<num_vecs>${nvecs}<\/num_vecs>/g" ${1}

# other redstar params
export NPT_USE_METANODES="YES"

# data placement choose from SMART SIMPLE GP (default SMART)
# export HADRON_DATA_PLACEMENT=GP

# edge cost method (unset for default, which is EDGE_AVERAGE_COST)
# export NPT_EDGE_COST_METHOD="NUMBER_APPEARANCE"
# NPT_EDGE_COST_METHOD  "NUMBER_APPEARANCE"
# NPT_EDGE_METHOD "SORT_DECREASE" "RANDOM" "SORT_INCREASE" "DO_NOTHING"

# df sb nb tb
# export scheduler=df
sch="df"

origdir=${pwd}
cd ${codedir}/redstar_kanoni

nexps=$(find -L 0-runlogs-0 -name "${fname}-*" -type d -printf '%f\n' | wc -l)
tmp=$(expr ${nexps} + 1)
tmp=$(printf "%03d" ${tmp})
outdir=0-runlogs-0/${fname}-${tmp}
mkdir ${outdir}
echo "out dir: ${outdir}"

TIMEFORMAT=%R

echo "running corr graph generation..."
# echo "../redstar/redstar/bin/redstar_corr_graph ${fname} out.xml > ${fname}.corr-graph.runlog 2>&1"
time ../redstar/redstar/bin/redstar_corr_graph ${fname} out.xml > ${fname}.corr-graph.runlog 2>&1

echo "running npt..."
# echo "../redstar/redstar/bin/redstar_npt ${fname} out.xml > ${fname}.npt.runlog 2>&1"
time ../redstar/redstar/bin/redstar_npt ${fname} out.xml > ${fname}.npt.runlog 2>&1

mv corr_graph.bin out.xml ${fname}.corr-graph.runlog ${fname}.npt.runlog interior_nodes*log ${outdir}

if [ -f "output.npt.sdb1" ]; then
    mv output.npt.sdb1 ${outdir}
fi

cp ${outdir}/${fname}.npt.runlog ${fname}.${sch}.${tmp}.npt.runlog
if [ -f "sq_cur_alive-size" ]; then
    mv "sq_cur_alive-size" "${fname}.${sch}.${tmp}.sq_cur_alive-size"
fi

if [ -f "sq_cur_alive-count" ]; then
    mv "sq_cur_alive-count" "${fname}.${sch}.${tmp}.sq_cur_alive-count"
fi

if [ -f "contraction-trace" ]; then
    mv "contraction-trace" "${fname}.${sch}.${tmp}.contraction-trace"
fi

if [ -f "sch-trace" ]; then
    mv "sch-trace" "${fname}.${sch}.${tmp}.sch-trace"
fi

if [ -f "ctr-details" ]; then
    mv "ctr-details" "${fname}.${sch}.${tmp}.ctr-details"
fi


cd ${origdir}

exit 0
