Correctong boot problems

cryosparc.sh

 #!/bin/bash -x
 
+set -o pipefail,errexit
+
 export PATH=${CRYOSPARC_MASTER_DIR}/bin:${CRYOSPARC_WORKER_DIR}/bin:${CRYOSPARC_MASTER_DIR}/deps/anaconda/bin/:$PATH
 export HOME=${USER_HOMEDIR:-/home/$USER}
 export LSCRATCH=${LSCRATCH:-$CRYOSPARC_DATADIR/scratch}
@@ -25,27 +27,34 @@ fi
 mkdir -p ${LSCRATCH}
 
 export CRYOSPARC_BASE_PORT=${CRYOSPARC_BASE_PORT:-"39000"}
-export CRYOSPARC_SUPERVISOR_SOCK_FILE="${LSCRATCH}/cryosparc-supervisor.sock" 
+export CRYOSPARC_SUPERVISOR_SOCK_FILE="${LSCRATCH}/cryosparc-supervisor.sock"
 
 echo "Starting cryosparc master..."
 cd ${CRYOSPARC_DATADIR}
+
 # modify configuration
 sed -i -e "s/^export CRYOSPARC_MASTER_HOSTNAME=.*$/export CRYOSPARC_MASTER_HOSTNAME=${CRYOSPARC_MASTER_HOSTNAME}/g" ${CRYOSPARC_DATADIR}/config.sh
-sed -i -e "s/^export CRYOSPARC_MASTER_HOSTNAME=.*$/export CRYOSPARC_MASTER_HOSTNAME=${CRYOSPARC_MASTER_HOSTNAME}/g" ${CRYOSPARC_DATADIR}/config.sh
 sed -i -e "s/^export CRYOSPARC_LICENSE_ID=.*$/export CRYOSPARC_LICENSE_ID=${CRYOSPARC_LICENSE_ID}/g" ${CRYOSPARC_DATADIR}/config.sh
 sed -i -e "s|^export CRYOSPARC_DB_PATH=.*$|export CRYOSPARC_DB_PATH=${CRYOSPARC_DATADIR}/cryosparc_database|g" ${CRYOSPARC_DATADIR}/config.sh
 sed -i -e "s/^export CRYOSPARC_BASE_PORT=.*$/export CRYOSPARC_BASE_PORT=${CRYOSPARC_BASE_PORT}/g" ${CRYOSPARC_DATADIR}/config.sh
-#printf "%s\n" "export CRYOSPARC_SUPERVISOR_SOCK_FILE=${CRYOSPARC_SUPERVISOR_SOCK_FILE}" wq | ed -s ${CRYOSPARC_DATADIR}/config.sh
-#printf "%s\n" "export CRYOSPARC_MONGO_EXTRA_FLAGS=\"  --unixSocketPrefix=${LSCRATCH}\"" wq | ed -s ${CRYOSPARC_DATADIR}/config.sh
-echo "export CRYOSPARC_SUPERVISOR_SOCK_FILE=${CRYOSPARC_SUPERVISOR_SOCK_FILE}" >> ${CRYOSPARC_DATADIR}/config.sh
-#echo "export CRYOSPARC_MONGO_EXTRA_FLAGS=\"  --unixSocketPrefix=${LSCRATCH}\"" >> ${CRYOSPARC_DATADIR}/config.sh
-if ! grep -q 'CRYOSPARC_FORCE_HOSTNAME=true' ${CRYOSPARC_DATADIR}/config.sh; then
+
+if ! grep -q 'CRYOSPARC_SUPERVISOR_SOCK_FILE' ${CRYOSPARC_DATADIR}/config.sh; then
+  echo "export CRYOSPARC_SUPERVISOR_SOCK_FILE=${CRYOSPARC_SUPERVISOR_SOCK_FILE}" >> ${CRYOSPARC_DATADIR}/config.sh
+else
+  sed -i -e "s/^export CRYOSPARC_SUPERVISOR_SOCK_FILE=.*$/export CRYOSPARC_SUPERVISOR_SOCK_FILE=${CRYOSPARC_SUPERVISOR_SOCK_FILE}/g" ${CRYOSPARC_DATADIR}/config.sh
+fi
+
+if ! grep -q 'CRYOSPARC_FORCE_HOSTNAME=true' ${CRYOSPARC_MASTER_DIR}/config.sh; then
   echo 'export CRYOSPARC_FORCE_HOSTNAME=true' >> ${CRYOSPARC_DATADIR}/config.sh
 fi
+
 echo '====='
 cat ${CRYOSPARC_DATADIR}/config.sh
 echo '====='
 
+#printf "%s\n" "export CRYOSPARC_MONGO_EXTRA_FLAGS=\"  --unixSocketPrefix=${LSCRATCH}\"" wq | ed -s ${CRYOSPARC_DATADIR}/config.sh
+#echo "export CRYOSPARC_MONGO_EXTRA_FLAGS=\"  --unixSocketPrefix=${LSCRATCH}\"" >> ${CRYOSPARC_DATADIR}/config.sh
+
 # modify mongo path
 #sed -i 's|MONGO_URL="mongodb://%(ENV_CRYOSPARC_MASTER_HOSTNAME)s:%(ENV_CRYOSPARC_MONGO_PORT)s|MONGO_URL=mongodb://cryosparc-fpoitevi:%(ENV_CRYOSPARC_MONGO_PORT)s|g' ${CRYOSPARC_MASTER_DIR}/supervisord.conf
 #sed -i 's|MONGO_OPLOG_URL="mongodb://%(ENV_CRYOSPARC_MASTER_HOSTNAME)s:%(ENV_CRYOSPARC_MONGO_PORT)s|MONGO_OPLOG_URL="mongodb://cryosparc-fpoitevi:%(ENV_CRYOSPARC_MONGO_PORT)s|g' ${CRYOSPARC_MASTER_DIR}/supervisord.conf
@@ -57,7 +66,8 @@ THIS_USER=$(whoami)
 #THIS_USER_SUFFIX=${USER_SUFFIX:-'slac.stanford.edu'}
 #ACCOUNT="${THIS_USER}@${THIS_USER_SUFFIX}"
 ACCOUNT="${THIS_USER}"
-rm -f "${SOCK_FILE}" || true
+
+rm -f "${CRYOSPARC_SUPERVISOR_SOCK_FILE}" || true
 cryosparcm restart
 
 # ensure that the mongo replset is correct
@@ -88,7 +98,12 @@ cryosparcm restart
 echo "Success starting cryosparc master!"
 
 # remove all existing worker threads
-${CRYOSPARC_MASTER_DIR}/bin/cryosparcm cli 'get_scheduler_targets()'  | python -c "import sys, ast, json; print( json.dumps(ast.literal_eval(sys.stdin.readline())) )" | jq '.[].name' | sed 's:"::g' | xargs -n1 -I \{\} ${CRYOSPARC_MASTER_DIR}/bin/cryosparcm cli 'remove_scheduler_target_node("'{}'")'
+${CRYOSPARC_MASTER_DIR}/bin/cryosparcm cli 'get_scheduler_targets()'  | python -c "import sys, ast, json; print( json.dumps(ast.literal_eval(sys.stdin.readline())) )" > /tmp/worker.json
+jq . /tmp/worker.json
+if [ "$(jq '.[]' /tmp/worker.json)" != "" ]
+then
+  jq '.[].name' /tmp/worker.json | sed 's:"::g' | xargs -n1 -I \{\} ${CRYOSPARC_MASTER_DIR}/bin/cryosparcm cli 'remove_scheduler_target_node("'{}'")'
+fi
 
 # add additional job lanes
 if [ "${CRYOSPACE_ADD_JOB_LANES}" == "1" ]; then
@@ -128,6 +143,13 @@ if [ "$CRYOSPARC_TAIL_LOGS" == "1" ]; then
   done
 fi
 
+while :
+do
+  echo 'heartbeat...'
+  sleep 300
+done
+
+
 ###
 # create firefox startup
 ###

slurm/kuma_h100/cluster_info.json

@@ -6,6 +6,5 @@
     "qsub_cmd_tpl" : "/opt/slurm/current/bin/sbatch --chdir /scratch  {{ script_path_abs }}",
     "qstat_cmd_tpl" : "/opt/slurm/current/bin/squeue -j {{ cluster_job_id }}",
     "qdel_cmd_tpl" : "/opt/slurm/current/bin/scancel {{ cluster_job_id }}",
-    "qinfo_cmd_tpl" : "/opt/slurm/current/bin/sinfo",
-    "transfer_cmd_tpl" : "scp {{ src_path }} kuma.hpc.epfl.ch:{{ dest_path }}"
+    "qinfo_cmd_tpl" : "/opt/slurm/current/bin/sinfo"
 }

slurm/kuma_l40s/cluster_info.json

@@ -6,6 +6,5 @@
     "qsub_cmd_tpl" : "/opt/slurm/current/bin/sbatch --chdir /scratch  {{ script_path_abs }}",
     "qstat_cmd_tpl" : "/opt/slurm/current/bin/squeue -j {{ cluster_job_id }}",
     "qdel_cmd_tpl" : "/opt/slurm/current/bin/scancel {{ cluster_job_id }}",
-    "qinfo_cmd_tpl" : "/opt/slurm/current/bin/sinfo",
-    "transfer_cmd_tpl" : "scp {{ src_path }} kuma.hpc.epfl.ch:{{ dest_path }}"
+    "qinfo_cmd_tpl" : "/opt/slurm/current/bin/sinfo"
 }