Skip to content
Snippets Groups Projects
Commit 5de4ca44 authored by Nicolas Richart's avatar Nicolas Richart
Browse files

Correctong boot problems

parent 99b4ba71
No related branches found
No related tags found
No related merge requests found
#!/bin/bash -x
set -o pipefail,errexit
export PATH=${CRYOSPARC_MASTER_DIR}/bin:${CRYOSPARC_WORKER_DIR}/bin:${CRYOSPARC_MASTER_DIR}/deps/anaconda/bin/:$PATH
export HOME=${USER_HOMEDIR:-/home/$USER}
export LSCRATCH=${LSCRATCH:-$CRYOSPARC_DATADIR/scratch}
......@@ -25,27 +27,34 @@ fi
mkdir -p ${LSCRATCH}
export CRYOSPARC_BASE_PORT=${CRYOSPARC_BASE_PORT:-"39000"}
export CRYOSPARC_SUPERVISOR_SOCK_FILE="${LSCRATCH}/cryosparc-supervisor.sock"
export CRYOSPARC_SUPERVISOR_SOCK_FILE="${LSCRATCH}/cryosparc-supervisor.sock"
echo "Starting cryosparc master..."
cd ${CRYOSPARC_DATADIR}
# modify configuration
sed -i -e "s/^export CRYOSPARC_MASTER_HOSTNAME=.*$/export CRYOSPARC_MASTER_HOSTNAME=${CRYOSPARC_MASTER_HOSTNAME}/g" ${CRYOSPARC_DATADIR}/config.sh
sed -i -e "s/^export CRYOSPARC_MASTER_HOSTNAME=.*$/export CRYOSPARC_MASTER_HOSTNAME=${CRYOSPARC_MASTER_HOSTNAME}/g" ${CRYOSPARC_DATADIR}/config.sh
sed -i -e "s/^export CRYOSPARC_LICENSE_ID=.*$/export CRYOSPARC_LICENSE_ID=${CRYOSPARC_LICENSE_ID}/g" ${CRYOSPARC_DATADIR}/config.sh
sed -i -e "s|^export CRYOSPARC_DB_PATH=.*$|export CRYOSPARC_DB_PATH=${CRYOSPARC_DATADIR}/cryosparc_database|g" ${CRYOSPARC_DATADIR}/config.sh
sed -i -e "s/^export CRYOSPARC_BASE_PORT=.*$/export CRYOSPARC_BASE_PORT=${CRYOSPARC_BASE_PORT}/g" ${CRYOSPARC_DATADIR}/config.sh
#printf "%s\n" "export CRYOSPARC_SUPERVISOR_SOCK_FILE=${CRYOSPARC_SUPERVISOR_SOCK_FILE}" wq | ed -s ${CRYOSPARC_DATADIR}/config.sh
#printf "%s\n" "export CRYOSPARC_MONGO_EXTRA_FLAGS=\" --unixSocketPrefix=${LSCRATCH}\"" wq | ed -s ${CRYOSPARC_DATADIR}/config.sh
echo "export CRYOSPARC_SUPERVISOR_SOCK_FILE=${CRYOSPARC_SUPERVISOR_SOCK_FILE}" >> ${CRYOSPARC_DATADIR}/config.sh
#echo "export CRYOSPARC_MONGO_EXTRA_FLAGS=\" --unixSocketPrefix=${LSCRATCH}\"" >> ${CRYOSPARC_DATADIR}/config.sh
if ! grep -q 'CRYOSPARC_FORCE_HOSTNAME=true' ${CRYOSPARC_DATADIR}/config.sh; then
if ! grep -q 'CRYOSPARC_SUPERVISOR_SOCK_FILE' ${CRYOSPARC_DATADIR}/config.sh; then
echo "export CRYOSPARC_SUPERVISOR_SOCK_FILE=${CRYOSPARC_SUPERVISOR_SOCK_FILE}" >> ${CRYOSPARC_DATADIR}/config.sh
else
sed -i -e "s/^export CRYOSPARC_SUPERVISOR_SOCK_FILE=.*$/export CRYOSPARC_SUPERVISOR_SOCK_FILE=${CRYOSPARC_SUPERVISOR_SOCK_FILE}/g" ${CRYOSPARC_DATADIR}/config.sh
fi
if ! grep -q 'CRYOSPARC_FORCE_HOSTNAME=true' ${CRYOSPARC_MASTER_DIR}/config.sh; then
echo 'export CRYOSPARC_FORCE_HOSTNAME=true' >> ${CRYOSPARC_DATADIR}/config.sh
fi
echo '====='
cat ${CRYOSPARC_DATADIR}/config.sh
echo '====='
#printf "%s\n" "export CRYOSPARC_MONGO_EXTRA_FLAGS=\" --unixSocketPrefix=${LSCRATCH}\"" wq | ed -s ${CRYOSPARC_DATADIR}/config.sh
#echo "export CRYOSPARC_MONGO_EXTRA_FLAGS=\" --unixSocketPrefix=${LSCRATCH}\"" >> ${CRYOSPARC_DATADIR}/config.sh
# modify mongo path
#sed -i 's|MONGO_URL="mongodb://%(ENV_CRYOSPARC_MASTER_HOSTNAME)s:%(ENV_CRYOSPARC_MONGO_PORT)s|MONGO_URL=mongodb://cryosparc-fpoitevi:%(ENV_CRYOSPARC_MONGO_PORT)s|g' ${CRYOSPARC_MASTER_DIR}/supervisord.conf
#sed -i 's|MONGO_OPLOG_URL="mongodb://%(ENV_CRYOSPARC_MASTER_HOSTNAME)s:%(ENV_CRYOSPARC_MONGO_PORT)s|MONGO_OPLOG_URL="mongodb://cryosparc-fpoitevi:%(ENV_CRYOSPARC_MONGO_PORT)s|g' ${CRYOSPARC_MASTER_DIR}/supervisord.conf
......@@ -57,7 +66,8 @@ THIS_USER=$(whoami)
#THIS_USER_SUFFIX=${USER_SUFFIX:-'slac.stanford.edu'}
#ACCOUNT="${THIS_USER}@${THIS_USER_SUFFIX}"
ACCOUNT="${THIS_USER}"
rm -f "${SOCK_FILE}" || true
rm -f "${CRYOSPARC_SUPERVISOR_SOCK_FILE}" || true
cryosparcm restart
# ensure that the mongo replset is correct
......@@ -88,7 +98,12 @@ cryosparcm restart
echo "Success starting cryosparc master!"
# remove all existing worker threads
${CRYOSPARC_MASTER_DIR}/bin/cryosparcm cli 'get_scheduler_targets()' | python -c "import sys, ast, json; print( json.dumps(ast.literal_eval(sys.stdin.readline())) )" | jq '.[].name' | sed 's:"::g' | xargs -n1 -I \{\} ${CRYOSPARC_MASTER_DIR}/bin/cryosparcm cli 'remove_scheduler_target_node("'{}'")'
${CRYOSPARC_MASTER_DIR}/bin/cryosparcm cli 'get_scheduler_targets()' | python -c "import sys, ast, json; print( json.dumps(ast.literal_eval(sys.stdin.readline())) )" > /tmp/worker.json
jq . /tmp/worker.json
if [ "$(jq '.[]' /tmp/worker.json)" != "" ]
then
jq '.[].name' /tmp/worker.json | sed 's:"::g' | xargs -n1 -I \{\} ${CRYOSPARC_MASTER_DIR}/bin/cryosparcm cli 'remove_scheduler_target_node("'{}'")'
fi
# add additional job lanes
if [ "${CRYOSPACE_ADD_JOB_LANES}" == "1" ]; then
......@@ -128,6 +143,13 @@ if [ "$CRYOSPARC_TAIL_LOGS" == "1" ]; then
done
fi
while :
do
echo 'heartbeat...'
sleep 300
done
###
# create firefox startup
###
......
......@@ -6,6 +6,5 @@
"qsub_cmd_tpl" : "/opt/slurm/current/bin/sbatch --chdir /scratch {{ script_path_abs }}",
"qstat_cmd_tpl" : "/opt/slurm/current/bin/squeue -j {{ cluster_job_id }}",
"qdel_cmd_tpl" : "/opt/slurm/current/bin/scancel {{ cluster_job_id }}",
"qinfo_cmd_tpl" : "/opt/slurm/current/bin/sinfo",
"transfer_cmd_tpl" : "scp {{ src_path }} kuma.hpc.epfl.ch:{{ dest_path }}"
"qinfo_cmd_tpl" : "/opt/slurm/current/bin/sinfo"
}
......@@ -6,6 +6,5 @@
"qsub_cmd_tpl" : "/opt/slurm/current/bin/sbatch --chdir /scratch {{ script_path_abs }}",
"qstat_cmd_tpl" : "/opt/slurm/current/bin/squeue -j {{ cluster_job_id }}",
"qdel_cmd_tpl" : "/opt/slurm/current/bin/scancel {{ cluster_job_id }}",
"qinfo_cmd_tpl" : "/opt/slurm/current/bin/sinfo",
"transfer_cmd_tpl" : "scp {{ src_path }} kuma.hpc.epfl.ch:{{ dest_path }}"
"qinfo_cmd_tpl" : "/opt/slurm/current/bin/sinfo"
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment