Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
C
Cryosparc Docker
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Package registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
SCITAS
Software stack
Cryosparc Docker
Commits
5de4ca44
Commit
5de4ca44
authored
4 months ago
by
Nicolas Richart
Browse files
Options
Downloads
Patches
Plain Diff
Correctong boot problems
parent
99b4ba71
No related branches found
No related tags found
No related merge requests found
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
cryosparc.sh
+31
-9
31 additions, 9 deletions
cryosparc.sh
slurm/kuma_h100/cluster_info.json
+1
-2
1 addition, 2 deletions
slurm/kuma_h100/cluster_info.json
slurm/kuma_l40s/cluster_info.json
+1
-2
1 addition, 2 deletions
slurm/kuma_l40s/cluster_info.json
with
33 additions
and
13 deletions
cryosparc.sh
+
31
−
9
View file @
5de4ca44
#!/bin/bash -x
set
-o
pipefail,errexit
export
PATH
=
${
CRYOSPARC_MASTER_DIR
}
/bin:
${
CRYOSPARC_WORKER_DIR
}
/bin:
${
CRYOSPARC_MASTER_DIR
}
/deps/anaconda/bin/:
$PATH
export
HOME
=
${
USER_HOMEDIR
:-
/home/
$USER
}
export
LSCRATCH
=
${
LSCRATCH
:-
$CRYOSPARC_DATADIR
/scratch
}
...
...
@@ -25,27 +27,34 @@ fi
mkdir
-p
${
LSCRATCH
}
export
CRYOSPARC_BASE_PORT
=
${
CRYOSPARC_BASE_PORT
:-
"39000"
}
export
CRYOSPARC_SUPERVISOR_SOCK_FILE
=
"
${
LSCRATCH
}
/cryosparc-supervisor.sock"
export
CRYOSPARC_SUPERVISOR_SOCK_FILE
=
"
${
LSCRATCH
}
/cryosparc-supervisor.sock"
echo
"Starting cryosparc master..."
cd
${
CRYOSPARC_DATADIR
}
# modify configuration
sed
-i
-e
"s/^export CRYOSPARC_MASTER_HOSTNAME=.*
$/
export CRYOSPARC_MASTER_HOSTNAME=
${
CRYOSPARC_MASTER_HOSTNAME
}
/g"
${
CRYOSPARC_DATADIR
}
/config.sh
sed
-i
-e
"s/^export CRYOSPARC_MASTER_HOSTNAME=.*
$/
export CRYOSPARC_MASTER_HOSTNAME=
${
CRYOSPARC_MASTER_HOSTNAME
}
/g"
${
CRYOSPARC_DATADIR
}
/config.sh
sed
-i
-e
"s/^export CRYOSPARC_LICENSE_ID=.*
$/
export CRYOSPARC_LICENSE_ID=
${
CRYOSPARC_LICENSE_ID
}
/g"
${
CRYOSPARC_DATADIR
}
/config.sh
sed
-i
-e
"s|^export CRYOSPARC_DB_PATH=.*
$|
export CRYOSPARC_DB_PATH=
${
CRYOSPARC_DATADIR
}
/cryosparc_database|g"
${
CRYOSPARC_DATADIR
}
/config.sh
sed
-i
-e
"s/^export CRYOSPARC_BASE_PORT=.*
$/
export CRYOSPARC_BASE_PORT=
${
CRYOSPARC_BASE_PORT
}
/g"
${
CRYOSPARC_DATADIR
}
/config.sh
#printf "%s\n" "export CRYOSPARC_SUPERVISOR_SOCK_FILE=${CRYOSPARC_SUPERVISOR_SOCK_FILE}" wq | ed -s ${CRYOSPARC_DATADIR}/config.sh
#printf "%s\n" "export CRYOSPARC_MONGO_EXTRA_FLAGS=\" --unixSocketPrefix=${LSCRATCH}\"" wq | ed -s ${CRYOSPARC_DATADIR}/config.sh
echo
"export CRYOSPARC_SUPERVISOR_SOCK_FILE=
${
CRYOSPARC_SUPERVISOR_SOCK_FILE
}
"
>>
${
CRYOSPARC_DATADIR
}
/config.sh
#echo "export CRYOSPARC_MONGO_EXTRA_FLAGS=\" --unixSocketPrefix=${LSCRATCH}\"" >> ${CRYOSPARC_DATADIR}/config.sh
if
!
grep
-q
'CRYOSPARC_FORCE_HOSTNAME=true'
${
CRYOSPARC_DATADIR
}
/config.sh
;
then
if
!
grep
-q
'CRYOSPARC_SUPERVISOR_SOCK_FILE'
${
CRYOSPARC_DATADIR
}
/config.sh
;
then
echo
"export CRYOSPARC_SUPERVISOR_SOCK_FILE=
${
CRYOSPARC_SUPERVISOR_SOCK_FILE
}
"
>>
${
CRYOSPARC_DATADIR
}
/config.sh
else
sed
-i
-e
"s/^export CRYOSPARC_SUPERVISOR_SOCK_FILE=.*
$/
export CRYOSPARC_SUPERVISOR_SOCK_FILE=
${
CRYOSPARC_SUPERVISOR_SOCK_FILE
}
/g"
${
CRYOSPARC_DATADIR
}
/config.sh
fi
if
!
grep
-q
'CRYOSPARC_FORCE_HOSTNAME=true'
${
CRYOSPARC_MASTER_DIR
}
/config.sh
;
then
echo
'export CRYOSPARC_FORCE_HOSTNAME=true'
>>
${
CRYOSPARC_DATADIR
}
/config.sh
fi
echo
'====='
cat
${
CRYOSPARC_DATADIR
}
/config.sh
echo
'====='
#printf "%s\n" "export CRYOSPARC_MONGO_EXTRA_FLAGS=\" --unixSocketPrefix=${LSCRATCH}\"" wq | ed -s ${CRYOSPARC_DATADIR}/config.sh
#echo "export CRYOSPARC_MONGO_EXTRA_FLAGS=\" --unixSocketPrefix=${LSCRATCH}\"" >> ${CRYOSPARC_DATADIR}/config.sh
# modify mongo path
#sed -i 's|MONGO_URL="mongodb://%(ENV_CRYOSPARC_MASTER_HOSTNAME)s:%(ENV_CRYOSPARC_MONGO_PORT)s|MONGO_URL=mongodb://cryosparc-fpoitevi:%(ENV_CRYOSPARC_MONGO_PORT)s|g' ${CRYOSPARC_MASTER_DIR}/supervisord.conf
#sed -i 's|MONGO_OPLOG_URL="mongodb://%(ENV_CRYOSPARC_MASTER_HOSTNAME)s:%(ENV_CRYOSPARC_MONGO_PORT)s|MONGO_OPLOG_URL="mongodb://cryosparc-fpoitevi:%(ENV_CRYOSPARC_MONGO_PORT)s|g' ${CRYOSPARC_MASTER_DIR}/supervisord.conf
...
...
@@ -57,7 +66,8 @@ THIS_USER=$(whoami)
#THIS_USER_SUFFIX=${USER_SUFFIX:-'slac.stanford.edu'}
#ACCOUNT="${THIS_USER}@${THIS_USER_SUFFIX}"
ACCOUNT
=
"
${
THIS_USER
}
"
rm
-f
"
${
SOCK_FILE
}
"
||
true
rm
-f
"
${
CRYOSPARC_SUPERVISOR_SOCK_FILE
}
"
||
true
cryosparcm restart
# ensure that the mongo replset is correct
...
...
@@ -88,7 +98,12 @@ cryosparcm restart
echo
"Success starting cryosparc master!"
# remove all existing worker threads
${
CRYOSPARC_MASTER_DIR
}
/bin/cryosparcm cli
'get_scheduler_targets()'
| python
-c
"import sys, ast, json; print( json.dumps(ast.literal_eval(sys.stdin.readline())) )"
| jq
'.[].name'
|
sed
's:"::g'
| xargs
-n1
-I
\{\}
${
CRYOSPARC_MASTER_DIR
}
/bin/cryosparcm cli
'remove_scheduler_target_node("'
{}
'")'
${
CRYOSPARC_MASTER_DIR
}
/bin/cryosparcm cli
'get_scheduler_targets()'
| python
-c
"import sys, ast, json; print( json.dumps(ast.literal_eval(sys.stdin.readline())) )"
>
/tmp/worker.json
jq
.
/tmp/worker.json
if
[
"
$(
jq
'.[]'
/tmp/worker.json
)
"
!=
""
]
then
jq
'.[].name'
/tmp/worker.json |
sed
's:"::g'
| xargs
-n1
-I
\{\}
${
CRYOSPARC_MASTER_DIR
}
/bin/cryosparcm cli
'remove_scheduler_target_node("'
{}
'")'
fi
# add additional job lanes
if
[
"
${
CRYOSPACE_ADD_JOB_LANES
}
"
==
"1"
]
;
then
...
...
@@ -128,6 +143,13 @@ if [ "$CRYOSPARC_TAIL_LOGS" == "1" ]; then
done
fi
while
:
do
echo
'heartbeat...'
sleep
300
done
###
# create firefox startup
###
...
...
This diff is collapsed.
Click to expand it.
slurm/kuma_h100/cluster_info.json
+
1
−
2
View file @
5de4ca44
...
...
@@ -6,6 +6,5 @@
"qsub_cmd_tpl"
:
"/opt/slurm/current/bin/sbatch --chdir /scratch {{ script_path_abs }}"
,
"qstat_cmd_tpl"
:
"/opt/slurm/current/bin/squeue -j {{ cluster_job_id }}"
,
"qdel_cmd_tpl"
:
"/opt/slurm/current/bin/scancel {{ cluster_job_id }}"
,
"qinfo_cmd_tpl"
:
"/opt/slurm/current/bin/sinfo"
,
"transfer_cmd_tpl"
:
"scp {{ src_path }} kuma.hpc.epfl.ch:{{ dest_path }}"
"qinfo_cmd_tpl"
:
"/opt/slurm/current/bin/sinfo"
}
This diff is collapsed.
Click to expand it.
slurm/kuma_l40s/cluster_info.json
+
1
−
2
View file @
5de4ca44
...
...
@@ -6,6 +6,5 @@
"qsub_cmd_tpl"
:
"/opt/slurm/current/bin/sbatch --chdir /scratch {{ script_path_abs }}"
,
"qstat_cmd_tpl"
:
"/opt/slurm/current/bin/squeue -j {{ cluster_job_id }}"
,
"qdel_cmd_tpl"
:
"/opt/slurm/current/bin/scancel {{ cluster_job_id }}"
,
"qinfo_cmd_tpl"
:
"/opt/slurm/current/bin/sinfo"
,
"transfer_cmd_tpl"
:
"scp {{ src_path }} kuma.hpc.epfl.ch:{{ dest_path }}"
"qinfo_cmd_tpl"
:
"/opt/slurm/current/bin/sinfo"
}
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment