Training/srun.sh at main · PictSure/Training · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
# test
srun \
--container-mounts=/netscratch/$USER:/netscratch/$USER,"`pwd`":"`pwd`",/ds:/ds \
--container-workdir="`pwd`" \
--container-image=/enroot/nvcr.io_nvidia_pytorch_23.06-py3.sqsh \
--task-prolog="`pwd`/install.sh" \
--gpus=1 --cpus-per-gpu=4 --partition=H100-SEE --mem=190000 --job-name=icltest \
python3 train.py --config ./configs/slurm.yaml

# S-Model
srun \
--container-mounts=/netscratch/$USER:/netscratch/$USER,"`pwd`":"`pwd`",/ds:/ds \
--container-workdir="`pwd`" \
--container-image=/enroot/nvcr.io_nvidia_pytorch_23.06-py3.sqsh \
--task-prolog="`pwd`/install.sh" \
--gpus=1 --cpus-per-gpu=16 --partition=H100-SEE --mem=220000 --job-name=pictsures \
python3 train.py --config ./configs/models/PictSureSResPre.yaml

# M-Model
srun \
--container-mounts=/netscratch/$USER:/netscratch/$USER,"`pwd`":"`pwd`",/ds:/ds \
--container-workdir="`pwd`" \
--container-image=/enroot/nvcr.io_nvidia_pytorch_23.06-py3.sqsh \
--task-prolog="`pwd`/install.sh" \
--gpus=1 --cpus-per-gpu=16 --partition=H100-SEE --mem-per-cpu=20GB --job-name=pictsurem  \
python3 train.py --config ./configs/models/PictSureM.yaml

# L-Model
srun \
--container-mounts=/netscratch/$USER:/netscratch/$USER,"`pwd`":"`pwd`",/ds:/ds \
--container-workdir="`pwd`" \
--container-image=/enroot/nvcr.io_nvidia_pytorch_23.06-py3.sqsh \
--task-prolog="`pwd`/install.sh" \
--gpus=1 --cpus-per-gpu=16 --partition=A100-80GB --mem-per-cpu=25G --job-name=pictsurel --time=3-00:00 \
python3 train.py --config ./configs/models/PictSureL.yaml

srun \
--container-mounts=/netscratch/$USER:/netscratch/$USER,"`pwd`":"`pwd`" \
--container-workdir="`pwd`" \
--container-image=/enroot/nvcr.io_nvidia_pytorch_23.06-py3.sqsh \
--task-prolog="`pwd`/install.sh" \
--gpus=1 --cpus-per-gpu=16 --partition=H100-SEE --mem=220000 --job-name=semantics \
python3 train.py --config ./configs/models/SemanticPSS.yaml -n -s


srun \
--container-mounts=/netscratch/$USER:/netscratch/$USER,"`pwd`":"`pwd`",/ds:/ds \
--container-workdir="`pwd`" \
--container-image=/enroot/nvcr.io_nvidia_pytorch_23.06-py3.sqsh \
--task-prolog="`pwd`/install.sh" \
--gpus=1 --cpus-per-gpu=16 --partition=H100-SEE --mem=220000 --job-name=visnets \
python3 train.py --config ./configs/models/PictSureSVisPre.yaml -n

srun \
--container-mounts=/netscratch/$USER:/netscratch/$USER,"`pwd`":"`pwd`",/ds:/ds \
--container-workdir="`pwd`" \
--container-image=/enroot/nvcr.io_nvidia_pytorch_23.06-py3.sqsh \
--task-prolog="`pwd`/install.sh" \
--gpus=1 --cpus-per-gpu=10 --partition=H100-SEE --mem=220000 --job-name=visnoembed \
python3 train.py --config ./configs/models/VisTripPreAll-S-noembed.yaml

srun \
--container-mounts=/netscratch/$USER:/netscratch/$USER,"`pwd`":"`pwd`",/ds:/ds \
--container-workdir="`pwd`" \
--container-image=/enroot/nvcr.io_nvidia_pytorch_23.06-py3.sqsh \
--task-prolog="`pwd`/install.sh" \
--gpus=1 --cpus-per-gpu=10 --partition=H100-SEE --mem=220000 --job-name=visnoembed2 \
python3 train.py --config ./configs/models/VisPreAll-S-noembed.yaml