scripts and typos

.gitignore

@@ -26,7 +26,7 @@ results_old/
 mod/
 obj/
 bin/
-wk/fort.90
+wk/fort*.90
 .directory
 checkpoint/
 FM/
@@ -34,7 +34,7 @@ iCa/
 *.out
 src/srcinfo.h
 src/srcinfo/srcinfo.h
-fort.90
+fort*.90
 local/
 *.sh
 Gallery/

matlab/continue_run.m

+%% Functions to modify preexisting fort.90 input file and launch on marconi
+function [] = continue_run(outfilename)
+    EXECNAME = 'helaz_3.8';
+    %% CLUSTER PARAMETERS
+    CLUSTER.PART  = 'prod';     % dbg or prod
+    CLUSTER.TIME  = '24:00:00'; % allocation time hh:mm:ss
+    if(strcmp(CLUSTER.PART,'dbg')); CLUSTER.TIME  = '00:30:00'; end;
+    CLUSTER.MEM   = '64GB';     % Memory
+    CLUSTER.JNAME = 'HeLaZ';% Job name
+    NP_P          = 2;          % MPI processes along p  
+    NP_KX         = 24;         % MPI processes along kx
+    % Compute processes distribution
+    Ntot = NP_P * NP_KX;
+    Nnodes = ceil(Ntot/48);
+    Nppn   = Ntot/Nnodes; 
+    CLUSTER.NODES =  num2str(Nnodes);  % MPI process along p
+    CLUSTER.NTPN  =  num2str(Nppn); % MPI process along kx
+    CLUSTER.CPUPT = '1';        % CPU per task
+    %%
+    RESDIR = ['../',outfilename(46:end-8),'/'];
+    BASIC.RESDIR = RESDIR;
+    FORT90 = [RESDIR,'fort.90'];
+  
+    % Read txt into cell A
+    fid = fopen(FORT90,'r');
+    i = 1;
+    tline = fgetl(fid);
+    A{i} = tline;
+    while ischar(tline)
+        i = i+1;
+        tline = fgetl(fid);
+        A{i} = tline;
+    end
+    fclose(fid);
+
+    % Find previous job2load
+    if( numel(A{5}) ==  numel('  RESTART    = .false.') )
+        A{5} = sprintf('  RESTART   = .true.');
+        J2L = 0;
+    else
+        line = A{39};
+        line = line(end-2:end);
+        if(line(1) == '='); line = line(end); end;
+        J2L = str2num(line)+1;
+    end
+    % Change job 2 load in fort.90
+    A{39} = ['  job2load      = ',num2str(J2L)];
+    disp(A{39})
+    % Change time step
+    line_= A{3};
+    dt_old = str2num(line_(13:end));
+    A{3} = ['  dt     = ',num2str(dt_old)];
+    % Increase endtime
+    A{4} = ['  tmax      = 20000'];
+    % Change collision operator
+    line_= A{43};
+    CO_old = str2num(line_(13:end));
+    A{43} = ['  CO      = ',num2str(2)];
+    % Put non linear term back
+    A{45} = ['  NL_CLOS = -1'];
+    % change HD
+    line_= A{47};
+    mu_old = str2num(line_(13:end));
+    A{47} = ['  mu      = ',num2str(0)];
+    % change L
+    line_= A{14};
+    L_old = str2num(line_(12:end));
+    A{14} = ['  Lx     = ',num2str(L_old)];
+    A{16} = ['  Ly     = ',num2str(L_old)];
+    % change eta N
+    line_= A{57};
+    etan_old = str2num(line_(13:end));
+    A{57} = ['  eta_n   = ',num2str(etan_old)];
+    % change eta B
+    line_= A{59};
+    etab_old = str2num(line_(13:end));
+    A{59} = ['  eta_B   = ',num2str(etab_old)];
+    % Rewrite fort.90
+    fid = fopen('fort.90', 'w');
+    for i = 1:numel(A)
+        if A{i+1} == -1
+            fprintf(fid,'%s', A{i});
+            break
+        else
+            fprintf(fid,'%s\n', A{i});
+        end
+    end
+    % Copy fort.90 into marconi
+    write_sbash_marconi
+    % Launch the job
+    system('ssh ahoffman@login.marconi.cineca.it sh HeLaZ/wk/setup_and_run.sh');
+end

matlab/load_marconi.m

@@ -14,12 +14,11 @@ function [ RESDIR ] = load_marconi( outfilename )
     disp(CMD);
     system(CMD);
     % Load the fort.90 as well in misc folder
-    CMD = ['scp -r ahoffman@login.marconi.cineca.it:',hostfolder,'/fort.90',' ',miscfolder];
+    CMD = ['scp -r ahoffman@login.marconi.cineca.it:',hostfolder,'/fort*.90',' ',miscfolder];
     disp(CMD);
     system(CMD);
     % Put it also in the result directory
-    CMD = ['scp -r ahoffman@login.marconi.cineca.it:',hostfolder,'/fort.90',' ',resultfolder];
+    CMD = ['scp -r ahoffman@login.marconi.cineca.it:',hostfolder,'/fort*.90',' ',resultfolder];
     disp(CMD);
     system(CMD);
 end
-

matlab/setup.m

@@ -111,7 +111,6 @@ BASIC.maxruntime = str2num(CLUSTER.TIME(1:2))*3600 ...
                    + str2num(CLUSTER.TIME(4:5))*60 ...
                    + str2num(CLUSTER.TIME(7:8));
 % Outputs parameters
-if RESTART; BASIC.RESTART = '.true.'; else; BASIC.RESTART = '.false.';end;
 OUTPUTS.nsave_0d = floor(1.0/SPS0D/DT);
 OUTPUTS.nsave_1d = -1;
 OUTPUTS.nsave_2d = floor(1.0/SPS2D/DT);
@@ -120,6 +119,7 @@ OUTPUTS.nsave_5d = floor(1.0/SPS5D/DT);
 OUTPUTS.nsave_cp = floor(1.0/SPSCP/DT);
 if W_DOUBLE; OUTPUTS.write_doubleprecision = '.true.'; else; OUTPUTS.write_doubleprecision = '.false.';end;
 if W_GAMMA;  OUTPUTS.write_gamma = '.true.'; else; OUTPUTS.write_gamma = '.false.';end;
+if W_HF;     OUTPUTS.write_hf    = '.true.'; else; OUTPUTS.write_hf    = '.false.';end;
 if W_PHI;    OUTPUTS.write_phi   = '.true.'; else; OUTPUTS.write_phi   = '.false.';end;
 if W_NA00;   OUTPUTS.write_Na00  = '.true.'; else; OUTPUTS.write_Na00  = '.false.';end;
 if W_NAPJ;   OUTPUTS.write_Napj  = '.true.'; else; OUTPUTS.write_Napj  = '.false.';end;
@@ -147,7 +147,7 @@ system(MAKE);
 %%
 disp(['Set up ',SIMID]);
 disp([resolution,gridname,degngrad]);
-if RESTART
+if JOB2LOAD>=0
 	disp(['- restarting from JOBNUM = ',num2str(JOB2LOAD)]); else
 	disp(['- starting from T = 0']);
 end

matlab/write_fort90.m

 function [INPUT] = write_fort90(OUTPUTS,GRID,MODEL,INITIAL,TIME_INTEGRATION,BASIC)
 % Write the input script "fort.90" with desired parameters
-INPUT = 'fort.90';
+INPUT = ['fort_',sprintf('%2.2d',OUTPUTS.job2load+1),'.90'];
 fid = fopen(INPUT,'wt');
 
 fprintf(fid,'&BASIC\n');
 fprintf(fid,['  nrun   = ', num2str(BASIC.nrun),'\n']);
 fprintf(fid,['  dt     = ', num2str(BASIC.dt),'\n']);
 fprintf(fid,['  tmax   = ', num2str(BASIC.tmax),'\n']);
-fprintf(fid,['  RESTART    = ', BASIC.RESTART,'\n']);
 fprintf(fid,['  maxruntime = ', num2str(BASIC.maxruntime),'\n']);
 fprintf(fid,'/\n');
 
 fprintf(fid,'&GRID\n');
-fprintf(fid,['  pmaxe  =', num2str(GRID.pmaxe),'\n']);
+fprintf(fid,['  pmaxe  = ', num2str(GRID.pmaxe),'\n']);
 fprintf(fid,['  jmaxe  = ', num2str(GRID.jmaxe),'\n']);
 fprintf(fid,['  pmaxi  = ', num2str(GRID.pmaxi),'\n']);
 fprintf(fid,['  jmaxi  = ', num2str(GRID.jmaxi),'\n']);
@@ -32,18 +31,18 @@ fprintf(fid,['  nsave_1d = ', num2str(OUTPUTS.nsave_1d),'\n']);
 fprintf(fid,['  nsave_2d = ', num2str(OUTPUTS.nsave_2d),'\n']);
 fprintf(fid,['  nsave_3d = ', num2str(OUTPUTS.nsave_3d),'\n']);
 fprintf(fid,['  nsave_5d = ', num2str(OUTPUTS.nsave_5d),'\n']);
-fprintf(fid,['  nsave_cp = ', num2str(OUTPUTS.nsave_cp),'\n']);
 fprintf(fid,['  write_doubleprecision = ', OUTPUTS.write_doubleprecision,'\n']);
 fprintf(fid,['  write_gamma = ', OUTPUTS.write_gamma,'\n']);
+fprintf(fid,['  write_hf    = ', OUTPUTS.write_hf,'\n']);
 fprintf(fid,['  write_phi   = ', OUTPUTS.write_phi,'\n']);
 fprintf(fid,['  write_Na00  = ', OUTPUTS.write_Na00,'\n']);
 fprintf(fid,['  write_Napj  = ', OUTPUTS.write_Napj,'\n']);
 fprintf(fid,['  write_Sapj  = ', OUTPUTS.write_Sapj,'\n']);
 fprintf(fid,['  write_dens  = ', OUTPUTS.write_dens,'\n']);
 fprintf(fid,['  write_temp  = ', OUTPUTS.write_temp,'\n']);
-fprintf(fid,['  resfile0      = ', OUTPUTS.resfile0,'\n']);
-fprintf(fid,['  rstfile0      = ', OUTPUTS.rstfile0,'\n']);
-fprintf(fid,['  job2load      = ', num2str(OUTPUTS.job2load),'\n']);
+fprintf(fid,['  resfile0    = ', OUTPUTS.resfile0,'\n']);
+fprintf(fid,['  rstfile0    = ', OUTPUTS.rstfile0,'\n']);
+fprintf(fid,['  job2load    = ', num2str(OUTPUTS.job2load),'\n']);
 fprintf(fid,'/\n');
 
 fprintf(fid,'&MODEL_PAR\n');
@@ -85,5 +84,5 @@ fprintf(fid,['  numerical_scheme = ', TIME_INTEGRATION.numerical_scheme,'\n']);
 fprintf(fid,'/');
 
 fclose(fid);
-system(['cp fort.90 ',BASIC.RESDIR,'/.']);
+system(['cp fort*.90 ',BASIC.RESDIR,'/.']);
 end

matlab/write_sbash_daint.m

@@ -10,7 +10,7 @@ fprintf(fid,[...
 ...
 'mkdir -p ', BASIC.RESDIR,'\n',...
 'cd ',BASIC.RESDIR,'\n',...
-'cp $HOME/HeLaZ/wk/fort.90 .\n',...
+'cp $HOME/HeLaZ/wk/fort*.90 .\n',...
 'cp $HOME/HeLaZ/wk/batch_script.sh .\n',...
 ...
 'jid=$(sbatch batch_script.sh)\n',...
@@ -54,4 +54,4 @@ fprintf(fid,[...
 fclose(fid);
 system(['cp batch_script.sh ',BASIC.RESDIR,'/.']);
 
-system('scp {fort.90,setup_and_run.sh,batch_script.sh} ahoffman@ela.cscs.ch:HeLaZ/wk');
+system('scp {fort*.90,setup_and_run.sh,batch_script.sh} ahoffman@ela.cscs.ch:HeLaZ/wk');
\ No newline at end of file

matlab/write_sbash_marconi.m

@@ -10,11 +10,11 @@ fprintf(fid,[...
 ...
 'mkdir -p ', BASIC.RESDIR,'\n',...
 'cd ',BASIC.RESDIR,'\n',...
-'cp $HOME/HeLaZ/wk/fort.90 .\n',...
+'cp $HOME/HeLaZ/wk/fort*.90 .\n',...
 'cp $HOME/HeLaZ/wk/batch_script.sh .\n',...
 ...
-'sbatch batch_script.sh\n',...
-'echo tail -f $CINECA_SCRATCH/HeLaZ',BASIC.RESDIR(3:end),'out.txt']);
+SBATCH_CMD,...
+'echo tail -f $CINECA_SCRATCH/HeLaZ',BASIC.RESDIR(3:end),'out']);
 
 fclose(fid);
 system(['cp setup_and_run.sh ',BASIC.RESDIR,'/.']);
@@ -31,15 +31,15 @@ fprintf(fid,[...
 '#SBATCH --cpus-per-task=', CLUSTER.CPUPT,'\n',...
 '#SBATCH --ntasks-per-node=', CLUSTER.NTPN,'\n',...
 '#SBATCH --mem=', CLUSTER.MEM,'\n',...
-'#SBATCH --error=err.txt\n',...
-'#SBATCH --output=out.txt\n',...
+'#SBATCH --error=err',num2str(JOB2LOAD+1),'.txt\n',...
+'#SBATCH --output=out_',num2str(JOB2LOAD+1),'.txt\n',...
 '#SBATCH --account=FUA35_TSVVT421\n',...
 '#SBATCH --partition=skl_fua_',CLUSTER.PART,'\n',...
 'module load autoload hdf5 fftw\n',...
-'srun --cpu-bind=cores ./../../../bin/',EXECNAME,' ',num2str(NP_P),' ',num2str(NP_KX)]);
+'srun --cpu-bind=cores ./../../../bin/',EXECNAME,' ',num2str(NP_P),' ',num2str(NP_KX),' ',num2str(JOB2LOAD+1)]);
 
 fclose(fid);
 system(['cp batch_script.sh ',BASIC.RESDIR,'/.']);
 
-system('scp {fort.90,setup_and_run.sh,batch_script.sh} ahoffman@login.marconi.cineca.it:/marconi/home/userexternal/ahoffman/HeLaZ/wk > trash.txt');
+system('scp {fort*.90,setup_and_run.sh,batch_script.sh} ahoffman@login.marconi.cineca.it:/marconi/home/userexternal/ahoffman/HeLaZ/wk > trash.txt');
 system('rm trash.txt');
\ No newline at end of file

src/tesend.F90

@@ -52,7 +52,7 @@ SUBROUTINE tesend
      RETURN
   END IF
   !________________________________________________________________________________
-  !                   5.  NRUN modified throught "stop file"
+  !                   5.  NRUN modified through "stop file"
   !
   IF( (my_id .EQ. 0) .AND. (MOD(cstep, ncheck_stop) == 0) ) THEN
      INQUIRE(file=stop_file, exist=mlexist)

wk/analysis_3D.m

@@ -2,15 +2,15 @@ addpath(genpath('../matlab')) % ... add
 addpath(genpath('../matlab/plots')) % ... add
 outfile ='';
 %% Directory of the simulation
-if 0% Local results
+if 1% Local results
 outfile ='';
 outfile ='';
 % outfile ='artificial_ZF_freeze/sim_A';
 % outfile ='simulation_B/cw_FCGK_kp_3.0';
-% outfile ='nonlin_FCGK/150x75_L_200_P_4_J_2_eta_0.6_nu_1e-01_FCGK_mu_0e+00';
+outfile ='nonlin_FCGK/150x75_L_200_P_4_J_2_eta_0.6_nu_1e-01_FCGK_mu_0e+00';
 % outfile ='nonlin_PAGK/100x50_L_200_P_4_J_2_eta_0.6_nu_1e-01_PAGK_mu_0e+00';
 % outfile ='nonlin_FCGK/100x50_L_200_P_4_J_2_eta_0.6_nu_1e-01_FCGK_mu_0e+00';
-outfile ='simulation_A';
+% outfile ='simulation_A';
 % outfile ='simulation_B/cw_SGGK_like_species';
 % outfile ='simulation_A/CO_damping_SGGK';
 % outfile ='simulation_A/cw_DGGK_eta_0.5';

wk/local_run.m

@@ -24,30 +24,26 @@ SPS2D   = 1;      % Sampling per time unit for 2D arrays
 SPS3D   = 1;      % Sampling per time unit for 3D arrays
 SPS5D   = 1/20;  % Sampling per time unit for 5D arrays
 SPSCP   = 0;    % Sampling per time unit for checkpoints/10
-RESTART = 0;      % To restart from last checkpoint
-JOB2LOAD= 0;
+JOB2LOAD= -1;
 %% OPTIONS AND NAMING
 % Collision operator
 % (0 : L.Bernstein, 1 : Dougherty, 2: Sugama, 3 : Pitch angle ; 4 : Coulomb; +/- for GK/DK)
-CO      = 4;
+CO      = 1;
 CLOS    = 0;   % Closure model (0: =0 truncation)
 NL_CLOS = -1;   % nonlinear closure model (-2: nmax = jmax, -1: nmax = jmax-j, >=0 : nmax = NL_CLOS)
-SIMID   = 'nonlin_FCGK';  % Name of the simulation
-% SIMID   = 'test_3D';  % Name of the simulation
+% SIMID   = 'nonlin_FCGK';  % Name of the simulation
+SIMID   = 'test';  % Name of the simulation
 % SIMID   = ['v3.0_P_',num2str(P),'_J_',num2str(J)];  % Name of the simulation
 NON_LIN = 1;   % activate non-linearity (is cancelled if KXEQ0 = 1)
 % INIT options
 INIT_ZF = 0; ZF_AMP = 0.0;
 INIT_BLOB = 0; WIPE_TURB = 0; WIPE_ZF = 0;
 %% OUTPUTS
-W_DOUBLE = 0;
-W_GAMMA  = 1;
-W_PHI    = 1;
-W_NA00   = 1;
-W_NAPJ   = 1;
-W_SAPJ   = 0;
-W_DENS   = 1;
-W_TEMP   = 1;
+W_DOUBLE = 1;
+W_GAMMA  = 1; W_HF     = 1;
+W_PHI    = 1; W_NA00   = 1;
+W_DENS   = 1; W_TEMP   = 1;
+W_NAPJ   = 1; W_SAPJ   = 0;
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %% unused
@@ -62,9 +58,9 @@ KPAR    = 0.0;    % Parellel wave vector component
 LAMBDAD = 0.0;
 kmax    = N*pi/L;% Highest fourier mode
 HD_CO   = 0.5;    % Hyper diffusivity cutoff ratio
-% kmaxcut = 2.5;
-MU      = NU_HYP/(HD_CO*kmax)^4 % Hyperdiffusivity coefficient
+MU      = NU_HYP/(HD_CO*kmax)^4; % Hyperdiffusivity coefficient
 NOISE0  = 1.0e-5;
+BCKGD0  = 0.0;    % Init background
 TAU     = 1.0;    % e/i temperature ratio
 ETAT    = 0.0;    % Temperature gradient
 ETAB    = 1.0;    % Magnetic gradient (1.0 to set R=LB)
@@ -73,6 +69,6 @@ MU_P    = 0.0;     % Hermite  hyperdiffusivity -mu_p*(d/dvpar)^4 f
 MU_J    = 0.0;     % Laguerre hyperdiffusivity -mu_j*(d/dvperp)^4 f
 %% Setup and file management
 setup
-system('rm fort.90');
+system('rm fort*.90');
 outfile = [BASIC.RESDIR,'out.txt'];
 disp(outfile);

wk/marconi_run.m

 clear all;
 addpath(genpath('../matlab')) % ... add
 SUBMIT = 1; % To submit the job automatically
+CHAIN  = 1; % To chain jobs (CHAIN = n will launch n jobs in chain)
 % EXECNAME = 'helaz_dbg';
-  EXECNAME = 'helaz_3.8';
+  EXECNAME = 'helaz_3.81';
 for ETAN = [1/0.6]
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %% Set Up parameters
@@ -10,15 +11,15 @@ for ETAN = [1/0.6]
 %% CLUSTER PARAMETERS
 CLUSTER.PART  = 'prod';     % dbg or prod
 % CLUSTER.PART  = 'dbg';
-CLUSTER.TIME  = '24:00:00'; % allocation time hh:mm:ss
+CLUSTER.TIME  = '20:00:00'; % allocation time hh:mm:ss
 if(strcmp(CLUSTER.PART,'dbg')); CLUSTER.TIME  = '00:30:00'; end;
 CLUSTER.MEM   = '128GB';     % Memory
 CLUSTER.JNAME = 'HeLaZ';% Job name
 NP_P          = 2;          % MPI processes along p
-NP_KX         = 24;         % MPI processes along kx
+NP_KX         = 48;         % MPI processes along kx
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %% PHYSICAL PARAMETERS
-NU      = 0.5;   % Collision frequency
+NU      = 0.1;   % Collision frequency
 ETAN    = 1.0/0.6;    % Density gradient drive (R/Ln)
 NU_HYP  = 0.0;
 %% GRID PARAMETERS
@@ -32,22 +33,21 @@ P       = 8;
 J       = 4;
 %% TIME PARAMETERS
 TMAX    = 10000;  % Maximal time unit
-DT      = 5e-3;   % Time step
+DT      = 8e-3;   % Time step
 SPS0D   = 1;      % Sampling per time unit for profiler
 SPS2D   = 1;      % Sampling per time unit for 2D arrays
 SPS3D   = 1/2;      % Sampling per time unit for 3D arrays
 SPS5D   = 1/100;  % Sampling per time unit for 5D arrays
 SPSCP   = 0;    % Sampling per time unit for checkpoints/10
-RESTART = 1;      % To restart from last checkpoint
-JOB2LOAD= 0;
+JOB2LOAD= -1; % start from t=0 if <0, else restart from outputs_$job2load
 %% OPTIONS AND NAMING
 % Collision operator
 % (0 : L.Bernstein, 1 : Dougherty, 2: Sugama, 3 : Pitch angle ; +/- for GK/DK)
-CO      = 2;
+CO      = 3;
 CLOS    = 0;   % Closure model (0: =0 truncation)
 NL_CLOS = -1;   % nonlinear closure model (-2: nmax = jmax, -1: nmax = jmax-j, >=0 : nmax = NL_CLOS)
-% SIMID   = 'test_3D_marconi';  % Name of the simulation
-SIMID   = 'simulation_B';  % Name of the simulation
+% SIMID   = 'test_chained_job';  % Name of the simulation
+SIMID   = 'simulation_A';  % Name of the simulation
 % SIMID   = ['v3.0_P_',num2str(P),'_J_',num2str(J)];  % Name of the simulation
 NON_LIN = 1;   % activate non-linearity (is cancelled if KXEQ0 = 1)
 % INIT options
@@ -55,13 +55,10 @@ INIT_ZF = 0; ZF_AMP = 0.0;
 INIT_BLOB = 0; WIPE_TURB = 0; WIPE_ZF = 0;
 %% OUTPUTS
 W_DOUBLE = 1;
-W_GAMMA  = 1;
-W_PHI    = 1;
-W_NA00   = 1;
-W_NAPJ   = 1;
-W_SAPJ   = 0;
-W_DENS   = 1;
-W_TEMP   = 1;
+W_GAMMA  = 1; W_HF     = 1;
+W_PHI    = 1; W_NA00   = 1; 
+W_DENS   = 1; W_TEMP   = 1;
+W_NAPJ   = 1; W_SAPJ   = 0;
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %% unused
@@ -95,13 +92,27 @@ CLUSTER.NTPN  =  num2str(Nppn); % MPI process along kx
 CLUSTER.CPUPT = '1';        % CPU per task
 %% Run file management scripts
 setup
+SBATCH_CMD = 'sbatch batch_script.sh\n';
 write_sbash_marconi
-system('rm fort.90 setup_and_run.sh batch_script.sh');
+system('rm fort*.90 setup_and_run.sh batch_script.sh');
 if(mod(NP_P*NP_KX,48)~= 0)
     disp('WARNING : unused cores (ntot cores must be a 48 multiple)');
 end
 if(SUBMIT)
-    system('ssh ahoffman@login.marconi.cineca.it sh HeLaZ/wk/setup_and_run.sh');
+    [~,job_info_] = system('ssh ahoffman@login.marconi.cineca.it sh HeLaZ/wk/setup_and_run.sh');
+    disp(job_info_);
+    jobid_ = job_info_(21:27);
+    if(CHAIN>0)
+        for CHAIN_IDX = 1:CHAIN
+            SBATCH_CMD = ['sbatch --dependency=afterok:',jobid_,' batch_script.sh\n'];
+            disp(SBATCH_CMD);
+            JOB2LOAD= JOB2LOAD+1;
+            setup
+            write_sbash_marconi
+            [~,job_info_] = system('ssh ahoffman@login.marconi.cineca.it sh HeLaZ/wk/setup_and_run.sh');
+            jobid_ = job_info_(21:27);
+        end
+    end
 end
 disp('done');
 end