#! /bin/ksh #----------------------------------------------------------------------------- # Generated by /work/mh0256/m300212/STORMTIDE_run/Tides_NCEP/build_mpiom-trunk_intel_17.0.0-intelmpi_2017.0.098/run/prepare_job_mpiom_hamocc, $Revision: 4490 $ #----------------------------------------------------------------------------- # Version for SLURM #SBATCH --job-name=hel17153-DZA_TP6ML40_mpiom-trunk #SBATCH --time=08:00:00 #SBATCH --output=hel17153-DZA_TP6ML40_mpiom-trunk.o-%j.out #SBATCH --error=hel17153-DZA_TP6ML40_mpiom-trunk.o-%j.out #SBATCH --nodes=200 #SBATCH --ntasks=7200 #SBATCH --exclusive #SBATCH --partition=compute2 #SBATCH --threads-per-core=2 #SBATCH --cpus-per-task=2 #SBATCH --account=mh0256 # required ---------------- # use the same module here as for compiling # source "/sw/rhel6-x64/tcl/modules-3.2.10/Modules/3.2.10/init/ksh" module load cdo #----------------------------------------------------------------------------- # # Job file to run MPIOM with NCEP6 forcing # #----------------------------------------------------------------------------- # # If a command has a non-zero exit status, execute ERR trap, if set, and exit # set -ex # #============================================================================= export OMP_NUM_THREADS=1 export MPIOM_THREADS=1 export MPIEXPORT="MPIOM_THREADS" # # hamocc # hamocc=false read_ghgfile=no enable_cyano=no enable_martin=no enable_isotopes=no # N deposition enable_ndepo=no use_transient_ndepo=no # start year of transient N-deposition ndep_start=1850 #passive tracers enable_cfc=no enable_age=no enable_decay=no enable_cs137=no # # profiling # profile= # # time control # nfixYearLen=-1 # length of the year (-1, 360, 365) ndays=0 nmonts=2 nyears=0 model_start=2000-01-01 model_end=2013-01-01 # # forcing time control # forcing_frequency=21600 forcing_start=2000-01-01 forcing_periodicity=-1 # forcing interpolation control lspat_interp_forcing=true ltime_interp_forcing=true EXPNO=hel17153-DZA_TP6ML40_mpiom-trunk echo "Experiment: ${EXPNO}" SINFO=yes SINFO_DIR=/home/mpim/m300212/experiment_info SHORT_INFO='not_avaible' nprocx=200 nprocy=36 (( ncpus = nprocx * nprocy )) # echo " CPUs: ${ncpus} (nprocx: ${nprocx}, nprocy: ${nprocy})" mpilaunch_args="" #----------------------------------------------------------------------------- # EXPDIR=/work/mh0256/m300212/STORMTIDE_run/Tides_NCEP_SAL2/experiments/hel17153-DZA_TP6ML40_mpiom-trunk # absolute path to model binary MODDIR=/work/mh0256/m300212/STORMTIDE_run/Tides_NCEP_SAL2/build_mpiom-trunk_intel_17.0.0-intelmpi_2017.0.098/bin MODBIN=mpiom.x # absolute path to directory with plenty of space: ARCDIR=/work/mh0256/m300212/STORMTIDE_run/Tides_NCEP_SAL2/experiments/hel17153-DZA_TP6ML40_mpiom-trunk # absolute path to directory with initial data: INITIAL_DATA=/pool/data/MPIOM/input/r0008 # horizontal and vertical resolution GRID=TP6M LEV=L40 #----------------------------------------------------------------------------- caulapuv=0.005 cah00=1000. dv0=0.2e-2 av0=0.2e-2 dback=1.05e-5 aback=5.e-5 cdvocon=0.1 cavocon=0.0 cwt=0.5e-3 cwa=0.75e-3 cstabeps=0.03 ibolk=500 ltidal=.true. ltidal_diag=.true. lsatune=0.09 iter_sor=300 rtsorpar=-999. iter_sor_hack=0 rtsorpar_hack=-999. iocad=3 iocaduv=3 lwith_ice_dynamics=.true. lisopyc=.true. luse_buoyancy_forcing=.true. luse_windstress_forcing=.true. lcalc_arcgri=.true. numriv=0 lwith_one_layer_shelfs=.false. SALINITY_CLIMATOLOGY=${GRID}${LEV}_INISAL_PHC TEMPERATURE_CLIMATOLOGY=${GRID}${LEV}_INITEM_PHC #TEMPERATURE_CLIMATOLOGY=ts_phc3.0_annual_mpiom_${GRID}_${LEV}.nc #SALINITY_CLIMATOLOGY=ts_phc3.0_annual_mpiom_${GRID}_${LEV}.nc set_nudge_sss=.true. set_nudge_sst=.false. set_nudge_sao_region1=.false. set_nudge_tho_region1=.false. set_nudge_sao_region2=.false. set_nudge_tho_region2=.false. if [ "${GRID}" = "GR60" ] ; then ie=60 je=50 DT="${DT:-10800}" tp=false elif [ "${GRID}" = "TOY" ] ; then ie=66 je=36 DT="${DT:-10800}" tp=true elif [ "${GRID}" = "MED11" ] ; then ie=503 je=231 DT="900" tp=false caulapuv=0.0042 cah00=400. cwt=2.5e-5 cstabeps=0.1 set_nudge_sao_region1=.true.,3.7e-7,1,1,1,120,8,29 set_nudge_tho_region1=.true.,3.7e-7,1,1,1,120,8,29 set_nudge_sao_region2=.true.,3.7e-7,1,9,8,133,8,29 set_nudge_tho_region2=.true.,3.7e-7,1,9,8,133,8,29 numriv=93 elif [ "${GRID}" = "K20" ] ; then STA=-1 ie=100 je=100 DT=3600 tp=false iocad=0 iocaduv=0 cah00=1000. caulapuv=0.001 ibolk=0 lisopyc=.false. dv0=0.0 av0=0.0 dback=0.0 aback=0.0 cdvocon=0.0 cavocon=0.0 cwt=0.0 cwa=0.0 set_nudge_sss=.false. set_nudge_sao_region1=.false. set_nudge_tho_region1=.false. lwith_ice_dynamics=.false. luse_buoyancy_forcing=.false. luse_windstress_forcing=.false. lcalc_arcgri=.true. elif [ "${GRID}" = "ZC01" ] ; then ie=101 je=51 DT="${DT:-3600}" tp=false caulapuv=0.001 set_nudge_sss=.false. set_nudge_sao_region1=.false. set_nudge_tho_region1=.false. lwith_ice_dynamics=.false. lisopyc=.false. luse_buoyancy_forcing=.false. luse_windstress_forcing=.false. lcalc_arcgri=.true. elif [ "${GRID}" = "GR30" ] ; then ie=122 je=101 DT="${DT:-8640}" tp=false elif [ "${GRID}" = "GR3M" ] ; then ie=144 je=87 DT="${DT:-8640}" tp=false elif [ "${GRID}" = "GR15" ] ; then caulapuv=0.006 ibolk=250 ie=256 je=220 DT="${DT:-3600}" tp=false elif [ "${GRID}" = "GI6" ] ; then SALINITY_CLIMATOLOGY=ts_phc3.0_annual_mpiom_${GRID}_${LEV}.nc TEMPERATURE_CLIMATOLOGY=ts_phc3.0_annual_mpiom_${GRID}_${LEV}.nc caulapuv=0.0045 ibolk=0 cah00=800. ie=802 je=436 DT="${DT:-900}" tp=false cwt=3.5e-4 relsal=1.5e-7 cdvocon=0.15 lwith_one_layer_shelfs=.true. elif [ "${GRID}" = "TP10" ] ; then ie=362 je=192 [[ "${ltidal}" = ".true." ]] && DT="${DT:-3600}" || DT="${DT:-5400}" tp=true elif [ "${GRID}" = "TP04" ] ; then ie=802 je=404 DT="${DT:-3600}" tp=true caulapuv=0.00375 iter_sor=300 rtsorpar=1.916 iter_sor_hack=10 rtsorpar_hack=0.7 elif [ "${GRID}" = "TP6M" ] ; then ie=3602 je=2394 DT="${DT:-180}" tp=true caulapuv=0.00375 ibolk=0 iter_sor=2400 rtsorpar=1.934 iter_sor_hack=10 rtsorpar_hack=0.7 iocad=8 iocaduv=8 STA=2 fi if [ "${LEV}" = "L3" ] ; then ke=3 elif [ "${LEV}" = "L10" ] ; then ke=10 elif [ "${LEV}" = "L20" ] ; then ke=20 elif [ "${LEV}" = "L29" ] ; then ke=29 elif [ "${LEV}" = "L31" ] ; then ke=31 elif [ "${LEV}" = "L40" ] ; then ke=40 elif [ "${LEV}" = "L80" ] ; then ke=80 fi #----------------------------------------------------------------------------- # cd ${EXPDIR} # output and rerun files are written into $ARCDIR # pwd #----------------------------------------------------------------------------- # # initialisation or rerun # # if mpiom finds a restart file it tries to continue the run if [[ -f rerun_${EXPNO}_mpiom.nc ]]; then restart=yes # file model_date.asc is updated by MPIOM at the end of each run # it contains 'run_start' 'run_end' and 'next_run_start' in YYYY-MM-DD format if [[ -f model_date.asc ]]; then model_date=$(NEWSTART USE ONLY FOR COMPLETELY NEW SETUP (new topography etc!!!) # istart=1 start from horizonally uniform t,s profiles # istart=2 start from levitus # istart=3 start from existing restart files rerun_mpiom, rerun_hamocc (default) # # Advection schemes # iocad=3 ADPO (default) # iocad=4 obsolete, use iocad=3 and ibbl_transport=1 # iocad=5 ADFS # # BBL transport (slope convection) # ibbl_transport=1 (default) # nfixYearLen = -1, 365 or 360 ; default -1 for 365/366 (leap years) # # ltidal=.true. enables eph.tidal sub model (default=false) # # nyears: number of years to be simulated (default=0) # nmonths: number of months to be simulated (default=1) # model_start=$(echo ${model_start} | sed -e "s/-/,/g" -e "s/^,/-/"),0,0,0 forcing_start=$(echo ${forcing_start} | sed -e "s/-/,/g" -e "s/^,/-/"),0,0,0 cat > OCECTL << EOF &proctl ltimer = .true. / &ocedim ie_g = ${ie} je_g = ${je} ke = ${ke} lbounds_exch_tp = .${tp}. / &nprocs nprocx=${nprocx} nprocy=${nprocy} / &ocectl dt = ${DT} caulapts= 0. caulapuv= ${caulapuv} aus = 0. cah00 = ${cah00} ibolk = ${ibolk} dv0 = ${dv0} av0 = ${av0} cwt = ${cwt} cwa = ${cwa} cstabeps= ${cstabeps} dback = ${dback} aback = ${aback} cdvocon = ${cdvocon} cavocon = ${cavocon} nfixYearLen = ${nfixYearLen} ltidal = ${ltidal} ltidal_diag=${ltidal_diag} lsatune=${lsatune} lswr_jerlov = .true. jerlov_atten = 0.06 jerlov_bluefrac = 0.41 lfb_bgc_oce = .false. nyears = ${nyears} nmonts = ${nmonts} ndays = ${ndays} istart = ${STA} set_nudge_sss = ${set_nudge_sss}, 3.3e-7 set_nudge_sst = ${set_nudge_sst}, 3.3e-7 set_nudge_sao_region1 = ${set_nudge_sao_region1} set_nudge_tho_region1 = ${set_nudge_tho_region1} set_nudge_sao_region2 = ${set_nudge_sao_region2} set_nudge_tho_region2 = ${set_nudge_tho_region2} lmpitype = .false. lnonblock = .true. icontro = 0 iocad = ${iocad} iocaduv = ${iocaduv} ibbl_transport = 1 lundelayed_momentum_advection = .true. rleadclose=0.25,3.0,2.0 h0=0.5 iter_sor = ${iter_sor} rtsorpar = ${rtsorpar} iter_sor_hack = ${iter_sor_hack} rtsorpar_hack = ${rtsorpar_hack} model_start_time = ${model_start} time_verbosity = 1 lzo_correct = .true. lsaoclose = .true. rice_cutoff_factor = 0.9 lcaesium_source = ${lcaesium_source} lakes = .false. lwith_ice_dynamics = ${lwith_ice_dynamics} lisopyc = ${lisopyc} luse_buoyancy_forcing = ${luse_buoyancy_forcing} luse_windstress_forcing = ${luse_windstress_forcing} lcalc_arcgri = ${lcalc_arcgri} luse_sp_diag = .true. fp_tracing_enabled = .false. testbed = 0 luse_cmip6_varlist = .true. numriv = ${numriv} lwith_one_layer_shelfs= ${lwith_one_layer_shelfs} / EOF if [ ${ke} -eq 3 ] ; then cat >> OCECTL << EOF2 &ocedzw cdzw = 12.,10.,5000., / EOF2 fi if [ ${ke} -eq 10 ] ; then cat >> OCECTL << EOF2 &ocedzw cdzw = 40., 50., 90., 160., 270., 390., 550., 900., 1300., 2300. / EOF2 fi if [ ${ke} -eq 20 ] ; then cat >> OCECTL << EOF2 &ocedzw cdzw = 20.,20., 20., 30.,40.,50.,70., 90.,120.,150.,180.,210.,250.,300., 400.,500.,600.,700.,900.,1400., / EOF2 fi if [ ${ke} -eq 29 ] ; then cat >> OCECTL << EOF2 &ocedzw cdzw = 12.,10.,10.,10.,11.,13.,16.,19.,23., 23., 28., 33., 40., 48., 58., 70., 84.,102.,122.,148.,178.,214.,258.,311., 375.,400.,450.,500.,500. / EOF2 fi if [ ${ke} -eq 31 ] ; then cat >> OCECTL << EOF2 &ocedzw cdzw = 10.,10.,10.,10.,10.,10.,10.,10.,10.,10., 15.,15.,15.,15.,15.,15.,15.,15.,20.,20., 20.,20.,20.,20.,25.,25.,30.,30.,30.,30.,30., / EOF2 fi if [ ${ke} -eq 40 ] ; then cat >> OCECTL << EOF2 &ocedzw cdzw = 20.,10.,10.,10.,10.,10.,13.,15.,20.,25., 30.,35.,40.,45.,50.,55.,60.,70.,80.,90., 100.,110.,120.,130.,140.,150.,170.,180.,190.,200., 220.,250.,270.,300.,350.,400.,450.,500.,500.,600., / EOF2 fi if [ ${ke} -eq 80 ] ; then cat >> OCECTL <> OCECTL <> OCECTL < NAMELIST_BGC << EOF2 &BGCCTL deltacalc = ${deltacalc} deltaorg = ${deltaorg} deltasil = ${deltasil} io_stdo_bgc = 8, kchck = 0, isac = 1, rmasko = -9e33 atm_co2 = ${co2ppm} l_cyadyn = ${LCYA} lmartin = ${LMAR} lndep = ${LNDEP} / &ioctl ! ! iolist elements from 100 - 80 are reserved for default output definitions ! iolist(100) = 99, 'rerun_${EXPNO}_hamocc.nc', '${FRMT}', 7, 10, 11, 12, 13, 14, 15, 16, 17, 20, 21, 22, 23, 24, 27, 28, 29, 30, 31, 37, 38, 41, 44, 45, 46, 47, 48, 49, 50, 51, 54, 55, 56, 57, 58, 59, 60, 61, 62, 64, 203, 204, 205${RCYA}${Riso} iolist(99) = 4002, 'inpdust_r2.nc', '${FRMT}', 34 $( if [ ${enable_ndepo} == "yes" ] then echo " iolist(98) = 4002,'inpnit.nc', '${FRMT}',35" fi ) iolist(97) = 3, '${EXPNO}_hamocc_data_3d_ym.nc', '${FRMT}', 7, 10, 11, 12, 14, 15, 16, 17, 20, 21, 22, 23, 24, 27, 28, 29, 31, 37, 158${OCYA}${Oiso} iolist(96) = 2, '${EXPNO}_hamocc_eu_data_mm.nc', '${FRMT}', 69, 70, 71, 84, 85, 86, 100, 101, 292, 293, 294, 295, 296, 297, 143, 510${EUCYA} iolist(95) = 2, '${EXPNO}_hamocc_data_2d_mm.nc', '${FRMT}', 67, 68, 72, 75, 78, 81, 92, 93, 94, 95, 107, 110, 111, 112, 114, 115, 116, 117, 120, 121, 122, 123, 124, 127, 129, 131, 137, 157, 159, 160, 161, 162, 163, 164, 165, 174, 185, 186, 200, 206, 207, 210, 211, 214, 215, 231${Aiso} iolist(94) = 101, '${EXPNO}_hamocc_co2.nc', '${FRMT}', 301, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 203, 204, 205 iolist(93) = 3, '${EXPNO}_hamocc_monitoring_ym.nc', '${FRMT}', 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 517, 519, 523${MCYA}, 529, 530, 531, 532, 533, 534, 535 iolist(92) = 3, '${EXPNO}_hamocc_sedi_ym.nc', '${FRMT}', 38, 41, 44, 45, 46, 47, 48, 49, 50, 51, 54, 55, 56, 57, 58, 59, 60, 94, 95, 96, 97, 167,168, 280, 281, 282, 283, 284, 285, 286, 287${Siso} ! iolist(91) = 2,'${EXPNO}_hamocc_eddy_mm.nc','${FRMT}',315,316,317,318,319,320, ! 321,322,323,324,325,326,234,235,236,72 ! iolist(90)=2,'${EXPNO}_hamocc_stations_mm.nc','${FRMT}', ! 1001,1002,1003,1004,1005,1006,1007,1008,1009,1010,1011, ! 2001,2002,2003,2004,2005,2006,2007,2008,2009,2010,2011, ! 3001,3002,3003,3004,3005,3006,3007,3008,3009,3010,3011 / EOF2 # fi #============================================================================= echo "Integration started on $(date)" export I_MPI_FABRICS=shm:dapl export I_MPI_FALLBACK=disable export I_MPI_SLURM_EXT=1 # Set I_MPI_LARGE_SCALE_THRESHOLD to a value larger than the number of your MPI-tasks if you use 8192 or more tasks. export I_MPI_LARGE_SCALE_THRESHOLD=8192 srun --propagate=STACK --kill-on-bad-exit=1 --cpu_bind=verbose,cores --distribution=block:block -n ${ncpus} ${mpilaunch_args} ${MODDIR}/${MODBIN} echo "Integration completed on $(date)" #============================================================================= if [[ "${profile}" != "" ]]; then set +e jobid=$(echo $LOADL_STEP_ID | awk -F. '{print $(NF-1)}') profdir=prof_${run_start}.${jobid} mkdir ${profdir} mv gmon.out* ${profdir} 2>/dev/null || echo 'INFO: no gmon.out profiling data present' mv poe.prof ${profdir}/mpiom_tprof.out 2>/dev/null || echo 'INFO: no tprof profiling data present' mv rusage ${profdir} 2>/dev/null || echo 'INFO: no rusage profiling data present' mv mpi_profile* single_trace* ${profdir} 2>/dev/null || echo 'INFO: no MPI profiling data present' mv hpc_profile*.hpm hpc_profile*.viz ${profdir} 2>/dev/null || echo 'INFO: no HPM profiling data present' cp -p OCECTL ${profdir} tar cvf ${profdir}/oceout.tar oceout* [[ ${hamocc} == "true" ]] && tar cvf ${profdir}/bgcout.tar bgcout* rm -f oceout* bgcout* fi [[ -d ${ARCDIR}/restart ]] || /bin/mkdir -p ${ARCDIR}/restart [[ -d ${ARCDIR}/outdata ]] || /bin/mkdir -p ${ARCDIR}/outdata model_date=$( $EXPNO.info<