#! /bin/ksh
#-----------------------------------------------------------------------------
# Generated by /work/mh0256/m300212/STORMTIDE_run/Tides_NCEP/build_mpiom-trunk_intel_17.0.0-intelmpi_2017.0.098/run/prepare_job_mpiom_hamocc, $Revision: 4490 $
#-----------------------------------------------------------------------------
# Version for SLURM
#SBATCH --job-name=hel17153-DZA_TP6ML40_mpiom-trunk
#SBATCH --time=08:00:00
#SBATCH --output=hel17153-DZA_TP6ML40_mpiom-trunk.o-%j.out
#SBATCH --error=hel17153-DZA_TP6ML40_mpiom-trunk.o-%j.out
#SBATCH --nodes=200
#SBATCH --ntasks=7200
#SBATCH --exclusive
#SBATCH --partition=compute2
#SBATCH --threads-per-core=2
#SBATCH --cpus-per-task=2
#SBATCH --account=mh0256
# required ----------------
# use the same module here as for compiling
#
source "/sw/rhel6-x64/tcl/modules-3.2.10/Modules/3.2.10/init/ksh"
module load cdo
#-----------------------------------------------------------------------------
#
# Job file to run MPIOM with NCEP6 forcing
#
#-----------------------------------------------------------------------------
#
# If a command has a non-zero exit status, execute ERR trap, if set, and exit
#
set -ex
#
#=============================================================================
export OMP_NUM_THREADS=1
export MPIOM_THREADS=1
export MPIEXPORT="MPIOM_THREADS"
#
# hamocc
#
hamocc=false
read_ghgfile=no
enable_cyano=no
enable_martin=no
enable_isotopes=no
# N deposition
enable_ndepo=no
use_transient_ndepo=no
# start year of transient N-deposition
ndep_start=1850
#passive tracers
enable_cfc=no
enable_age=no
enable_decay=no
enable_cs137=no
#
# profiling
#
profile=
#
# time control
#
nfixYearLen=-1 # length of the year (-1, 360, 365)
ndays=0
nmonts=2
nyears=0
model_start=2000-01-01
model_end=2013-01-01
#
# forcing time control
#
forcing_frequency=21600
forcing_start=2000-01-01
forcing_periodicity=-1
# forcing interpolation control
lspat_interp_forcing=true
ltime_interp_forcing=true
EXPNO=hel17153-DZA_TP6ML40_mpiom-trunk
echo "Experiment: ${EXPNO}"
SINFO=yes
SINFO_DIR=/home/mpim/m300212/experiment_info
SHORT_INFO='not_avaible'
nprocx=200
nprocy=36
(( ncpus = nprocx * nprocy ))
#
echo " CPUs: ${ncpus} (nprocx: ${nprocx}, nprocy: ${nprocy})"
mpilaunch_args=""
#-----------------------------------------------------------------------------
#
EXPDIR=/work/mh0256/m300212/STORMTIDE_run/Tides_NCEP_SAL2/experiments/hel17153-DZA_TP6ML40_mpiom-trunk
# absolute path to model binary
MODDIR=/work/mh0256/m300212/STORMTIDE_run/Tides_NCEP_SAL2/build_mpiom-trunk_intel_17.0.0-intelmpi_2017.0.098/bin
MODBIN=mpiom.x
# absolute path to directory with plenty of space:
ARCDIR=/work/mh0256/m300212/STORMTIDE_run/Tides_NCEP_SAL2/experiments/hel17153-DZA_TP6ML40_mpiom-trunk
# absolute path to directory with initial data:
INITIAL_DATA=/pool/data/MPIOM/input/r0008
# horizontal and vertical resolution
GRID=TP6M
LEV=L40
#-----------------------------------------------------------------------------
caulapuv=0.005
cah00=1000.
dv0=0.2e-2
av0=0.2e-2
dback=1.05e-5
aback=5.e-5
cdvocon=0.1
cavocon=0.0
cwt=0.5e-3
cwa=0.75e-3
cstabeps=0.03
ibolk=500
ltidal=.true.
ltidal_diag=.true.
lsatune=0.09
iter_sor=300
rtsorpar=-999.
iter_sor_hack=0
rtsorpar_hack=-999.
iocad=3
iocaduv=3
lwith_ice_dynamics=.true.
lisopyc=.true.
luse_buoyancy_forcing=.true.
luse_windstress_forcing=.true.
lcalc_arcgri=.true.
numriv=0
lwith_one_layer_shelfs=.false.
SALINITY_CLIMATOLOGY=${GRID}${LEV}_INISAL_PHC
TEMPERATURE_CLIMATOLOGY=${GRID}${LEV}_INITEM_PHC
#TEMPERATURE_CLIMATOLOGY=ts_phc3.0_annual_mpiom_${GRID}_${LEV}.nc
#SALINITY_CLIMATOLOGY=ts_phc3.0_annual_mpiom_${GRID}_${LEV}.nc
set_nudge_sss=.true.
set_nudge_sst=.false.
set_nudge_sao_region1=.false.
set_nudge_tho_region1=.false.
set_nudge_sao_region2=.false.
set_nudge_tho_region2=.false.
if [ "${GRID}" = "GR60" ] ; then
ie=60
je=50
DT="${DT:-10800}"
tp=false
elif [ "${GRID}" = "TOY" ] ; then
ie=66
je=36
DT="${DT:-10800}"
tp=true
elif [ "${GRID}" = "MED11" ] ; then
ie=503
je=231
DT="900"
tp=false
caulapuv=0.0042
cah00=400.
cwt=2.5e-5
cstabeps=0.1
set_nudge_sao_region1=.true.,3.7e-7,1,1,1,120,8,29
set_nudge_tho_region1=.true.,3.7e-7,1,1,1,120,8,29
set_nudge_sao_region2=.true.,3.7e-7,1,9,8,133,8,29
set_nudge_tho_region2=.true.,3.7e-7,1,9,8,133,8,29
numriv=93
elif [ "${GRID}" = "K20" ] ; then
STA=-1
ie=100
je=100
DT=3600
tp=false
iocad=0
iocaduv=0
cah00=1000.
caulapuv=0.001
ibolk=0
lisopyc=.false.
dv0=0.0
av0=0.0
dback=0.0
aback=0.0
cdvocon=0.0
cavocon=0.0
cwt=0.0
cwa=0.0
set_nudge_sss=.false.
set_nudge_sao_region1=.false.
set_nudge_tho_region1=.false.
lwith_ice_dynamics=.false.
luse_buoyancy_forcing=.false.
luse_windstress_forcing=.false.
lcalc_arcgri=.true.
elif [ "${GRID}" = "ZC01" ] ; then
ie=101
je=51
DT="${DT:-3600}"
tp=false
caulapuv=0.001
set_nudge_sss=.false.
set_nudge_sao_region1=.false.
set_nudge_tho_region1=.false.
lwith_ice_dynamics=.false.
lisopyc=.false.
luse_buoyancy_forcing=.false.
luse_windstress_forcing=.false.
lcalc_arcgri=.true.
elif [ "${GRID}" = "GR30" ] ; then
ie=122
je=101
DT="${DT:-8640}"
tp=false
elif [ "${GRID}" = "GR3M" ] ; then
ie=144
je=87
DT="${DT:-8640}"
tp=false
elif [ "${GRID}" = "GR15" ] ; then
caulapuv=0.006
ibolk=250
ie=256
je=220
DT="${DT:-3600}"
tp=false
elif [ "${GRID}" = "GI6" ] ; then
SALINITY_CLIMATOLOGY=ts_phc3.0_annual_mpiom_${GRID}_${LEV}.nc
TEMPERATURE_CLIMATOLOGY=ts_phc3.0_annual_mpiom_${GRID}_${LEV}.nc
caulapuv=0.0045
ibolk=0
cah00=800.
ie=802
je=436
DT="${DT:-900}"
tp=false
cwt=3.5e-4
relsal=1.5e-7
cdvocon=0.15
lwith_one_layer_shelfs=.true.
elif [ "${GRID}" = "TP10" ] ; then
ie=362
je=192
[[ "${ltidal}" = ".true." ]] && DT="${DT:-3600}" || DT="${DT:-5400}"
tp=true
elif [ "${GRID}" = "TP04" ] ; then
ie=802
je=404
DT="${DT:-3600}"
tp=true
caulapuv=0.00375
iter_sor=300
rtsorpar=1.916
iter_sor_hack=10
rtsorpar_hack=0.7
elif [ "${GRID}" = "TP6M" ] ; then
ie=3602
je=2394
DT="${DT:-180}"
tp=true
caulapuv=0.00375
ibolk=0
iter_sor=2400
rtsorpar=1.934
iter_sor_hack=10
rtsorpar_hack=0.7
iocad=8
iocaduv=8
STA=2
fi
if [ "${LEV}" = "L3" ] ; then
ke=3
elif [ "${LEV}" = "L10" ] ; then
ke=10
elif [ "${LEV}" = "L20" ] ; then
ke=20
elif [ "${LEV}" = "L29" ] ; then
ke=29
elif [ "${LEV}" = "L31" ] ; then
ke=31
elif [ "${LEV}" = "L40" ] ; then
ke=40
elif [ "${LEV}" = "L80" ] ; then
ke=80
fi
#-----------------------------------------------------------------------------
#
cd ${EXPDIR} # output and rerun files are written into $ARCDIR
#
pwd
#-----------------------------------------------------------------------------
#
# initialisation or rerun
#
# if mpiom finds a restart file it tries to continue the run
if [[ -f rerun_${EXPNO}_mpiom.nc ]]; then
restart=yes
# file model_date.asc is updated by MPIOM at the end of each run
# it contains 'run_start' 'run_end' and 'next_run_start' in YYYY-MM-DD format
if [[ -f model_date.asc ]]; then
model_date=$(<model_date.asc)
run_start=$(echo ${model_date} | awk '{print $3}') # start date of actual run
else
run_start=${model_start}
fi
STA=3 # start from restart files
else
restart=no
run_start=${model_start}
# STA=1 # start from uniform ocean T=0.C and S=35.0
# STA=2 # start from Levitus
STA="${STA:-2}"
# set_nudge_sao_region1=.true.
# set_nudge_tho_region1=.true.
fi
# specification of files
#
#-----------------------------------------------------------------------------
#
#CP='cp -p'
#CP='ln -sf'
CP="/sw/rhel6-x64/cdo/cdo-1.8.0-magicsxx-gcc48/bin/cdo -b 64B copy"
CP2EXT="/sw/rhel6-x64/cdo/cdo-1.8.0-magicsxx-gcc48/bin/cdo -f ext -b 64B copy"
if [ ${lcalc_arcgri} == ".false." ] ; then
\cp ${INITIAL_DATA}/${GRID}/${GRID}_arcgri.nc arcgri.nc
fi
#\cp ${INITIAL_DATA}/${GRID}/${GRID}_topo topo
\cp /work/mh0256/m300212/work/be0510/b380002/MPIOM/experiments/tp6ml40_omip_restore/topo.lastversion topo
$CP ${INITIAL_DATA}/${GRID}/${GRID}_anta anta
\cp ${INITIAL_DATA}/${GRID}/${GRID}_BEK BEK
set +e
chmod u+rw BEK
set -e
if [ ${STA} == "2" ] ; then
/sw/rhel6-x64/cdo/cdo-1.8.0-magicsxx-gcc48/bin/cdo -selcode,5 ${INITIAL_DATA}/${GRID}/${SALINITY_CLIMATOLOGY} INISAL
/sw/rhel6-x64/cdo/cdo-1.8.0-magicsxx-gcc48/bin/cdo -selcode,2 ${INITIAL_DATA}/${GRID}/${TEMPERATURE_CLIMATOLOGY} INITEM
fi
if [[ ${set_nudge_sao_region1} == .true.* || ${set_nudge_sao_region2} == .true.* ]] ; then
/sw/rhel6-x64/cdo/cdo-1.8.0-magicsxx-gcc48/bin/cdo -setcode,58 -selcode,5 ${INITIAL_DATA}/${GRID}/${SALINITY_CLIMATOLOGY} RELSAL
fi
if [[ ${set_nudge_tho_region1} == .true.* || ${set_nudge_tho_region2} == .true.* ]] ; then
/sw/rhel6-x64/cdo/cdo-1.8.0-magicsxx-gcc48/bin/cdo -setcode,59 -selcode,2 ${INITIAL_DATA}/${GRID}/${TEMPERATURE_CLIMATOLOGY} RELTEM
fi
if [[ ${set_nudge_sss} == .true.* ]] ; then
/sw/rhel6-x64/cdo/cdo-1.8.0-magicsxx-gcc48/bin/cdo -sellevidx,1 -setcode,62 -selcode,5 ${INITIAL_DATA}/${GRID}/${SALINITY_CLIMATOLOGY} RELSSS
fi
if [[ ${set_nudge_sst} == .true.* ]] ; then
/sw/rhel6-x64/cdo/cdo-1.8.0-magicsxx-gcc48/bin/cdo -sellevidx,1 -setcode,63 -selcode,2 ${INITIAL_DATA}/${GRID}/${TEMPERATURE_CLIMATOLOGY} RELSST
fi
if [ "${GRID}" = "MED11" ] ; then
$CP ${INITIAL_DATA}/RUNOFF/runoff_obs_med runoff_obs
$CP ${INITIAL_DATA}/RUNOFF/runoff_pos_med runoff_pos
fi
#-----------------------------------------------------------------------------
if [ ${hamocc} == "true" ] ; then
# monthly mean dust field for hamocc
\rm -f inpdust_r2.nc
mm=${run_start%-??}
mm=${mm#????-}
/sw/rhel6-x64/cdo/cdo-1.8.0-magicsxx-gcc48/bin/cdo -setcode,34 -selmon,${mm}/12 ${INITIAL_DATA}/${GRID}/${GRID}_MAHOWALDDUST_2005.nc inpdust_r2.nc
if [ ${enable_ndepo} == "yes" ] ; then
\rm -f inpnit.nc
ryear=${run_start%-??-??}
syear=${model_start%-??-??}
year=$( expr ${ndep_start} + ${ryear} - ${syear})
if [ ${use_transient_ndepo} == "yes" ] ; then
/sw/rhel6-x64/cdo/cdo-1.8.0-magicsxx-gcc48/bin/cdo -selmon,${mm}/12 -selyear,${year} ${INITIAL_DATA}/${GRID}/${GRID}_ndepo_CMIP_NCAR_CCMI-1-0_gr_185001-201412.nc inpnit.nc
if [ ${year} -ge '2015' ] ; then
# data for 2015-2100 are not yet available
# /sw/rhel6-x64/cdo/cdo-1.8.0-magicsxx-gcc48/bin/cdo -selmon,${mm}/12 -selyear,${year} ${INITIAL_DATA}/${GRID}/${GRID}_ndepo_CMIP_NCAR_CCMI-1-0_gr_201501-210012.nc inpnit.nc
echo "Data for 2015-2100 are not yet available: exiting"
exit 1
fi
else
/sw/rhel6-x64/cdo/cdo-1.8.0-magicsxx-gcc48/bin/cdo -selmon,${mm}/12 ${INITIAL_DATA}/${GRID}/${GRID}_ndepo_CMIP_NCAR_CCMI-1-0_gr_185001-185012-clim.nc inpnit.nc
fi
fi
if [ ${read_ghgfile} == "yes" ] ; then
\cp ${INITIAL_DATA}/GHG/ghg_lawdome_giss_esrl_noaa_2015.txt ghg.txt
set +e
chmod u+rw ghg.txt
set -e
fi
fi
set +e
chmod u+rw arcgri.nc topo anta BEK GI* INITEM INISAL
set -e
#-----------------------------------------------------------------------------
for lll in 1 ; do
echo ${run_start}
(( ntime = 86400 / forcing_frequency ))
if [ ${enable_cfc} == "yes" ] ; then
CFC=',42,45'
typeset -Z4 cfc_year
cfc_year=$(echo ${run_start} | awk -F\- '{print $1}')
if [ ${cfc_year} -le '1910' ] ; then
CFC=''
cfc_year=1910
fi
cdo selyear,${cfc_year} ${INITIAL_DATA}/CFC/CFC_Atm_Hist_2015.nc CFC_Atm.nc
fi
if [ ${enable_age} == "yes" ] ; then
AGE=',39'
fi
if [ ${enable_decay} == "yes" ] ; then
DECAY=',40'
fi
lcaesium_source='.false.'
if [ ${enable_cs137} == "yes" ] ; then
CS137=',41'
lcaesium_source=.true.
caesium_source_latlon=34.42,141.02 #Fukushima
fi
OMIP_DATA=/pool/data/MPIOM/input/r0008/SOURCE/forcing365/original_data
NCEP_DATA=/pool/data/MPIOM/input/r0008/SOURCE/ncep
NOAA_DATA=
ERA_DATA=
ERAINT_DATA=
forcing=NCEP6
# use OMIP runoff data
if [[ ! -s omip_366_era15_runoff.${ntime}.nc ]] then
/sw/rhel6-x64/cdo/cdo-1.8.0-magicsxx-gcc48/bin/cdo seltimestep,365 ${OMIP_DATA}/runoff.nc runoff_1day.nc # select last day
/sw/rhel6-x64/cdo/cdo-1.8.0-magicsxx-gcc48/bin/cdo -O mergetime -settaxis,0000-01-01,00:00,1day ${OMIP_DATA}/runoff.nc -setdate,0000-12-31 runoff_1day.nc -setdate,0001-01-01 runoff_1day.nc omip_366_era15_runoff.nc # append 2 days
if [[ ${ntime} -gt 1 ]]; then # time interpolation
/sw/rhel6-x64/cdo/cdo-1.8.0-magicsxx-gcc48/bin/cdo intntime,${ntime} omip_366_era15_runoff.nc omip_366_era15_runoff.${ntime}.nc
else # daily forcing
cp -p omip_366_era15_runoff.nc omip_366_era15_runoff.${ntime}.nc
fi
fi
typeset -Z4 year yearend
year=${run_start%-??-??}
forcing_start_year=${forcing_start%-??-??}
(( yearend = year + nyears + nmonts/12 + ndays/365 + 1 ))
while [[ ${year} -le ${yearend} ]]; do
(( forcing_year = year + forcing_start_year - ${model_start%-??-??} ))
if [[ ${forcing_periodicity} -gt 0 ]]; then
(( forcing_year = forcing_start_year + (forcing_year-forcing_start_year)%forcing_periodicity ))
fi
$CP ${OMIP_DATA}/land_sea_mask.ECMWF.nc NCEP_LSM_RUNOFF
$CP ${NCEP_DATA}/land.sfc.gauss.nc NCEP_LSM
$CP ${NCEP_DATA}/uflx.sfc.gauss.${forcing_year}.nc NCEP_WIX_${forcing_year}
$CP ${NCEP_DATA}/vflx.sfc.gauss.${forcing_year}.nc NCEP_WIY_${forcing_year}
$CP ${NCEP_DATA}/air.2m.gauss.${forcing_year}.nc NCEP_TEM_${forcing_year}
$CP ${NCEP_DATA}/prate.sfc.gauss.${forcing_year}.nc NCEP_PREC_${forcing_year}
/sw/rhel6-x64/cdo/cdo-1.8.0-magicsxx-gcc48/bin/cdo mulc,0.89 ${NCEP_DATA}/dswrf.sfc.gauss.${forcing_year}.nc NCEP_SWRAD_${forcing_year}
$CP ${NCEP_DATA}/tdew.sfc.gauss.${forcing_year}.nc NCEP_TDEW_${forcing_year}
$CP ${NCEP_DATA}/wnd.10m.gauss.${forcing_year}.nc NCEP_WIND10_${forcing_year}
$CP ${NCEP_DATA}/tcdc.eatm.gauss.${forcing_year}.nc NCEP_CLOUD_${forcing_year}
ln -sf omip_366_era15_runoff.${ntime}.nc NCEP_RIV_${forcing_year} # use OMIP runoff data
#$CP ${NCEP_DATA}/runof.sfc.gauss.${forcing_year}.nc NCEP_RIV_${forcing_year}
#$CP ${NCEP_DATA}/pres.sfc.gauss.${forcing_year}.nc NCEP_PRESS_${forcing_year}
#$CP ${NCEP_DATA}/dlwrf.sfc.gauss.${forcing_year}.nc NCEP_LWRAD_${forcing_year}
#$CP ${NCEP_DATA}/uwnd.10m.gauss.${forcing_year}.nc NCEP_U10_${forcing_year}
#$CP ${NCEP_DATA}/vwnd.10m.gauss.${forcing_year}.nc NCEP_V10_${forcing_year}
(( year = year + 1 ))
done
#
# istart=0 >NEWSTART USE ONLY FOR COMPLETELY NEW SETUP (new topography etc!!!)
# istart=1 start from horizonally uniform t,s profiles
# istart=2 start from levitus
# istart=3 start from existing restart files rerun_mpiom, rerun_hamocc (default)
#
# Advection schemes
# iocad=3 ADPO (default)
# iocad=4 obsolete, use iocad=3 and ibbl_transport=1
# iocad=5 ADFS
#
# BBL transport (slope convection)
# ibbl_transport=1 (default)
# nfixYearLen = -1, 365 or 360 ; default -1 for 365/366 (leap years)
#
# ltidal=.true. enables eph.tidal sub model (default=false)
#
# nyears: number of years to be simulated (default=0)
# nmonths: number of months to be simulated (default=1)
#
model_start=$(echo ${model_start} | sed -e "s/-/,/g" -e "s/^,/-/"),0,0,0
forcing_start=$(echo ${forcing_start} | sed -e "s/-/,/g" -e "s/^,/-/"),0,0,0
cat > OCECTL << EOF
&proctl
ltimer = .true.
/
&ocedim
ie_g = ${ie}
je_g = ${je}
ke = ${ke}
lbounds_exch_tp = .${tp}.
/
&nprocs
nprocx=${nprocx}
nprocy=${nprocy}
/
&ocectl
dt = ${DT}
caulapts= 0.
caulapuv= ${caulapuv}
aus = 0.
cah00 = ${cah00}
ibolk = ${ibolk}
dv0 = ${dv0}
av0 = ${av0}
cwt = ${cwt}
cwa = ${cwa}
cstabeps= ${cstabeps}
dback = ${dback}
aback = ${aback}
cdvocon = ${cdvocon}
cavocon = ${cavocon}
nfixYearLen = ${nfixYearLen}
ltidal = ${ltidal}
ltidal_diag=${ltidal_diag}
lsatune=${lsatune}
lswr_jerlov = .true.
jerlov_atten = 0.06
jerlov_bluefrac = 0.41
lfb_bgc_oce = .false.
nyears = ${nyears}
nmonts = ${nmonts}
ndays = ${ndays}
istart = ${STA}
set_nudge_sss = ${set_nudge_sss}, 3.3e-7
set_nudge_sst = ${set_nudge_sst}, 3.3e-7
set_nudge_sao_region1 = ${set_nudge_sao_region1}
set_nudge_tho_region1 = ${set_nudge_tho_region1}
set_nudge_sao_region2 = ${set_nudge_sao_region2}
set_nudge_tho_region2 = ${set_nudge_tho_region2}
lmpitype = .false.
lnonblock = .true.
icontro = 0
iocad = ${iocad}
iocaduv = ${iocaduv}
ibbl_transport = 1
lundelayed_momentum_advection = .true.
rleadclose=0.25,3.0,2.0
h0=0.5
iter_sor = ${iter_sor}
rtsorpar = ${rtsorpar}
iter_sor_hack = ${iter_sor_hack}
rtsorpar_hack = ${rtsorpar_hack}
model_start_time = ${model_start}
time_verbosity = 1
lzo_correct = .true.
lsaoclose = .true.
rice_cutoff_factor = 0.9
lcaesium_source = ${lcaesium_source}
lakes = .false.
lwith_ice_dynamics = ${lwith_ice_dynamics}
lisopyc = ${lisopyc}
luse_buoyancy_forcing = ${luse_buoyancy_forcing}
luse_windstress_forcing = ${luse_windstress_forcing}
lcalc_arcgri = ${lcalc_arcgri}
luse_sp_diag = .true.
fp_tracing_enabled = .false.
testbed = 0
luse_cmip6_varlist = .true.
numriv = ${numriv}
lwith_one_layer_shelfs= ${lwith_one_layer_shelfs}
/
EOF
if [ ${ke} -eq 3 ] ; then
cat >> OCECTL << EOF2
&ocedzw
cdzw = 12.,10.,5000.,
/
EOF2
fi
if [ ${ke} -eq 10 ] ; then
cat >> OCECTL << EOF2
&ocedzw
cdzw = 40., 50., 90., 160., 270., 390., 550., 900., 1300., 2300.
/
EOF2
fi
if [ ${ke} -eq 20 ] ; then
cat >> OCECTL << EOF2
&ocedzw
cdzw = 20.,20., 20., 30.,40.,50.,70.,
90.,120.,150.,180.,210.,250.,300.,
400.,500.,600.,700.,900.,1400.,
/
EOF2
fi
if [ ${ke} -eq 29 ] ; then
cat >> OCECTL << EOF2
&ocedzw
cdzw = 12.,10.,10.,10.,11.,13.,16.,19.,23., 23., 28., 33., 40.,
48., 58., 70., 84.,102.,122.,148.,178.,214.,258.,311.,
375.,400.,450.,500.,500.
/
EOF2
fi
if [ ${ke} -eq 31 ] ; then
cat >> OCECTL << EOF2
&ocedzw
cdzw = 10.,10.,10.,10.,10.,10.,10.,10.,10.,10.,
15.,15.,15.,15.,15.,15.,15.,15.,20.,20.,
20.,20.,20.,20.,25.,25.,30.,30.,30.,30.,30.,
/
EOF2
fi
if [ ${ke} -eq 40 ] ; then
cat >> OCECTL << EOF2
&ocedzw
cdzw = 20.,10.,10.,10.,10.,10.,13.,15.,20.,25.,
30.,35.,40.,45.,50.,55.,60.,70.,80.,90.,
100.,110.,120.,130.,140.,150.,170.,180.,190.,200.,
220.,250.,270.,300.,350.,400.,450.,500.,500.,600.,
/
EOF2
fi
if [ ${ke} -eq 80 ] ; then
cat >> OCECTL <<EOF2
&ocedzw
cdzw = 12.,10.,10.,10.,10.,10.,10.,11.,11.,12.,
13.,13.,14.,14.,15.,16.,16.,17.,18.,19.,
20.,21.,21.,22.,24.,25.,26.,27.,28.,29.,
31.,32.,34.,35.,37.,39.,40.,42.,44.,46.,
48.,50.,53.,55.,58.,60.,63.,66.,69.,72.,
76.,79.,83.,87.,91.,95.,99.,104.,108.,113.,
119.,124.,130.,136.,142.,149.,155.,163.,170.,178.,
186.,195.,204.,213.,223.,233.,244.,255.,267.,279.,
/
EOF2
fi
# I/O (ioctl) namelist syntax
#
# iolist(N) = TYPE, FILE, FORMAT, CODE[, CODE...]
# N: unique number from 1 to 100 (not necessarily in sequence)
# TYPE: 99: restart file, 90: snapshot at start, 98: snapshot at end,
# 91: start file (read-only),
# 1: daily mean, 2: monthly mean, 3: annual mean, 4: per timestep,
# 5: 12-hourly mean, 6: 6h mean, 7: 3h mean, 8: 2h mean, 9: 1h mean.
# Adding 100 gives snapshot instead, eg 101: snapshot at end of day;
# adding 200 leaves land unmasked, 400 uses double precision output,
# eg monthly output formatted like a restart file is 702 (2+100+200+400);
# adding 800 defines input instead of output, eg 802: read record from
# file for every month, 890: read at start (same as 91);
# adding 1600 triggers synchronized mode, i.e. write buffers are flushed
# after every step, useful for debugging;
# adding 3200 allows input files to start over at end of file,
# needed for multi-year runs with climatology,
# adding 6400 triggers time minimum instead of time mean,
# adding 12800 triggers time maximum instead of time mean,
# adding 25600 triggers time summation instead of time mean,
# NB: for input files, offsets 100-400, 1600, and file format are ignored!
# FILE: file name enclosed by quotes ('...') or double quotes ("...")
# FORMAT: file format: 'nc' NetCDF 1, 'nc2' NetCDF 2/3, 'nc4' NetCDF 4,
# 'grb' GRIB 1, 'grbsz'/'sz' Compressed GRIB
# CODE...: list of variable code numbers (max. 255 codes)
FRMT=nc2
cat >> OCECTL <<EOF2
&ioctl
iolist(100) = 99, 'rerun_${EXPNO}_mpiom.nc', '${FRMT}', 1, 2, 3, 4, 5, 7, 9, 10,
13, 15, 35, 36, 82, 99, 110, 111, 141, 501, 502, 503, 504, 507, ${CFC} ${AGE} ${DECAY} ${CS137}
iolist(93) = 109, '${EXPNO}_mpiom_data_zos_1hr.nc','${FRMT}',1
! iolist(19) = 109, '${EXPNO}_mpiom_u_3d_1hr.nc','${FRMT}',3
! iolist(18) = 109, '${EXPNO}_mpiom_v_3d_1hr.nc','${FRMT}',4
! iolist(17) = 109, '${EXPNO}_mpiom_w_3d_1hr.nc','${FRMT}',7
! iolist(16) = 109, '${EXPNO}_mpiom_p_3d_1hr.nc','${FRMT}',6
! iolist(15) = 109, '${EXPNO}_mpiom_rho_3d_1hr.nc','${FRMT}',8
! iolist(14) = 109, '${EXPNO}_mpiom_tho_3d_1hr.nc','${FRMT}',2
! iolist(13) = 109, '${EXPNO}_mpiom_sao_3d_1hr.nc','${FRMT}',5
iolist(98) = 2, '${EXPNO}_mpiom_timeser_mm.nc','${FRMT}', 508, 509, 515, 516, 512,513,514,515,516,517,518,519,520,644,645,724,725,684,685,590,591,592,593,594,595,596,597,598,599,612,622,632,642,652,662,672,682,692,702,712,722,732,742,752,762,770,771,772,780,781,782,790,791,792,800,801,802,810,811,812,600,601,602,606,607,608,609,809,820,821,822,823,824,825,826,827,828,829,830,831,832,833,834,835,836,837,838,839,840,841,842,843,844,845,846,847,848,849,850,851,852,853,854,855,856,857,858,859,860,861,862,863,864,865,866,867,868,869,870,871,872,873,874,875,876,877,878,879,880,881,882,883,884,885,886,887,888,889,890,891,892,893,894,895,896,897,898,899,900,901,902,903,904,905,906,907,908,909,910,911,912,913,914,915,916,917,918,919,920,921,922,923,924,925,926,927,928,929,930,931,932,933,934,935,936,937,938,939,940,941,942,943,944,945,946,947,948,949,950,951,952,953,954,955,956,957,958,959,960,961,962,963,613,614,615,623,624,625,633,634,635,703,704,705,1030,1031,1032,1033,1034,1035
iolist(97) = 2, '${EXPNO}_mpiom_data_moc_mm.nc','${FRMT}', 93, 94, 95, 96, 97, 98, 100, 101, 102, 106, 107, 108,
1093,1094,1095,1096,1097,1098,1099,1100,1101, 1112, 1113, 1114
iolist(96) = 2, '${EXPNO}_mpiom_monitoring_mm.nc','${FRMT}', 508, 509, 515, 516, 519, 520, 622, 652,
672, 703, 800, 836, 837, 900, 901, 936, 950, 951, 952, 953,
955, 956, 958, 959, 961, 962, 1001
iolist(92) = 2, '${EXPNO}_mpiom_velocity_EKE_mm.nc','${FRMT}', 232, 237, 3, 4
iolist(99) = 2, '${EXPNO}_mpiom_data_2d_mm.nc','${FRMT}', 1, 11, 12, 13, 14,
15, 16, 17, 19, 20, 27, 35, 36, 66, 67, 70, 78, 79, 104, 109, 123, 124, 125, 126, 127, 128,
134, 136, 138, 139, 140, 141, 149, 150, 151, 153, 165, 169, 170, 181, 183, 215, 216,
217, 218, 219, 220, 221, 222, 223, 224, 226, 227, 228, 229, 275, 276, 277, 278, 280, 285, 286,
287, 288, 521, 522, 523, 524, 548, 549,
1013, 1015, 1016, 1018, 1019, 1020, 1021, 1022, 1023, 1024, 1025, 1026, 1027, 1028, 1036, 1037, 1038,
1039, 1040, 1041, 1042, 1043, 1044 ,1047, 1048
iolist(95) = 2, '${EXPNO}_mpiom_data_3d_mm.nc', '${FRMT}', 2, 3, 4, 5, 6, 7, 18, 21, 22, 23, 24, 54, 110, 112, 135, 197, 203, 204, 207, 208, 214, 279 ${AGE} ${DECAY} ${CS137}
iolist(76) = 1,'${EXPNO}_mpiom_100m_dm.nc','${FRMT}',245,246,247,248,253
iolist(75) = 1,'${EXPNO}_mpiom_2000m_dm.nc','${FRMT}',249,250,251,252,253,254
iolist(90) = 1, '${EXPNO}_mpiom_data_2d_dm.nc','${FRMT}', 12, 13, 14, 15, 16, 104, 109, 140, 141, 1016, 1022, 1023, 1024, 1025, 1026, 1027, 1028
$(
if [ ${enable_cfc} == "yes" ]
then
echo "iolist(82) = 803,'CFC_Atm.nc', 'nc4',43,44,46,47"
echo "iolist(83) = 2,'${EXPNO}_mpiom_cfc_mm.nc','${FRMT}' ${CFC}"
fi
)
$(
if [ ${STA} == "2" ]
then
echo " iolist(84) = 890,'INISAL', 'nc4',5"
echo " iolist(85) = 890,'INITEM', 'nc4',2"
fi
)
$(
if [[ ${set_nudge_sao_region1} == .true.* || ${set_nudge_sao_region2} == .true.* ]] ; then
echo " iolist(86) = 890,'RELSAL', 'nc4',58"
fi
)
$(
if [[ ${set_nudge_tho_region1} == .true.* || ${set_nudge_tho_region2} == .true.* ]] ; then
echo " iolist(87) = 890,'RELTEM', 'nc4',59"
fi
)
$(
if [[ ${set_nudge_sss} == .true.* ]] ; then
echo " iolist(88) = 890,'RELSSS', 'nc4',62"
fi
)
$(
if [[ ${set_nudge_sst} == .true.* ]] ; then
echo " iolist(89) = 890,'RELSST', 'nc4',63"
fi
)
/
EOF2
cat >> OCECTL <<EOF2
&forcctl
cforcdata='NCEP'
forcing_frequency = ${forcing_frequency}
forcing_start_time = ${forcing_start}
lwrite_forcing=.false.
lspat_interp_forcing=.${lspat_interp_forcing}.
ltime_interp_forcing=.${ltime_interp_forcing}.
forcing_periodicity = ${forcing_periodicity}
luse_model_time = .true.
lperiodic_forcing=.false.
ldebug_forcing=.false.
ldiff_runoff_grid=.true.
/
EOF2
#
# hamocc
#
if [ ${hamocc} == "true" ] ; then
millennium_ctrl=false
if [ ${millennium_ctrl} = "false" ]; then
deltacalc=0.0
deltaorg=0.0
deltasil=0.0
else
deltacalc=493.57
deltaorg=2.457
deltasil=0.22
fi
if [ ${read_ghgfile} == "yes" ] ; then
typeset -Z4 ghg_year
ghg_year=${run_start%-??-??}
co2ppm=$(awk "/^ *${ghg_year}\.00/ { print \$2 }" ghg.txt)
else
co2ppm=278.0
fi
if [ ${enable_isotopes} == "yes" ] ; then
Riso=', 8, 9, 18, 19, 25, 26, 39, 40, 42, 43, 52, 53, 89, 90'
Oiso=', 8, 9, 18, 19, 25, 26, 89, 90'
Siso=', 39, 40, 42, 43, 52, 53, 87, 88, 288, 289'
Aiso=', 103, 104'
else
Riso=''
Oiso=''
Siso=''
Aiso=''
fi
if [ ${enable_cyano} == "yes" ] ; then
RCYA=', 145'
OCYA=', 145, 166'
MCYA=', 520'
EUCYA=', 297'
LCYA='.true.'
else
RCYA=''
MCYA=''
OCYA=''
EUCYA=''
LCYA='.false.'
fi
if [ ${enable_martin} == "yes" ] ; then
LMAR='.true.'
else
LMAR='.false.'
fi
if [ ${enable_ndepo} == "yes" ] ; then
LNDEP='.true.'
else
LNDEP='.false.'
fi
#creates daily averages for bgc timeseries
(( TSDT = 86400 / DT ))
# For ioctl namelist syntax, see MPIOM section above
cat > NAMELIST_BGC << EOF2
&BGCCTL
deltacalc = ${deltacalc}
deltaorg = ${deltaorg}
deltasil = ${deltasil}
io_stdo_bgc = 8,
kchck = 0,
isac = 1,
rmasko = -9e33
atm_co2 = ${co2ppm}
l_cyadyn = ${LCYA}
lmartin = ${LMAR}
lndep = ${LNDEP}
/
&ioctl
!
! iolist elements from 100 - 80 are reserved for default output definitions
!
iolist(100) = 99, 'rerun_${EXPNO}_hamocc.nc', '${FRMT}', 7, 10, 11, 12, 13, 14,
15, 16, 17, 20, 21, 22, 23, 24, 27, 28, 29, 30, 31, 37, 38,
41, 44, 45, 46, 47, 48, 49, 50, 51, 54, 55, 56, 57, 58, 59,
60, 61, 62, 64, 203, 204, 205${RCYA}${Riso}
iolist(99) = 4002, 'inpdust_r2.nc', '${FRMT}', 34
$(
if [ ${enable_ndepo} == "yes" ]
then
echo " iolist(98) = 4002,'inpnit.nc', '${FRMT}',35"
fi
)
iolist(97) = 3, '${EXPNO}_hamocc_data_3d_ym.nc', '${FRMT}', 7, 10, 11, 12, 14,
15, 16, 17, 20, 21, 22, 23, 24, 27, 28, 29, 31, 37, 158${OCYA}${Oiso}
iolist(96) = 2, '${EXPNO}_hamocc_eu_data_mm.nc', '${FRMT}', 69, 70, 71, 84, 85,
86, 100, 101, 292, 293, 294, 295, 296, 297, 143, 510${EUCYA}
iolist(95) = 2, '${EXPNO}_hamocc_data_2d_mm.nc', '${FRMT}', 67, 68, 72, 75, 78,
81, 92, 93, 94, 95, 107, 110, 111, 112, 114, 115, 116, 117,
120, 121, 122, 123, 124, 127, 129, 131, 137, 157, 159, 160,
161, 162, 163, 164, 165, 174, 185, 186, 200, 206, 207, 210, 211,
214, 215, 231${Aiso}
iolist(94) = 101, '${EXPNO}_hamocc_co2.nc', '${FRMT}', 301, 303, 304, 305, 306,
307, 308, 309, 310, 311, 312, 313, 314, 203, 204, 205
iolist(93) = 3, '${EXPNO}_hamocc_monitoring_ym.nc', '${FRMT}', 500, 501, 502,
503, 504, 505, 506, 507, 508, 509, 517, 519, 523${MCYA}, 529, 530,
531, 532, 533, 534, 535
iolist(92) = 3, '${EXPNO}_hamocc_sedi_ym.nc', '${FRMT}', 38, 41, 44, 45, 46, 47,
48, 49, 50, 51, 54, 55, 56, 57, 58, 59, 60, 94, 95, 96, 97,
167,168, 280, 281, 282, 283, 284, 285, 286, 287${Siso}
! iolist(91) = 2,'${EXPNO}_hamocc_eddy_mm.nc','${FRMT}',315,316,317,318,319,320,
! 321,322,323,324,325,326,234,235,236,72
! iolist(90)=2,'${EXPNO}_hamocc_stations_mm.nc','${FRMT}',
! 1001,1002,1003,1004,1005,1006,1007,1008,1009,1010,1011,
! 2001,2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,
! 3001,3002,3003,3004,3005,3006,3007,3008,3009,3010,3011
/
EOF2
#
fi
#=============================================================================
echo "Integration started on $(date)"
export I_MPI_FABRICS=shm:dapl
export I_MPI_FALLBACK=disable
export I_MPI_SLURM_EXT=1
# Set I_MPI_LARGE_SCALE_THRESHOLD to a value larger than the number of your MPI-tasks if you use 8192 or more tasks.
export I_MPI_LARGE_SCALE_THRESHOLD=8192
srun --propagate=STACK --kill-on-bad-exit=1 --cpu_bind=verbose,cores --distribution=block:block -n ${ncpus} ${mpilaunch_args} ${MODDIR}/${MODBIN}
echo "Integration completed on $(date)"
#=============================================================================
if [[ "${profile}" != "" ]]; then
set +e
jobid=$(echo $LOADL_STEP_ID | awk -F. '{print $(NF-1)}')
profdir=prof_${run_start}.${jobid}
mkdir ${profdir}
mv gmon.out* ${profdir} 2>/dev/null || echo 'INFO: no gmon.out profiling data present'
mv poe.prof ${profdir}/mpiom_tprof.out 2>/dev/null || echo 'INFO: no tprof profiling data present'
mv rusage ${profdir} 2>/dev/null || echo 'INFO: no rusage profiling data present'
mv mpi_profile* single_trace* ${profdir} 2>/dev/null || echo 'INFO: no MPI profiling data present'
mv hpc_profile*.hpm hpc_profile*.viz ${profdir} 2>/dev/null || echo 'INFO: no HPM profiling data present'
cp -p OCECTL ${profdir}
tar cvf ${profdir}/oceout.tar oceout*
[[ ${hamocc} == "true" ]] && tar cvf ${profdir}/bgcout.tar bgcout*
rm -f oceout* bgcout*
fi
[[ -d ${ARCDIR}/restart ]] || /bin/mkdir -p ${ARCDIR}/restart
[[ -d ${ARCDIR}/outdata ]] || /bin/mkdir -p ${ARCDIR}/outdata
model_date=$(<model_date.asc)
run_start=$(echo ${model_date} | awk '{print $1}')
run_end=$(echo ${model_date} | awk '{print $2}')
next_run_start=$(echo ${model_date} | awk '{print $3}')
timestamp_out=${run_start}_${run_end}
timestamp_res=${run_end}
# restart files
for i in rerun_${EXPNO}_* ; do
if [ -r $i ]; then
# insert timestamp between basename and extension
n=`echo $i | sed "s,^\(.*\)\.\(.*\)$,\1_${timestamp_out}.\2,"`
cp $i ${ARCDIR}/restart/${n}
fi
done
# outdata files
for i in ${EXPNO}_mpiom_* ${EXPNO}_hamocc_* ; do
if [ -r $i ]; then
# insert timestamp between basename and extension
n=`echo $i | sed "s,^\(.*\)\.\(.*\)$,\1_${timestamp_out}.\2,"`
mv $i ${ARCDIR}/outdata/${n}
fi
done
if [[ ${hamocc} == "true" ]] && [[ "${profile}" == "" ]]; then
cp bgcout ${ARCDIR}/outdata/bgcout_${timestamp_out}
cp oceout ${ARCDIR}/outdata/oceout_${timestamp_out}
fi
#=============================================================================
#=============================================================================
done
#=============================================================================
if [[ ${SINFO} == "yes" ]]; then
#write experiment short info sheet to SINFO_DIR
cat > $EXPNO.info<<EOF
user: ${LOGNAME}
experiment_number : ${EXPNO}
working_directory : ${EXPDIR}
archive_directory : ${ARCDIR}
executable : ${MODDIR}/${MODBIN}
grid : ${GRID}
level : ${LEV}
enable_hamocc : ${hamocc}
intial_data : ${INITIAL_DATA}
forcing_data : NCEP6
experiment_start : ${model_start}
experiment_end : ${model_end}
restart : ${restart}
experiment_short_info : ${SHORT_INFO}
EOF
if [[ ! -f ${SINFO_DIR} ]]; then
set +e
mkdir -p ${SINFO_DIR}
set -e
fi
if [[ ! -f ${SINFO_DIR}/${EXPNO}.info && -d ${SINFO_DIR} && -w ${SINFO_DIR} ]]; then
\cp $EXPNO.info ${SINFO_DIR}/${EXPNO}.info
fi
fi
#=============================================================================
nsdate=$(echo ${next_run_start} | tr -d '-')
fdate=$(echo ${model_end} | tr -d '-')
if [ ${nsdate} -lt ${fdate} ] ; then
echo "submitting next job"
sbatch /work/mh0256/m300212/STORMTIDE_run/Tides_NCEP_SAL2/experiments/hel17153-DZA_TP6ML40_mpiom-trunk/hel17153-DZA_TP6ML40_mpiom-trunk.job
fi
exit