! bn_aggregate_u.jnl
! Union aggregations including tests of UNION in combination
! with TSERIES, ENSEMBLE, and FMRC aggregations
! acm 7/15/2016
\
! create some variables in separate datasets.
define axis/t=1-jan-2010:10-jan-2010:1/units=days/t0=1-jan-2010 tax
let/title=sst  sst = x[x=1:5] + 1.3*y[y=1:4] + t[gt=tax]
let/title=temp temp = 0.8*x[x=1:5] - y[y=1:4] + t[gt=tax]
let/title=U    u = x[x=1:5] + 0.4*y[y=1:4] - 1.1* t[gt=tax]
let/title=V    v = x[x=1:5] - 0.2*y[y=1:4] + 1.1* t[gt=tax]

save/clobber/file="tmp/uagg_1.nc" sst
save/clobber/file="tmp/uagg_2.nc" temp
save/clobber/file="tmp/uagg_3.nc" u
save/clobber/file="tmp/uagg_4.nc" v

save/l=1:3/clobber/file="tmp/uagg_1_t1.nc" sst
save/l=4:6/clobber/file="tmp/uagg_1_t2.nc" sst
save/l=7:10/clobber/file="tmp/uagg_1_t3.nc" sst

can dat/all
union tmp/uagg_1.nc,tmp/uagg_2.nc,tmp/uagg_3.nc,tmp/uagg_4.nc
sh dat
vec/l=5 u,v

cancel dat/all

! issue a note if a variable is already in the Union
! a new file with different defiontions of sst and temp
!
let/title="sst in uagg_5"  sst = 1.1*x[x=1:5] + 1.3*y[y=1:4] + t[gt=tax]
let/title="temp in uagg_5" temp = 0.9*x[x=1:5] - y[y=1:4] + t[gt=tax]
save/clobber/file="tmp/uagg_5.nc" sst,temp

! In the union set, sst comes from uagg_1, temp comes from uagg_5
union uu = tmp/uagg_1.nc,tmp/uagg_5.nc
sh dat/att
can dat/all

! Or use /quiet to supress the note
union/quiet uu = tmp/uagg_1.nc,tmp/uagg_5.nc

let tnames = SPAWN("ls -1 tmp/uagg_1_*.nc")
tseries  temp_time_agg = tnames
sh dat
union temp_time_agg, tmp/uagg_2.nc,tmp/uagg_3.nc,tmp/uagg_4.nc

shade/l=2 sst - temp


can dat/all; can var/all; can sym/all

! exercise 6D aggregations: UNION with T, E, and F simultaneously

! do not bother to echo the creation of files and the definitions of the
! many time aggregations
CANCEL MODE VERIFY



! assume the aggregate_tef script has alrady run, 
! leaving the examplefiles it writes, in tmp/

! check for those files.
IF `test_opendap("tmp/f03_e2_t3.nc") NE 0` THEN
   Say Exiting in bn_aggregate_u.jnl. The script needs files written by bn_aggregate_tef.jnl which are not found
   exit/program
ENDIF

! *** create an artificial file collection with another variable, fcst_3 ***

! a 20x20 lat-long region
define axis/x=140e:160e:1 xax
define axis/y=10n:20n:1 yax
define axis/Z=0:500:500/EDGES/units=m zax   ! a single point axis

! a 20 year monthly axis starting in Jan 1950
LET start_year = 1950
LET nyears = 20
LET indices = L[L=1:`nyears*12`]
LET month = MOD(indices-1,12)+1
LET year = start_year + INT((indices-1)/12)
DEFINE AXIS/UNITS=days/T0=1-jan-1900/EDGES truemonth = DAYS1900(year,month,1)

! define a moving field with errors increasing from a given forecast date
let vy = 1/500      ! speed that forecast structure deviates from truth
let vx = (1+(T[gt=truemonth]-tstart)/2000)* 1/300
let xt = x[gx=xax]-vx*T[gt=truemonth]
let yt = y[gy=yax]-vy*T[gt=truemonth]
let field = COS((xt-150)/5)*COS((yt-30)/5) + Z[gz=zax]
let fcst = ensemble_variation * (field + fcst_error)
define attribute fcst.im_everywhere = "so promote me"
set variable/title="some variable" fcst
let static_error = SIN((x[gx=xax]-140)/5)*SIN((y[gy=yax]-20)/5) + RANDU(x[gx=xax]+y[gy=yax])
let structure_drift = (t[gt=truemonth]-tstart)/5000 * static_error
let random_error = (t[gt=truemonth]-tstart)/2000 * RANDU(t[gt=truemonth]+x[gx=xax]+y[gy=yax])
let fcst_error = structure_drift + random_error

! Define another variable fcst_3
let/title="another variable" fcst_3 = fcst + 3 + 0.1*fcst_error


repeat/name=fdate/range=1:7:3 (\
  let tstart = t[gt=truemonth,l=`fdate`];\
  repeat/name=ens/range=1:2:1 (\
    let ensemble_variation = 1 + (ens-3)/20;\  
    let realization = `ens`;\
    save/clobber/file="tmp/f`(fdate-1)/3+1,zw=2`_e`ens`_tu1.nc"/l=`fdate`:`fdate+11` fcst_3, realization;\
    save/clobber/file="tmp/f`(fdate-1)/3+1,zw=2`_e`ens`_tu2.nc"/l=`fdate+12`:`fdate+23` fcst_3, realization;\
    save/clobber/file="tmp/f`(fdate-1)/3+1,zw=2`_e`ens`_tu3.nc"/l=`fdate+24`:`fdate+35` fcst_3, realization))

cancel variables/all

! ***** end of TEFU file creation ****************

SET MODE VERIFY
! define T aggregations

! timeseries of files with variable fcst_1
repeat/name=fdate/range=1:7:3 (\
  let tstart = t[gt=truemonth,l=`fdate`];\
  repeat/name=ens/range=1:2:1 (\
    let realization = `ens`;\
    TSERIES  t_f`(fdate-1)/3+1,zw=2`_e`ens` = SPAWN("ls -1 tmp/f`(fdate-1)/3+1,zw=2`_e`ens`_t?.nc")  ))

! timeseries of files with variable fcst_3
repeat/name=fdate/range=1:7:3 (\
  let tstart = t[gt=truemonth,l=`fdate`];\
  repeat/name=ens/range=1:2:1 (\
    let realization = `ens`;\
    TSERIES  t_f`(fdate-1)/3+1,zw=2`_e`ens`_3 = SPAWN("ls -1 tmp/f`(fdate-1)/3+1,zw=2`_e`ens`_tu?.nc")  ))

cancel variable realization

SET MODE VERIFY
! now union these to make datsets with fcnst_1 and fcst_3.

show data/files/br; 

DEFINE DATA/AGGREGAT/U  u1 = t_f01_e1_3, t_f01_e1
DEFINE DATA/AGGREGAT/U  u2 = t_f01_e2, t_f01_e2_3
show data u1 u2
can data u1 u2

! Make a batch of files, timeseries aggregations union-ed.
! the /QUIET supresses the note about skipping duplicate variables.
repeat/name=fdate/range=1:7:3 (\
  let tstart = t[gt=truemonth,l=`fdate`];\
  repeat/name=ens/range=1:2:1 (\
    let realization = `ens`;\
    UNION/quiet  t_f_u`(fdate-1)/3+1,zw=2`_e`ens` = t_f`(fdate-1)/3+1,zw=2`_e`ens`, t_f`(fdate-1)/3+1,zw=2`_e`ens`_3;\
    show data/br t_f_u`(fdate-1)/3+1,zw=2`_e`ens`))

cancel variable realization

! should share the same time axis
show grid fcst[d=t_f01_e1]
show grid fcst_3[d=t_f_u01_e2]

! Now go on and aggregate these time & union aggregations, in E and F

! ======== TEST SET 1:  aggregate E first, then F

! define tue aggregations -- only the first remains unhidden
ENSEMBLE      tue_f01 = t_f_u01_e1, t_f_u01_e2
ENSEMBLE/hide tue_f02 = t_f_u02_e1, t_f_u02_e1
ENSEMBLE/hide tue_f03 = t_f_u03_e1, t_f_u03_e2
SHOW DATA/brief
SHOW DATA/FULL tue_f03
SHOW DATA/ATT tue_f03   ! notice the promotion of fcst.im_everywhere

! define tuef aggregation
FMRC/HIDE tuef = tue_f01, tue_f02, tue_f03
SHOW DATA/brief/files
show data/full tuef

! The time-union-ensemble-forecast aggregation

set data tuef
show data tuef
sh grid fcst 
sh grid fcst_3

! simplest acess -- compact form -- just demonstrate it doesn't fail
stat/brief fcst[i=11:12, j=5:6, l=31:32, m=1:2, n=2:3]
stat/brief fcst_3[i=11:12, j=5:6, l=31:32, m=1:2, n=2:3]

! diagonal forecast view of ensemble mean
let diag_fcst = fcst[gt(tf_times)=TF_CAL_T,gf(tf_times)=TF_LAG_F]
shade/x=150E/y=15n diag_fcst[m=1:2@ave]

let diag_fcst_3 = fcst_3[gt(tf_times)=TF_CAL_T,gf(tf_times)=TF_LAG_F]
shade/x=150E/y=15n diag_fcst[m=1:2@ave]

! the difference will be just a small variation from the value 3
let diff_of_union = diag_fcst - diag_fcst_3

shade/x=150E/y=15n  diff_of_union[m=1:2@ave]
CANCEL DATA/ALL


