Commit 0abec463 authored by Ian Boyd's avatar Ian Boyd
Browse files

Upload New File

parent 1cc85c79
; 2011-06-01 Christian Retscher adapted to new GEOMS
; 2011-06-07 Christian Retscher H5 option added
; 2011-08-10 Ian Boyd send NDACC variable names to AmestoPointer_xx.pro;
; Convert fill values from string to double before sending to AmestoPointer_xx.pro to
; avoid round errors when using IDL default rules; account for no data quality
; information in the header of the duo3YYMM.gol files
; 2011-08-16 Ian Boyd account for if there is only a single altitude reported in the profile
; - previously redimensioned the vames array to a single dimension in this case, whereas
; a 2-D array is required; Force McGee 't2' files to call the template ss_te_pil_tg.meta
; template; Add Read_Instr_Metadata and Read_Species_Metadata
; 2011-09-02 Ian Boyd combined all AmestoPointer routines into one, so call changed in the code
; to refelect this
; 2013-03-01 Ian Boyd change SOURCE.PRODUCT format to Archive.Source_Archive.File_Archive.File.Generation.Date;
; Include Ames comments array as an argument in AmestoPointer procedure
; 2013-11-09 Ian Boyd add alt_inst variable so that the altitide of the instrument can be used to identify
; the correct site of the measurement at la Reunion
pro Read_Instr_Metadata,filename ,ga,ivar_data
;
; Read ASCII AVDC type metadata file containing instrument/pi specific information
; leblanc@tmf.jpl.nasa.gov (original code from BoBo)
; 2010-05-22
; ++++++++++++++++++++
;
openr,runit,filename,/get_lun
a='' & nattgen=31
while(not eof(runit)) do begin
readf,runit,a
if (strmid(a,0,19) eq '! Global Attributes' ) then begin
ga=indgen(nattgen,/string)
i=0
while (i lt nattgen ) do begin
readf,runit,a
if (strmid(a,0,1) ne '!' ) then begin
ga[i]=strtrim(a,2)
i=i+1
endif
endwhile
; Identify index of global attribute named "DATA_VARIABLES" for use in the species meta file
wv=where(strcmp(ga,'DATA_VARIABLES',14,/fold_case))
if (wv[0] ne -1 ) then ivar_data=wv
endif
endwhile
close,runit & free_lun, runit
return
end
pro Read_Species_Metadata,filename,ga,ivar_data ,va
; Read ASCII AVDC type metadata file containing template species information
; leblanc@tmf.jpl.nasa.gov (original code by BoBo)
; 2010-05-22
; ++++++++++++++++++++
openr,runit,filename,/get_lun
a='' & nv=0
while(not eof(runit)) do begin
readf,runit,a
; Determine the number of variables to be read in the species template file and assign corresponding global attribute (DATA_VARIABLES)
if strmid(a,0,14) eq 'DATA_VARIABLES' then begin
ga[ivar_data]=strtrim(a,2) & n_var=n_elements(strsplit(ga[ivar_data],';',/extract))
va=indgen(11,n_var,/string)
endif
if (a eq '! Variable Attributes' ) then begin
i=0
while (i lt 11) do begin
readf,runit,a
if (strmid(a,0,1) ne '!') then begin
va[i,nv]=strtrim(a,2)
i=i+1
endif
endwhile
nv=nv+1
endif
endwhile
close, runit & free_lun,runit
return
end
pro NDACC_Lidar_HDF,site,species,year,month, pi ,ames_dir=ames_dir $
,meta_dir=meta_dir $
,hdf_dir=hdf_dir $
,tav=tav $
,pi_metafile=pi_metafile $
,species_metafile=species_metafile $
,mcgee=mcgee $
,h5=h5 $
,nc=nc
;This procedure:
; 1/ Analyzes which NDACC lidar instrument/dataset is to be written out in HDF
; 2/ a/ If instrument/dataset exists in Ames format: Reads the data in Ames file, and re-assign them in proper HDF SDS
; b/ If instrument/dataset does not exist in Ames format: Prepare HDF SDS from user-defined input variables
; 3/ Calls procedure "idlcr8hdf.pro" which performs all QA checks and meta data compliance checks, then writes SDS into HDF output file
;Input (mandatory) arguments:
; site: 2 character string with definition following that of NDACC Data Handling Facility ("eu"=eureka, "tm"=Table Mountain, "ho"=Hohenpeissenberg, etc.)
; pi: 2 character string with definition following that of NDACC Data Handling Facility ("ne"=Neuber, "sw"=Swart, "md"=McDermid, etc.)
; species: 2 character string with definition following that of NDACC Data Handling Facility ("o3"=stratopsheric ozone, "te"=temperature, "to"=tropospheric ozone, etc.)
;Input (optional) keywords
; ames_dir= Name of directory where input Ames data file should be read (default is ~/Ames)
; meta_dir= Name of directory where input Meta data file should be read (default is ~/Meta)
; hdf_dir= Name of directory where input Ames data file should be read (default is ~/HDF)
; tav= Name of Table Attribute Value (TAV) file (default name provided by AVDC/EVDC/GECA)
; pi_metafile= Name of meta file where input GEOMS global attributes should be read
; species_metafile= Name of meta file where input GEOMS variable attributes should be read
;if keyword_set(tav) then tav=tav else tav='tableattrvalue_03R004_idl.dat' ; Default TAV name as provided by AVDC
NDACC_Lidar_Lib ;Routines in NDACC_Lidar_Lib required (must be in search path)
Forward_Function JulVar, StrJul, StrTL, SysDate ;Ensure these calls are treated as Functions
print, ames_dir
if keyword_set(tav) then tav=tav else tav='tableattrvalue.dat' ; Default TAV name as provided by AVDC
if keyword_set(meta_dir) then meta_dir=meta_dir else meta_dir='Meta/' ; Default Meta files directory is ~/Meta directoy
if keyword_set(ames_dir) then ames_dir=ames_dir else ames_dir='Ames/' ; Default Ames files directory is ~/Ames directoy
if keyword_set(hdf_dir) then hdf_dir=hdf_dir else hdf_dir='HDF/' ; Default HDF files directory is ~/HDF directoy
if keyword_set(mcgee) then mcgee=mcgee else mcgee=''
ay=strmid(strtrim(year,2),2,2) & ayy=strmid(strtrim(year,2),0,4) & amth=StrTL(strtrim(month,2),format='(i2)')
if species eq 'o3' or species eq 'to' or species eq 'te' then goto,species_ok
print,'Incorrect Species. Please retry...' & stop
species_ok:
amesfile=ames_dir+site+species+ay+amth+'.'+pi+'l'
if pi eq 'mg' then begin
; values for mcgee must be "tg" for STROZ or "t2" for AT
if keyword_set(mcgee) then amesfile=ames_dir+site+mcgee+ay+amth+'.'+pi+'l' else stop
endif else mcgee=''
if keyword_set(pi_metafile) then pi_metafile=pi_metafile else pi_metafile=meta_dir+site+'_'+species+'_'+pi+'l.meta'
if pi eq 'mg' then begin
if mcgee eq 'tg' then pi_metafile=meta_dir+site+'_'+species+'_'+pi+'l_'+mcgee+'.meta' $
else pi_metafile=meta_dir+site+'_'+species+'_'+pi+'l_'+'tg'+'.meta'
endif
if keyword_set(species_metafile) then species_metafile=species_metafile else species_metafile=meta_dir+'Lidar_'+species+'.meta'
; Read Ames data
ndscheader=' '
if findfile( amesfile ) eq '' then begin
print, 'ames file not found'
exit
endif
ex_rd2110,amesfile ,xname=xname, xvalues=xvals $
,yname=yname, yvalues=yvals, lengths=nypts $
,vname=vname, data=dat, vmiss=vmiss $
,aname=aname, auxdat=auxdat, amiss=amiss $
,n_dimens=n_dimens, nauxv=nauxv, err=err $
,oname=oname,sname=sname,org=org,mname=mname $
,date=date, mod_date=mod_date $
,spec_comments=scom, norm_comments=ncom $
,ndscheader=ndscheader
nprof=n_elements(xvals) & ncolumn=n_elements(vname)
; Reform Ames data into a 2D array (instead of 1D as output by ex_rd2110)
nystart=0*nypts & nyend=nystart & nystart(0)=0 & nyend(0)=nypts(0)-1 & nyptsmax=max(nypts)
dat2d=fltarr(nprof,nyptsmax,ncolumn) & dat2d(0,0:nypts(0)-1,*)=dat(nystart(0):nyend(0),*) & yvals2d=fltarr(nprof,nyptsmax) & yvals2d(0,0:nypts(0)-1)=yvals(nystart(0):nyend(0))
for iprof=1,nprof-1 do begin
nystart(iprof)=nyend(iprof-1)+1
nyend(iprof)=nystart(iprof)+nypts(iprof)-1
dat2d(iprof,0:nypts(iprof)-1,*)=dat(nystart(iprof):nyend(iprof),*)
if nypts(iprof) lt nyptsmax then for icolumn=0,ncolumn-1 do dat2d(iprof,nypts(iprof):nyptsmax-1,icolumn)=vmiss(icolumn) ;For shorter profiles: filling upper points with missing values (poimary variable)
yvals2d(iprof,0:nypts(iprof)-1)=yvals(nystart(iprof):nyend(iprof))
if nypts(iprof) lt nyptsmax then yvals2d(iprof,nypts(iprof):nyptsmax-1)=vmiss(0) ;For shorter profiles: filling upper points with missing values (independent variable)
endfor
; Read AVDC-compliant Meta data
Read_Instr_Metadata,pi_metafile ,ga,ivar_data ; Global Attributes (except "data_variables")
Read_Species_Metadata,species_metafile,ga,ivar_data ,va ; Variable Attributes + global attribute "data_variables"
nvar=n_elements(va(0,*))
; Determine the variable names
var_name=indgen(nvar,/string)
for n=0,nvar-1 do begin
for m=0,n_elements(va[*,0])-1 do begin
str_tmp=strtrim(strsplit(va[m,n],'=',/extract),2)
if (str_tmp[0] eq 'VAR_NAME') then var_name[n]=strtrim(str_tmp[1],2)
endfor
endfor
; Determine the new fill values:
fill=strarr(nvar) ;reform(va[10,*])
for n=0,nvar-1 do begin
str_tmp=strsplit(va[10, n],' =',/extract,count=nstr_tmp)
if nstr_tmp ge 2 then fill[n]=str_tmp[1]
endfor
; Output data for each profile:
s=0
for i=0,nprof-1 do begin
print,''
print,'------------------------------------------------------'
print,''
print,'Converting profile # '+strtrim(i+1,2)+' / '+strtrim(nprof,2)
xames=xvals(i)
yames=reform(yvals2d(i,0:nypts(i)-1))
vames=reform(dat2d(i,0:nypts(i)-1,*))
if nypts(i) eq 1 then vames=reform(vames,nypts(i),ncolumn) ;only one datapoint in the profile so make 2-D
auxames=reform(auxdat(i,*))
data_desc=strlowcase(vname) ;holding variable for NDACC variable names
fill=double(fill) ;convert from string representation to double
;*************************************************************************************************************************
;*************************************************************************************************************************
;*********** INSTRUMENT/PI-DEPENDENT CALL BELOW ******************************************************
instpi=strlowcase(pi+species+mcgee) ;mcgee eq '' if not a mcgee file o/w eq 'tg' or 't2'
AmesToPointer,instpi,site,var_name,fill,xames,yames,vames,vmiss,auxames,ncom,tmp,data_desc,data_cav,alt_inst
;if pi eq 'md' then AmesToPointer_md,species,site ,var_name,fill ,xames,yames,vames,vmiss,auxames ,tmp,data_desc,data_cav
;if pi eq 'sw' then AmesToPointer_sw,species,site ,var_name,fill ,xames,yames,vames,vmiss,auxames ,tmp,data_desc,data_cav
;if pi eq 'sb' then AmesToPointer_sb,species,site ,var_name,fill ,xames,yames,vames,vmiss,auxames ,tmp,data_desc,data_cav
;if pi eq 'hg' then AmesToPointer_hg,species,site ,var_name,fill ,xames,yames,vames,vmiss,auxames ,tmp,data_desc,data_cav
;if pi eq 'go' then AmesToPointer_go,species,site ,var_name,fill ,xames,yames,vames,vmiss,auxames ,tmp,data_desc,data_cav
;if pi eq 'na' then AmesToPointer_na,species,site ,var_name,fill ,xames,yames,vames,vmiss,auxames ,tmp,data_desc,data_cav
;if pi eq 'ha' then AmesToPointer_ha,species,site ,var_name,fill ,xames,yames,vames,vmiss,auxames ,tmp,data_desc,data_cav
;if pi eq 'an' then AmesToPointer_an,species,site ,var_name,fill ,xames,yames,vames,vmiss,auxames ,tmp,data_desc,data_cav
;if pi eq 'mg' and mcgee eq 'tg' then AmesToPointer_mg,species,site ,var_name,fill ,xames,yames,vames,vmiss,auxames ,tmp,data_desc,data_cav
;if pi eq 'mg' and mcgee eq 't2' then AmesToPointer_mg,species,site ,var_name,fill ,xames,yames,vames,vmiss,auxames ,tmp,data_desc,data_cav ,mcgee='t2'
;*********** INSTRUMENT/PI-DEPENDENT CALL ABOVE ******************************************************
;*************************************************************************************************************************
;*************************************************************************************************************************
; Assign DATA_MODIFICATIONS, DATA_FILE_VERSION, DATA_DESCRIPTION, and DATA_CAVEATS
; For Reunion check for St. Denis or Maido site and change DATA_LOCATION accordingly
data_qual=['FINAL','PRELIMINARY','OTHER']
versionndsc=strmid(ndscheader,strlen(ndscheader)-2,2) & if pi eq 'sb' then versionndsc='50'
if (pi eq 'go') and (site eq 'du') and (species eq 'o3') then nodatamod=1 else nodatamod=0 ;ndscheader does not include the data quality code
if site eq 're' then begin
if alt_inst lt 1000. then addloc='LA.REUNION.STDENIS' else addloc='LA.REUNION.MAIDO'
endif else addloc=''
if strmid(versionndsc,0,1) eq '0' then versionhdf=strmid(versionndsc,1,1) & if strmid(versionndsc,0,1) ne '0' then versionhdf=strmid(versionndsc,0,1)+'.'+strmid(versionndsc,1,1)
FOR n = 0, N_ELEMENTS(ga[*])-1 DO BEGIN
str_tmp = STRTRIM(STRSPLIT(ga[n], '=', /EXTRACT), 2)
IF ( str_tmp[0] EQ 'DATA_MODIFICATIONS' ) and (nodatamod eq 0) THEN $
ga[n] = STRTRIM(str_tmp[0], 2) + ' = ' + data_qual(strmid(ndscheader,strlen(ndscheader)-4,1)) + ' (from NDSC data status flag)
; IF ( str_tmp[0] EQ 'DATA_FILE_VERSION' ) THEN ga[n] = STRTRIM(str_tmp[0], 2) + ' = v' + versionhdf
; IF ( str_tmp[0] EQ 'DATA_FILE_VERSION' ) THEN ga[n] = STRTRIM(str_tmp[0], 2) + ' = 001'
IF ( str_tmp[0] EQ 'DATA_CAVEATS' ) THEN ga[n] = STRTRIM(str_tmp[0], 2) + ' = ' + data_cav
IF ( str_tmp[0] EQ 'DATA_LOCATION' ) and (addloc NE '') THEN ga[n] = STRTRIM(str_tmp[0], 2) + ' = ' + addloc
ENDFOR
; Assign the size of each variable
FOR m = 0, nvar-2 DO BEGIN ;Handle SOURCE.PRODUCT separately
FOR n = 0, N_ELEMENTS(va[*, 0])-1 DO BEGIN
str_tmp = STRTRIM(STRSPLIT(va[n, m], '=', /EXTRACT), 2)
IF ( str_tmp[0] EQ 'VAR_NAME' ) THEN var_name[m] = STRTRIM(str_tmp[1], 2)
IF ( str_tmp[0] EQ 'VAR_SIZE' ) THEN va[n, m] = STRTRIM(str_tmp[0], 2) + ' = ' + $
STRING(N_ELEMENTS(*tmp[m].data), FORMAT='(I5)')
ENDFOR
ENDFOR
; Define and allocate values to pointer
;determine the maximum number of attributes
n_atts=N_ELEMENTS(va[*,0])
sds_set = {va_v:PTR_NEW(), va_l:PTR_NEW(), data:PTR_NEW()}
sds=REPLICATE(sds_set, nvar, n_atts)
FOR n = 0, nvar-2 DO BEGIN ;Handle SOURCE.PRODUCT separately
wv = WHERE ( var_name EQ *tmp[n].name )
IF ( wv[0] NE -1 ) THEN BEGIN
FOR m = 0, n_atts-1 DO BEGIN
res=STRSPLIT(va[m,n],' =',/Extract, Count=rcnt)
sds[n,m].va_l = PTR_NEW(res[0])
IF rcnt EQ 1 THEN res=[res[0],' '] $
ELSE IF rcnt NE 2 THEN res=[res[0],STRTRIM(STRMID(va[m,n],STRPOS(va[m,n],'=')+1),2)]
sds[n,m].va_v = PTR_NEW(res[1])
ENDFOR
sds[n,0].data = PTR_NEW(*tmp[n].data)
ENDIF
ENDFOR
;Do SOURCE.PRODUCT
;Change mod_date format from YYMMDD to YYYYMMDD
if fix(strmid(mod_date,0,2)) lt 50 then mod_date2='20'+strtrim(mod_date,2) $
else mod_date2='19'+strtrim(mod_date,2)
FOR m = 0, n_atts-1 DO BEGIN
res=STRSPLIT(va[m,nvar-1],' =',/Extract, Count=rcnt)
sds[nvar-1,m].va_l = PTR_NEW(res[0])
IF rcnt EQ 1 THEN res=[res[0],' '] $
ELSE IF rcnt NE 2 THEN res=[res[0],STRTRIM(STRMID(va[m,nvar-1],STRPOS(va[m,nvar-1],'=')+1),2)]
sds[nvar-1,m].va_v = PTR_NEW(res[1])
ENDFOR
datasize='NDACC_'+FILE_BASENAME(amesfile)+'_'+mod_date2
sds[nvar-1,0].data=PTR_NEW(datasize)
; Write to AVDC HDF file
w = WHERE ( *sds[5,0].data NE fill[5] ) ; check that O3 data is available in profile
;IF ( w[0] NE -1 ) THEN IDLcr8HDF, ga, sds, '~/Work/CR8/tableattrvalue_idl.dat' ,hdfdir=hdfdir,meta_dir=meta_dir
;IF ( w[0] NE -1 ) THEN IDLcr8HDF, ga, sds, '~/Work/CR8/tableattrvalue_idl.dat'
rerr = ''
IF ( w[0] NE -1 ) THEN begin
if (keyword_set(h5)) then begin
IDLcr8HDF,ga,sds,tav,hdf_dir,rerr,/LOG,/h5
end else if (keyword_set(nc)) then begin
IDLcr8HDF,ga,sds,tav,hdf_dir,rerr,/LOG,/nc
end else begin
IDLcr8HDF,ga,sds,tav,hdf_dir,rerr,/LOG
end
end
ENDFOR
HEAP_GC
;STOP
END
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment