Commit 603c31f8 authored by Ian Boyd's avatar Ian Boyd
Browse files

Replace idlcr8hdf.pro

parent 43ce5656
;Main Program Version: idlcr8hdf.pro v4.0b48, 20180901
;Main Program Version: idlcr8hdf.pro v4.0b50, 20190506
; Written by Ian Boyd for the AVDC - iboyd@astro.umass.edu
;
;Sub-versions (refer to idlcr8hdf-v4.0_Readme.pdf for full history)
......@@ -33,7 +33,7 @@ PRO idlcr8hdf_common
COMMON TABLEDATA, tab_arr,tab_ver,tab_type
COMMON METADATA, meta_arr,attr_arr_glob,attr_arr_data,attr_free,mv_lng,mv_dbl,mv_str
COMMON DATA, ds,ndm,nvn,vn,vu,vnchange,vserror,qa_yes
COMMON WIDGET_WIN, wtxt,b1,lineno,base,o3,dux,rerr
COMMON WIDGET_WIN, wtxt,b1,lineno,base,o3,dux,rerr,lvals
END ;Procedure idlcr8hdf_common
......@@ -66,9 +66,15 @@ ELSE IF uv EQ 'idlcr8hdf.log' THEN IF o3[2] EQ uv THEN o3[2]='0' ELSE o3[2]=uv $
;Assign/Remove Pop-up window for Log Output button event to/from a variable name
ELSE IF uv EQ 'Pop' THEN IF o3[3] EQ uv THEN o3[3]='0' ELSE o3[3]=uv $
;Assign button event to a variable name
ELSE IF (uv EQ 'H4') OR (uv EQ 'H5') OR (uv EQ 'NC') THEN o3[0]=uv $
ELSE IF (uv EQ 'H4') OR (uv EQ 'H5') OR (uv EQ 'NC') THEN BEGIN
o3[0]=uv
IF uv EQ 'H5' THEN BEGIN
WIDGET_CONTROL,ev.id+1,Sensitive=1
WIDGET_CONTROL,ev.id+2,Sensitive=1,Set_Combobox_Select=0
ENDIF
ENDIF ELSE IF uv EQ '0' THEN IF ev.str EQ 'None' THEN o3[0]='H5' ELSE o3[0]='H5_'+STRTRIM(ev.str,2) $
ELSE o3[0]='0' ;Cancel button chosen
IF (uv NE 'AVK') AND (uv NE 'idlcr8hdf.log') AND (uv NE 'Pop') THEN WIDGET_CONTROL,ev.top,/DESTROY
IF (uv NE 'AVK') AND (uv NE 'idlcr8hdf.log') AND (uv NE 'Pop') AND (uv NE 'H5') THEN WIDGET_CONTROL,ev.top,/DESTROY
END ;Intro_Event
......@@ -104,8 +110,10 @@ PRO intro, intype
;
;Subroutines Called: INTRO_EVENT (via XMANAGER)
nhdr=43 & errtxt=STRARR(nhdr)
vertxt=['idlcr8hdf-v4.0_Readme.pdf','v4.0b48 September 2018']
COMMON WIDGET_WIN
nhdr=44 & errtxt=STRARR(nhdr)
vertxt=['idlcr8hdf-v4.0_Readme.pdf','v4.0b49 December 2018']
errtxt[1]='Welcome to IDLcr8HDF. This program creates GEOMS compliant HDF4, HDF5 and netCDF files'
errtxt[2]='(also refer to '+vertxt[0]+').'
errtxt[4]='Inputs to the program (IDL Virtual Machine (VM) and IDL Licensed (LIC) Versions):'
......@@ -126,28 +134,29 @@ errtxt[19]=' RETERR - String variable to which any error(s) are written.'
errtxt[21]='For IDL VM, input is by ''DIALOG_BOXES''. For IDL LIC, input can be by ''DIALOG_BOXES'' or passed'
errtxt[22]='by calling the program with one of the following command line options:'
errtxt[23]=' 1. idlcr8hdf (Opens this box, and allows the user the option to continue with file inputs).'
errtxt[24]=' 2. idlcr8hdf,METAFILE,DATAFILE(s),TAVFILE,OUTDIR[,RETERR][,/H5][,/NC][,/AVK][,/Log][,/Popup] or'
errtxt[25]=' idlcr8hdf,'''','''','''',''''[,/H5][,/NC][,/AVK][,/Log][,/Popup] (For null string, DIALOG_BOXES will prompt for input).'
errtxt[26]=' 3. idlcr8hdf,GA,DS,TAVFILE,OUTDIR[,RETERR][,/H5][,/NC][,/AVK][,/Log][,/Popup] or
errtxt[27]=' idlcr8hdf,GA,DS[,/H5][,/NC][,/AVK][,/Log][,/Popup] (Inputs are from session memory, DIALOG_BOX(s) will'
errtxt[24]=' 2. idlcr8hdf,METAFILE,DATAFILE(s),TAVFILE,OUTDIR[,RETERR][,/H5][,/Cn][,/NC][,/AVK][,/Log][,/Popup] or'
errtxt[25]=' idlcr8hdf,'''','''','''',''''[,/H5][,/Cn][,/NC][,/AVK][,/Log][,/Popup] (For null string, DIALOG_BOXES will prompt for input).'
errtxt[26]=' 3. idlcr8hdf,GA,DS,TAVFILE,OUTDIR[,RETERR][,/H5][,/Cn][,/NC][,/AVK][,/Log][,/Popup] or
errtxt[27]=' idlcr8hdf,GA,DS[,/H5][,/Cn][,/NC][,/AVK][,/Log][,/Popup] (Inputs are from session memory, DIALOG_BOX(s) will'
errtxt[28]=' prompt for input if TAVFILE or OUTDIR are not included).'
errtxt[29]=' /H5, /NC, /AVK, /Log and /Popup keywords are used in place of the options given below (HDF4 is default).'
errtxt[31]='Contacts -'
errtxt[32]=' Ian Boyd (iboyd@astro.umass.edu)'
errtxt[33]=' Department of Astronomy, 619 Lederle GRC, University of Massachusetts'
errtxt[34]=' 710 North Pleasant St, Amherst, MA 01002, USA'
errtxt[36]=' Ghassan Taha, AVDC Project Manager (ghassan.taha@nasa.gov)'
errtxt[37]=' NASA Goddard Space Flight Center, Code 613.3'
errtxt[38]=' Greenbelt, MD 20771, USA'
errtxt[40]='AVDC Website: Tools and documentation available from http://avdc.gsfc.nasa.gov/Overview/index.html'
errtxt[42]='To continue, please choose from the options below (Note: HDF5 only available on IDL6.2 or greater).'
errtxt[29]=' /H5, /Cn, /NC, /AVK, /Log and /Popup keywords are used in place of the options given below (HDF4 is default).'
errtxt[30]='The /Cn keyword enables compression and shuffling of HDF5 files only (C1=low, C9=high). Default is no compression.'
errtxt[32]='Contacts -'
errtxt[33]=' Ian Boyd (iboyd@astro.umass.edu)'
errtxt[34]=' BC Scientific Consulting'
errtxt[35]=' 6 University Drive, Ste 206-135, Amherst, MA 01002, USA'
errtxt[37]=' Ann Mari Fjaeraa, EVDC Project Manager (amf@nilu.no)'
errtxt[38]=' Norwegian Institute for Air Research, Instituttveien 18'
errtxt[39]=' Postbox 100, N-2027 KJELLER, NORWAY'
errtxt[41]='EVDC Website: Tools and documentation available from http://evdc.esa.int/'
errtxt[43]='To continue, please choose from the options below (Note: HDF5 only available on IDL6.2 or greater).'
errtxt=' '+errtxt
;Set-up text display widget
IF intype EQ -2 THEN xtxt=' - Command Line Input Error' ELSE xtxt=''
IF intype EQ -3 THEN optsens=0 ELSE optsens=1
base=WIDGET_BASE(Title='idlcr8hdf '+vertxt[1]+xtxt,Tlb_Frame_Attr=1,/Column) ;,Tab_Mode=1)
wtxt=WIDGET_TEXT(base,xsize=90,ysize=25,/Scroll)
wtxt=WIDGET_TEXT(base,xsize=102,ysize=25,/Scroll)
base3=WIDGET_BASE(base,/Nonexclusive)
logtext='Append log output to the file ''idlcr8hdf.log'' '
IF intype EQ -3 THEN logtext=logtext+'(No log or netCDF file output permitted in IDL DEMO Mode)' $
......@@ -162,10 +171,13 @@ b6=WIDGET_BUTTON(base3,value=avktext,uvalue='AVK',frame=3) ;,Tooltip=AVKTip)
base2=WIDGET_BASE(base,/Row)
tip='Left Mouse Click or Tab to entry and hit <Spacebar>'
b1=WIDGET_BUTTON(base2,value='HDF4',uvalue='H4',frame=3) ;,Tooltip=Tip)
b7=WIDGET_BUTTON(base2,value='netCDF3',uvalue='NC',frame=3,SENSITIVE=optsens)
IF FLOAT(!Version.Release) GE 6.2 THEN $
b2=WIDGET_BUTTON(base2,value='HDF5',uvalue='H5',frame=3) $;,Tooltip=Tip) $
ELSE b2=WIDGET_BUTTON(base2,value='HDF5',Sensitive=0,frame=3)
b7=WIDGET_BUTTON(base2,value='netCDF',uvalue='NC',frame=3,SENSITIVE=optsens)
b8=Widget_Label(base2,Value=' Compression ',Sensitive=0)
b9=Widget_Combobox(base2,Value=lvals,Sensitive=0,uvalue='0')
b3=WIDGET_BUTTON(base2,value='Stop',uvalue='CANCEL',frame=3) ;,ToolTip=Tip)
WIDGET_CONTROL,base,/Realize
WIDGET_CONTROL,b4,/Input_Focus
......@@ -200,7 +212,7 @@ END ;IDLcr8HDF_Event
FUNCTION is_a_number_hdf,value
FUNCTION is_a_number_hdf, value
ON_IOERROR, ConversionError
IF STRTRIM(value,2) EQ '' THEN RETURN, 0B
n=DOUBLE(value)
......@@ -1215,6 +1227,9 @@ PRO geoms_rule_changes, code, in1, in2, in3, in4
; Fix issue with DATA_TEMPLATE identification (rule 9) - Version 4.0b37
; 20171121: Fix bug that selected the incorrect DATA_TEMPLATE in some situations (rule 9)
; - Version 4.0b43
; 20190506: Add FILE_META_VERSION information to INFORMATION/ERROR comment when
; DATA_TEMPLATE value is not as expected; Fix bug that caused a crash if the
; file does not use a DATA_TEMPLATE (rule 9) - Version 4.0b50
;
; Inputs: code - Integer value identifying type of check to carry out
; in1 - First set of inputs required for checks (optional, dependent on code value)
......@@ -1740,6 +1755,7 @@ CASE 1 OF
;2nd Check - More than one option so try and match 2nd part of DATA_SOURCE with 4th part in TAV entry
tab_arr_sub=tab_arr_sub[ci]
tab_arr_part_sub=STRARR(ccnt)
numi=-1
FOR i=0,ccnt-1 DO BEGIN
rest=STRSPLIT(tab_arr_sub[i],'-',/EXTRACT)
rest=[rest,'','',''] ;ensure there are at least 4 sub-values
......@@ -1747,11 +1763,11 @@ CASE 1 OF
IF IS_A_NUMBER_HDF(rest[3]) THEN numi=i
ENDFOR
ci=WHERE(STRUPCASE(tab_arr_part_sub) EQ res1[1],ccnt)
IF ccnt EQ 0 THEN BEGIN
IF (ccnt EQ 0) AND (numi NE -1) THEN BEGIN
;2nd part is likely a gas e.g. FTIR.CO, so use template where 4th part of TAV entry is a number
geoms_te=tab_arr_sub[numi]
ENDIF ELSE IF ccnt EQ 1 THEN geoms_te=tab_arr_sub[ci[0]] $ ;unique template found
ELSE BEGIN ;ccnt GT 1, Try alternatives for H2O and O3, or test for fourth part
ELSE IF ccnt GT 1 THEN BEGIN ;ccnt GT 1, Try alternatives for H2O and O3, or test for fourth part
tasx=tab_arr_sub[ci]
CASE 1 OF
(res1[0] EQ 'UVVIS') AND (res1[1] EQ 'DOAS'): BEGIN
......@@ -1766,13 +1782,22 @@ CASE 1 OF
ENDCASE
ci=WHERE(STRPOS(STRUPCASE(tasx),res1x) NE -1,ccnt)
IF ccnt EQ 1 THEN geoms_te=tasx[ci[0]]
ENDELSE
ENDIF
ENDIF
IF geoms_te NE '' THEN BEGIN ;DATA_TEMPLATE value found based on DATA_SOURCE
IF gcnt EQ 1 THEN BEGIN ;DATA_TEMPLATE label is present
fmvi=WHERE(in1 EQ 'FILE_META_VERSION',fmvcnt)
fmvtxt=''
IF fmvcnt EQ 1 THEN BEGIN
resfmv=STRSPLIT(in2[fmvi[0]],' ;',/EXTRACT,COUNT=fmvcnt)
IF fmvcnt GE 1 THEN BEGIN
fmv_ver=resfmv[0]
fmvtxt=' and '+fmv_ver+' metadata definitions'
ENDIF
ENDIF
in2[gi[0]]=geoms_te ;Add value to DATA_TEMPLATE label
IF qa_yes THEN itxt=' expected to be ' ELSE itxt=' renamed '
infotxt='2 DATA_TEMPLATE value'+itxt+geoms_te+' based on DATA_SOURCE value'
infotxt='2 DATA_TEMPLATE value'+itxt+geoms_te+' based on DATA_SOURCE value'+fmvtxt
INFOTXT_OUTPUT,infotxt
std_txt=''
ENDIF ELSE IF gcnt EQ 0 THEN in3[0]=geoms_te ;Need to add new Global Attribute Information
......@@ -3920,7 +3945,7 @@ mxdatalen=0L ;default value of string length
testdatalen=MAX(STRLEN(dtest))
IF testdatalen GT mxdatalen THEN mxdatalen=testdatalen
;Format the first data value to the maximum string length for H5 only (white space removed when writing to the file)
IF (mxdatalen NE 0L) AND (o3[0] EQ 'H5') THEN BEGIN
IF (mxdatalen NE 0L) AND (STRPOS(o3[0],'H5') NE -1) THEN BEGIN
;Done for H5 only because H5S_CREATE_SIMPLE function uses string length of the 1st entry to set length for all entries
vfsx='(A-'+STRTRIM(mxdatalen,2)+')' ;A- does left justified text instead of right justified
dtest[0]=STRING(format=vfsx,dtest[0])
......@@ -5112,7 +5137,7 @@ END ;Procedure MakeAN
PRO avdc_hdf5_write, hdffilename, natts, av
PRO avdc_hdf5_write, hdffilename, natts, av, geoms_output
;IDL subroutine to write AVDC-type global attributes and datasets to the Root
;Group in an HDF5 file.
;##########################################################################
......@@ -5133,6 +5158,8 @@ PRO avdc_hdf5_write, hdffilename, natts, av
; string datasets do a STRTRIM of the dataset when writing to the file, as
; the first dataset entry may have had whitespace added so that the
; maximum dataset length is correctly determined - Version 4.0b26
; 20181221: Allow compression and shuffle options when creating HDF5 files
; - Version 4.0b49
;
; Inputs: hdffilename - a string holding the name of the HDF file being created
; and written to
......@@ -5155,6 +5182,13 @@ PRO avdc_hdf5_write, hdffilename, natts, av
COMMON METADATA
COMMON DATA
;Check for compression setting
comp_fac=0
IF STRLEN(geoms_output) EQ 4 THEN BEGIN
val_chk=STRMID(geoms_output,3,1)
IF IS_A_NUMBER_HDF(val_chk) THEN comp_fac=FIX(val_chk)
ENDIF
;Create an HDF5 file
hdf_file_id=H5F_CREATE(hdffilename)
H5F_CLOSE, hdf_file_id
......@@ -5230,7 +5264,11 @@ FOR i=0,nvn-1 DO BEGIN
dtype_id=H5T_IDL_CREATE(data)
dspace_id=H5S_CREATE_SIMPLE(vs)
;create dataset in the output file
dset_id=H5D_CREATE(sd_id,vn[i],dtype_id,dspace_id)
IF comp_fac EQ 0 THEN dset_id=H5D_CREATE(sd_id,vn[i],dtype_id,dspace_id) $
ELSE BEGIN
;Perform compression and shuffling of the dataset
dset_id=H5D_CREATE(sd_id,vn[i],dtype_id,dspace_id,CHUNK_DIMENSIONS=vs,GZIP=comp_fac,/SHUFFLE)
ENDELSE
;write data to dataset - for string datasets remove unwanted white space
IF ndm[i,8] EQ 6 THEN H5D_WRITE,dset_id,STRTRIM(data,2) ELSE H5D_WRITE,dset_id,data
......@@ -5479,7 +5517,7 @@ END ;Procedure AVDC_NC_Write
PRO avdc_hdf_write, hdffilename
PRO avdc_hdf_write, hdffilename, geoms_output
;IDL subroutine to write AVDC-type global attributes and datasets to a Scientific
;Dataset (SDS) in HDF4 (Bojkov et al.,2004) or HDF5. This routine creates the HDF4
;file if requested, and calls the AVDC_HDF5_WRITE routine to create the HDF5 file if
......@@ -5716,20 +5754,21 @@ IF ftype EQ 'hdf' THEN BEGIN
HDF_SD_END,sd_id
;The HDF_CLOSE procedure closes the HDF4 file associated with the given file handle.
HDF_CLOSE,hdf_file_id
ENDIF ELSE IF ftype EQ 'h5' THEN AVDC_HDF5_WRITE,hdffilename,natts,av $
ENDIF ELSE IF ftype EQ 'h5' THEN AVDC_HDF5_WRITE,hdffilename,natts,av,geoms_output $
ELSE AVDC_NC_WRITE,hdffilename,natts,av,var_depend,var_size
END ;Procedure AVDC_HDF_Write
PRO idlcr8hdf, ga, sds, tav, odir, reterr, H5=o1, AVK=o2, LOG=o4, POPUP=o5, QA=o6, NOHDF=o7, NC=o8, DATETIME=o9
PRO idlcr8hdf, ga, sds, tav, odir, reterr, H5=o1, AVK=o2, LOG=o4, POPUP=o5, QA=o6, NOHDF=o7, NC=o8, DATETIME=o9, $
C1=o11, C2=o12, C3=o13, C4=o14, C5=o15, C6=o16, C7=o17, C8=o18, C9=o19
;Main IDL program to create HDF4 or HDF5 format files for submission to the AVDC, NDACC, or
;NILU (ESA Envisat) databases.
;
;Program documentation, idlcr8hdf-v4.0_Readme.pdf, available from http://avdc.gsfc.nasa.gov.
;
;Program sub-version 4.0b48 (20180901)
;Program sub-version 4.0b50 (20190506)
; ----------
;Written by Ian Boyd for the AVDC - iboyd@astro.umass.edu
;
......@@ -5779,6 +5818,7 @@ PRO idlcr8hdf, ga, sds, tav, odir, reterr, H5=o1, AVK=o2, LOG=o4, POPUP=o5, QA=o
; inputs are in the form of a structure - Version 4.0b40
; 20180218: Ensure variable attribute value (vav) is only a single variable when calling
; PRE_DEFINED_ATT_CHECKS - Version 4.0b45
; 20181221: Allow compression and shuffle options when creating HDF5 files - Version 4.0b49
;
; Inputs: DIALOG PROMPTS WITH IDL VIRTUAL MACHINE (File input only)
; Metadata template file - The program will fill in missing spaces based on data file and
......@@ -5801,7 +5841,7 @@ PRO idlcr8hdf, ga, sds, tav, odir, reterr, H5=o1, AVK=o2, LOG=o4, POPUP=o5, QA=o
;
; OPTIONS
; H5 - output will be as an HDF5 file instead of the standard HDF4
; NC - output will be as a netCDF file instead of the standard HDF4
; NC - output will be as a netCDF3 file instead of the standard HDF4
; AVK - to add a sentence to VAR_NOTES indicating array order for Averaging Kernel
; datasets
; Log - to append input/output information as well as warnings and errors to a log file
......@@ -5813,6 +5853,7 @@ PRO idlcr8hdf, ga, sds, tav, odir, reterr, H5=o1, AVK=o2, LOG=o4, POPUP=o5, QA=o
; idlcr8qa.log instead of idlcr8hdf.log
; NoHDF - an HDF file will not be created, but the logfile idlcr8hdf.log will automatically
; be generated
; Cn - to enable compression and shuffling of HDF5 files only (ignored for netCDF3 and HDF4)
;
; Output: An HDF file formatted to GEOMS standards. Keyword options can also result
; in an output log file (idlcr8hdf.log), or a pop-up box showing log output. If the
......@@ -5878,14 +5919,29 @@ rerr=['NA','NA'] ;initialize return error string
IF intype LT 0 THEN BEGIN ;either no input parameters or invalid second parameter
o3=['0','0','0','0','0','0']
lvals=['None','1','2','3','4','5','6','7','8','9'] ;compression level for HDF5 option
INTRO,intype ;Open Intro Box and determine HDF output format (HDF4 or HDF5)
IF o3[0] EQ '0' THEN BEGIN
STOP_WITH_ERROR,'','',lu & RETURN
ENDIF
IF o3[3] EQ 'Pop' THEN o3[3]='' ELSE o3[3]='D_'
ENDIF ELSE BEGIN ;Set options (HDF4, HDF5, netCDF, AVK, LOGFILE, POP-UP, QA, NOHDF)
IF KEYWORD_SET(o1) THEN o3=['H5','0','0','D_','0','0'] $
ELSE IF KEYWORD_SET(o8) THEN o3=['NC','0','0','D_','0','0'] $
IF KEYWORD_SET(o1) THEN BEGIN
o3=['H5','0','0','D_','0','0']
CASE 1 OF
KEYWORD_SET(o11): o3[0]='H5_1'
KEYWORD_SET(o12): o3[0]='H5_2'
KEYWORD_SET(o13): o3[0]='H5_3'
KEYWORD_SET(o14): o3[0]='H5_4'
KEYWORD_SET(o15): o3[0]='H5_5'
KEYWORD_SET(o16): o3[0]='H5_6'
KEYWORD_SET(o17): o3[0]='H5_7'
KEYWORD_SET(o18): o3[0]='H5_8'
KEYWORD_SET(o19): o3[0]='H5_9'
ELSE:
ENDCASE
;PRINT,o3
ENDIF ELSE IF KEYWORD_SET(o8) THEN o3=['NC','0','0','D_','0','0'] $
ELSE o3=['H4','0','0','D_','0','0']
IF KEYWORD_SET(o2) THEN o3[1]='AVK' ;AVK option
IF KEYWORD_SET(o4) THEN o3[2]='idlcr8hdf.log' ;Log option
......@@ -6282,7 +6338,7 @@ FOR ndf=0,nfile-1 DO BEGIN
reterr=rerr[0] & RETURN
ENDIF
;Call the Procedure to construct the HDF output filename based on input metadata.
IF o3[0] EQ 'H5' THEN hdffilename='.h5' $
IF STRPOS(o3[0],'H5') NE -1 THEN hdffilename='.h5' $
ELSE IF o3[0] EQ 'NC' THEN hdffilename='.nc' $
ELSE hdffilename='.hdf'
FIND_HDF_FILENAME,hdffilename
......@@ -6297,7 +6353,8 @@ FOR ndf=0,nfile-1 DO BEGIN
;Call the procedure to write the Global Attributes, Variable Attributes and Data
;to an HDF4 or HDF5 file
AVDC_HDF_WRITE,hdffilename
geoms_output=o3[0]
AVDC_HDF_WRITE,hdffilename,geoms_output
FILE_MOVE, hdffilename, outdir+hdffilename,/Allow_Same,/Overwrite
hdffilename=outdir+hdffilename
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment