Commit 78e4034c authored by Ian Boyd's avatar Ian Boyd

Replace idlcr8hdf.pro

parent 59484541
;Main Program Version: idlcr8hdf.pro v4.0b52, 20190824
;Main Program Version: idlcr8hdf.pro v4.0b54, 20201103
; Written by Ian Boyd for the AVDC - iboyd@astro.umass.edu
;
;Sub-versions (refer to idlcr8hdf-v4.0_Readme.pdf for full history)
......@@ -113,7 +113,7 @@ PRO intro, intype
COMMON WIDGET_WIN
nhdr=44 & errtxt=STRARR(nhdr)
vertxt=['idlcr8hdf-v4.0_Readme.pdf','v4.0b52 August 2019']
vertxt=['idlcr8hdf-v4.0_Readme.pdf','v4.0b54 March 2020']
errtxt[1]='Welcome to IDLcr8HDF. This program creates GEOMS compliant HDF4, HDF5 and netCDF files'
errtxt[2]='(also refer to '+vertxt[0]+').'
errtxt[4]='Inputs to the program (IDL Virtual Machine (VM) and IDL Licensed (LIC) Versions):'
......@@ -1230,6 +1230,8 @@ PRO geoms_rule_changes, code, in1, in2, in3, in4
; 20190506: Add FILE_META_VERSION information to INFORMATION/ERROR comment when
; DATA_TEMPLATE value is not as expected; Fix bug that caused a crash if the
; file does not use a DATA_TEMPLATE (rule 9) - Version 4.0b50
; 20191205: Add rule 11 which does checks on optional VERSION_NAME sub-field of
; DATA_SOURCE - Version 4.0b53
;
; Inputs: code - Integer value identifying type of check to carry out
; in1 - First set of inputs required for checks (optional, dependent on code value)
......@@ -1284,6 +1286,8 @@ PRO geoms_rule_changes, code, in1, in2, in3, in4
; Inputs: in1=m_v0, in2=m_v1, in3=ga_chk
; Code 10: Do checks on ISO646-US ASCII character set
; Inputs: in1=meta_arr or dtest, in2=vn[vc] (Variable Name for datasets only)
; Code 11: Do checks on DATA_VERSION_NAME part of DATA_SOURCE value
; Inputs: in1=DATA_VERSION_NAME, in2=meta_arr
COMMON TABLEDATA
COMMON METADATA
......@@ -1903,7 +1907,29 @@ CASE 1 OF
ENDIF
ENDFOR
END
code EQ 11: BEGIN ;Check DATA_VERSION_NAME part of the DATA_SOURCE
;Check that entry is made up of dot-separated alpha_numeric words
n_char=STRLEN(in1)
strok=1B ;default is that entry is OK
test2=0B & test3=0B ;will change to 1 if test conditions are true
charchkm1=0B ;to check test character against the previous character
FOR i=0,n_char-1 DO BEGIN
charchk=BYTE(STRMID(in1,i,1))
test1=(charchk EQ 46B) OR ((charchk GE 65B) AND (charchk LE 90B)) OR $ ;'.' or A-Z
((charchk GE 97B) AND (charchk LE 122B)) OR $ ;a-z
((charchk GE 48B) AND (charchk LE 57B)) ;0-9
IF (i EQ 0) OR (i EQ n_char-1) THEN test2=charchk EQ 46B ;i.e. first or last character is a '.'
test3=(charchk EQ 46B) AND (charchkm1 EQ 46B) ;i.e. two consecutive dots
IF (~test1) OR (test2) OR (test3) THEN strok=0B
charchkm1=charchk
ENDFOR
IF ~strok THEN BEGIN
;write error message
infotxt='3 VERSION_NAME sub-value of the DATA_SOURCE must consist of dot-separated alpha-numeric words: '
infotxt=infotxt+in1
INFOTXT_OUTPUT,infotxt
ENDIF
END
ENDCASE
END ;Procedure GEOMS_Rule_Changes
......@@ -2090,6 +2116,12 @@ PRO read_metadata, metafile, inf
; 20151109: Add log message when adding/checking for missing mandatory variable attributes - Version 4.0b34
; 20151116: Fix bug when doing whitespace checks and there is no metadata entry - Version 4.0b35
; 20180528: Fix bug when adding non-standard global attributes to meta_arr - Version 4.0b47
; 20191205: Do checks for sub-value separators (. or ; or _) being at the start or end of attribute
; values, and for consecutive sub-value separators (e.g. ;;). This can occur as the STRSPLIT
; command can remove these while testing sub-values, so they are not detected as part of
; any tests - Version 4.0b53
; 20200311: Do not include '.' separator when doing checks for sub-value separators at the start or end of
; the attribute values e.g. allow DO_NAME=Boyd;Ian S. - Version 4.0b54
;
; Inputs: metafile - Either, the filename of the input file containing the Metadata or, if program
; input is via string array and Structure, a string array containing the Global
......@@ -2157,8 +2189,9 @@ attr_arr_data_opt=['VAR_NOTES']
;Attributes that have user defined values
attr_free=['DATA_DESCRIPTION','DATA_MODIFICATIONS','DATA_CAVEATS',$
'DATA_RULES_OF_USE','DATA_ACKNOWLEDGEMENT',$
'VAR_DESCRIPTION','VAR_NOTES']
'DATA_RULES_OF_USE','DATA_ACKNOWLEDGEMENT','DATA_QUALITY', $
'DATA_PROCESSOR','FILE_PROJECT_ID','FILE_ASSOCIATION', $
'VAR_DESCRIPTION','VAR_NOTES']
;Atrributes that have numeric values (except for string datatype)
num_atts=['VAR_VALID_MIN','VAR_VALID_MAX','VAR_FILL_VALUE','_fillvalue','valid_range']
......@@ -2166,8 +2199,11 @@ num_atts=['VAR_VALID_MIN','VAR_VALID_MAX','VAR_FILL_VALUE','_fillvalue','valid_r
;To check for pre-defined attributes
ncsa=['long_name','units','format','coordsys','valid_range','_fillvalue','scale_factor', $
'scale_factor_err','add_offset','add_offset_err','calibrated_nt']
;Possible sub-value separators
sv_sep=['.','_',';']
nchar=65535 ;limit on number of characters
nchar=65535L ;limit on number of characters
;possible error messages for this procedure
errtxt=STRARR(9)
......@@ -2261,8 +2297,10 @@ IF ucnt NE 0L THEN BEGIN ;some attribute labels are not fully uppercase
ENDFOR
ENDIF
;Check for and remove leading and trailing spaces in Metadata (sub-)values (except for Free_Attributes and VAR_FILL_VALUEs)
;Also check for values with excessive length
;Check for and remove leading and trailing spaces in Metadata (sub-)values (except for free-text attributes
;and VAR_FILL_VALUEs). Also for non-free-text attributes check if the first and last characters are
;sub-value separators, or if there are consecutive sub-value separators in the attribute value.
;Also check for values with excessive length.
IF qa_yes THEN wstxt='are not permitted' ELSE wstxt='removed' ;used when reporting white space in sub-values
FOR i=0L,nrline-1L DO BEGIN
......@@ -2270,11 +2308,11 @@ FOR i=0L,nrline-1L DO BEGIN
;IF scnt EQ 0 THEN si=WHERE(m_v0[i] EQ attr_free,scnt)
si=WHERE(m_v0[i] EQ attr_free,scnt)
IF scnt EQ 0 THEN BEGIN ;not a free text attribute
nmi=WHERE(STRUPCASE(m_v0[i]) EQ STRUPCASE(num_atts),nmcnt) ;is it an attribute that should have a numeric value
res=STRSPLIT(m_v1[i],';',/Extract,COUNT=svcnt)
spi=WHERE(STRTRIM(res,2) NE res, spcnt)
spi=WHERE(STRTRIM(res,2) NE res,spcnt)
IF spcnt NE 0 THEN BEGIN
IF (svcnt EQ 1) AND (qa_yes) THEN BEGIN
nmi=WHERE(STRUPCASE(m_v0[i]) EQ STRUPCASE(num_atts),nmcnt) ;is it an attribute that should have a numeric value
IF (nmcnt NE 0) AND (STRTRIM(res[0],2) NE '') THEN infotxt='' $ value should be a number not a string so do not check for spaces
ELSE IF res[0] EQ ' ' THEN infotxt='' $ there could be a single space in the HDF file if there is no entry
ELSE infotxt='2 Leading or trailing spaces in '+m_v0[i]+'='+m_v1[i]+' value '+wstxt
......@@ -2286,6 +2324,33 @@ FOR i=0L,nrline-1L DO BEGIN
m_v1[i]=res[0]
IF svcnt GT 1 THEN FOR j=1,svcnt-1 DO m_v1[i]=m_v1[i]+';'+res[j]
ENDIF
IF nmcnt EQ 0 THEN BEGIN
;Check that first and last characters of the metadata value aren't sub-value separators
;and that there are no consecutive sub-value separators
mv1len=STRLEN(m_v1[i]) ;string length of attribute value
mv1start=STRMID(m_v1[i],0,1) ;first character
mv1end=STRMID(m_v1[i],mv1len-1L,1) ;last character
FOR k=0,1 DO BEGIN
writeonce=1B ;so that error message is only written once for each attribute
IF k EQ 0 THEN $
infotxt='3 Sub-value separators at the start or end of the attribute value are not permitted' $
ELSE infotxt='3 Consecutive sub-value separators are not permitted'
FOR j=0,N_ELEMENTS(sv_sep)-1 DO BEGIN
spchk=sv_sep[j]+sv_sep[j]
;Note: in some cases a '.' may be present at the end of a sub-value e.g. DO_NAME=Boyd;Ian S.
IF (k EQ 0) AND (sv_sep[j] NE '.') THEN test=(mv1start EQ sv_sep[j]) OR (mv1end EQ sv_sep[j]) $
ELSE IF k EQ 1 THEN test=STRPOS(m_v1[i],spchk) NE -1 $
ELSE test=0B
IF (test) AND (writeonce) THEN BEGIN
IF mv1len GT 50 THEN infotxt=infotxt+' in '+m_v0[i]+' attribute' $
ELSE infotxt=infotxt+': '+m_v0[i]+'='+m_v1[i]
INFOTXT_OUTPUT,infotxt
writeonce=0B
ENDIF
ENDFOR
ENDFOR
ENDIF
ENDIF
;check for string length of attribute value
IF STRLEN(m_v1[i]) GT nchar THEN BEGIN
......@@ -2698,6 +2763,8 @@ PRO extract_and_test, dentry, cpos, nel, ta1, ta2, taname, aname, ti
; message - Version 4.0b41
; 20171121: Fix error caused by incorrectly referring to the variable infoerr as inforerr
; - Version 4.0b43
; 20191205: Account for DATA_SOURCE being able to have 2 or 3 sub-values, where the third sub-
; value is a VERSION_NAME - Version 4.0b53
;
; Inputs: dentry - the Metadata entry to be checked (everything after the '=')
; cpos - a pattern used by StrSplit on DEntry to split it into its component parts e.g. ';' or '_'
......@@ -2731,8 +2798,9 @@ infoerr=['COUNTRY','AFFILIATION','FILE_ACCESS','FILE_META_VERSION','DATA_LOCATIO
'DATA_DISCIPLINE','DATA_GROUP'] ;do not stop the program for errors in these attributes
;Check number of Metadata entry components is as expected
achk=STRSPLIT(dentry,cpos,/Extract)
IF N_ELEMENTS(achk) NE nel THEN BEGIN
achk=STRSPLIT(dentry,cpos,/Extract,COUNT=n_achk)
oki=WHERE(n_achk EQ nel,okcnt) ;Note: DATA_SOURCE can have 2 or 3 sub-values (nel=[2,3] from 4.0b53)
IF okcnt EQ 0 THEN BEGIN
STOP_WITH_ERROR,o3[3]+proname+taname+'='+dentry+': ',errtxt2[0]+STRTRIM(nel,2)+'.',lu
RETURN
ENDIF
......@@ -2858,6 +2926,8 @@ PRO check_metadata
; 20170331: Allow for COUNTRY value in XX_ADDRESS to be all uppercase or have initial capitalization
; (allow for special cases e.g. Cote d'Ivoire; words such as the, and, of also not
; capitalized) - Version 4.0b42
; 20191205: Add check for optional VERSION_NAME sub-value of DATA_SOURCE. If present then call
; GEOMS_RULE_CHANGES to do checks on the value - Version 4.0b53
;
; Inputs: meta_arr - a string array containing the Global and Variable Attributes
; tab_arr - a string array containing the TAV or table.dat file attributes
......@@ -3021,8 +3091,16 @@ FOR i=0,N_ELEMENTS(meta_arr)-1 DO BEGIN
ENDIF
ta2=tab_arr[ai[0],2:tab_arr[ai[0],1]+1] ;AFFILIATION/DATA_SOURCE_02 ENTRIES
taname=res[0]+atx+'/'+tab_arr[ai[0],0] ;name combination for error message
EXTRACT_AND_TEST,res[1],'_',2,ta1,ta2,taname,res[1],ti
n_sv=[2,3] ;possible number of DATA_SOURCE sub-values
EXTRACT_AND_TEST,res[1],'_',n_sv,ta1,ta2,taname,res[1],ti
IF STRLEN(rerr[0]) GT 2 THEN RETURN
achk=STRSPLIT(res[1],'_',/EXTRACT,COUNT=n_achk)
IF n_achk EQ 3 THEN BEGIN
;Check DATA_VERSION_NAME if there are 3 sub-values in DATA_SOURCE (introduced v4.0b53)
GEOMS_RULE_CHANGES,11,achk[2],meta_arr
ENDIF
ENDIF ELSE IF res[0] EQ 'DATA_VARIABLES' THEN BEGIN
;make into 2-D array and separate out the components of the VAR_NAMES
achk=STRSPLIT(res[1],';',/Extract,COUNT=nel)
......@@ -4916,8 +4994,8 @@ FOR i=0,N_ELEMENTS(dt)-1 DO BEGIN
IF i EQ 5 THEN ai=WHERE(attr_arr_glob EQ dt[i]) $
ELSE ai=WHERE(attr_arr_glob EQ 'DATA_'+dt[i])
res=STRSPLIT(meta_arr[ai[0]],' =;',/Extract)
IF (N_ELEMENTS(res) NE nei) AND ((i LE 2) OR (i GE 6)) THEN BEGIN
res=STRSPLIT(meta_arr[ai[0]],' =;',/Extract,COUNT=cres)
IF (cres NE nei) AND ((i LE 2) OR (i GE 6)) THEN BEGIN
;infotxt='3 '+errtxt[0]+dt[i]+errtxt[1]+nes
;INFOTXT_OUTPUT,infotxt
STOP_WITH_ERROR,o3[3]+proname+meta_arr[ai[0]]+': ',errtxt[0]+dt[i]+errtxt[1]+STRTRIM(nei-1,2),lu
......@@ -5785,7 +5863,7 @@ PRO idlcr8hdf, ga, sds, tav, odir, reterr, H5=o1, AVK=o2, LOG=o4, POPUP=o5, QA=o
;
;Program documentation, idlcr8hdf-v4.0_Readme.pdf, available from http://avdc.gsfc.nasa.gov.
;
;Program sub-version 4.0b52 (20190824)
;Program sub-version 4.0b54 (20201103)
; ----------
;Written by Ian Boyd for the AVDC - iboyd@astro.umass.edu
;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment