Replace idlcr8hdf.pro

idlcr8hdf.pro

-;Main Program Version: idlcr8hdf.pro v4.0b52, 20190824
+;Main Program Version: idlcr8hdf.pro v4.0b53, 20191205
 ;  Written by Ian Boyd for the AVDC - iboyd@astro.umass.edu
 ;
 ;Sub-versions (refer to idlcr8hdf-v4.0_Readme.pdf for full history)
@@ -113,7 +113,7 @@ PRO intro, intype
 COMMON WIDGET_WIN
 
 nhdr=44 & errtxt=STRARR(nhdr)
-vertxt=['idlcr8hdf-v4.0_Readme.pdf','v4.0b52 August 2019']
+vertxt=['idlcr8hdf-v4.0_Readme.pdf','v4.0b53 December 2019']
 errtxt[1]='Welcome to IDLcr8HDF.  This program creates GEOMS compliant HDF4, HDF5 and netCDF files'
 errtxt[2]='(also refer to '+vertxt[0]+').'
 errtxt[4]='Inputs to the program (IDL Virtual Machine (VM) and IDL Licensed (LIC) Versions):'
@@ -1230,6 +1230,8 @@ PRO geoms_rule_changes, code, in1, in2, in3, in4
 ;    20190506: Add FILE_META_VERSION information to INFORMATION/ERROR comment when 
 ;              DATA_TEMPLATE value is not as expected; Fix bug that caused a crash if the
 ;              file does not use a DATA_TEMPLATE (rule 9) - Version 4.0b50
+;    20191205: Add rule 11 which does checks on optional VERSION_NAME sub-field of
+;              DATA_SOURCE - Version 4.0b53
 ;
 ;  Inputs: code - Integer value identifying type of check to carry out
 ;          in1 - First set of inputs required for checks (optional, dependent on code value)
@@ -1284,6 +1286,8 @@ PRO geoms_rule_changes, code, in1, in2, in3, in4
 ;          Inputs: in1=m_v0, in2=m_v1, in3=ga_chk
 ;  Code 10: Do checks on ISO646-US ASCII character set
 ;           Inputs: in1=meta_arr or dtest, in2=vn[vc] (Variable Name for datasets only)
+;  Code 11: Do checks on DATA_VERSION_NAME part of DATA_SOURCE value
+;           Inputs: in1=DATA_VERSION_NAME, in2=meta_arr
 
 COMMON TABLEDATA
 COMMON METADATA
@@ -1903,7 +1907,29 @@ CASE 1 OF
         ENDIF
       ENDFOR
     END
-
+  code EQ 11: BEGIN ;Check DATA_VERSION_NAME part of the DATA_SOURCE
+      ;Check that entry is made up of dot-separated alpha_numeric words
+      n_char=STRLEN(in1)
+      strok=1B ;default is that entry is OK
+      test2=0B & test3=0B ;will change to 1 if test conditions are true
+      charchkm1=0B ;to check test character against the previous character
+      FOR i=0,n_char-1 DO BEGIN
+        charchk=BYTE(STRMID(in1,i,1))
+        test1=(charchk EQ 46B) OR ((charchk GE 65B) AND (charchk LE 90B)) OR $ ;'.' or A-Z
+              ((charchk GE 97B) AND (charchk LE 122B)) OR $ ;a-z
+              ((charchk GE 48B) AND (charchk LE 57B)) ;0-9
+        IF (i EQ 0) OR (i EQ n_char-1) THEN test2=charchk EQ 46B ;i.e. first or last character is a '.'
+        test3=(charchk EQ 46B) AND (charchkm1 EQ 46B) ;i.e. two consecutive dots
+        IF (~test1) OR (test2) OR (test3) THEN strok=0B
+        charchkm1=charchk 
+      ENDFOR
+      IF ~strok THEN BEGIN
+        ;write error message
+        infotxt='3 VERSION_NAME sub-value of the DATA_SOURCE must consist of dot-separated alpha-numeric words: '
+        infotxt=infotxt+in1
+        INFOTXT_OUTPUT,infotxt
+      ENDIF
+    END  
 ENDCASE
 
 END ;Procedure GEOMS_Rule_Changes
@@ -2090,6 +2116,10 @@ PRO read_metadata, metafile, inf
 ;    20151109: Add log message when adding/checking for missing mandatory variable attributes - Version 4.0b34
 ;    20151116: Fix bug when doing whitespace checks and there is no metadata entry - Version 4.0b35
 ;    20180528: Fix bug when adding non-standard global attributes to meta_arr - Version 4.0b47
+;    20191205: Do checks for sub-value separators (. or ; or _) being at the start or end of attribute
+;              values, and for consecutive sub-value separators (e.g. ;;). This can occur as the STRSPLIT
+;              command can remove these while testing sub-values, so they are not detected as part of 
+;              any tests - Version 4.0b53 
 ;
 ;  Inputs: metafile - Either, the filename of the input file containing the Metadata or, if program
 ;                     input is via string array and Structure, a string array containing the Global
@@ -2157,8 +2187,9 @@ attr_arr_data_opt=['VAR_NOTES']
 
 ;Attributes that have user defined values
 attr_free=['DATA_DESCRIPTION','DATA_MODIFICATIONS','DATA_CAVEATS',$
-  'DATA_RULES_OF_USE','DATA_ACKNOWLEDGEMENT',$
-  'VAR_DESCRIPTION','VAR_NOTES']
+           'DATA_RULES_OF_USE','DATA_ACKNOWLEDGEMENT','DATA_QUALITY', $
+           'DATA_PROCESSOR','FILE_PROJECT_ID','FILE_ASSOCIATION', $
+           'VAR_DESCRIPTION','VAR_NOTES']
 
 ;Atrributes that have numeric values (except for string datatype)
 num_atts=['VAR_VALID_MIN','VAR_VALID_MAX','VAR_FILL_VALUE','_fillvalue','valid_range']
@@ -2166,8 +2197,11 @@ num_atts=['VAR_VALID_MIN','VAR_VALID_MAX','VAR_FILL_VALUE','_fillvalue','valid_r
 ;To check for pre-defined attributes
 ncsa=['long_name','units','format','coordsys','valid_range','_fillvalue','scale_factor', $
       'scale_factor_err','add_offset','add_offset_err','calibrated_nt']
+      
+;Possible sub-value separators
+sv_sep=['.','_',';']
 
-nchar=65535 ;limit on number of characters
+nchar=65535L ;limit on number of characters
 
 ;possible error messages for this procedure
 errtxt=STRARR(9)
@@ -2261,8 +2295,10 @@ IF ucnt NE 0L THEN BEGIN ;some attribute labels are not fully uppercase
   ENDFOR
 ENDIF
 
-;Check for and remove leading and trailing spaces in Metadata (sub-)values (except for Free_Attributes and VAR_FILL_VALUEs)
-;Also check for values with excessive length
+;Check for and remove leading and trailing spaces in Metadata (sub-)values (except for free-text attributes 
+;and VAR_FILL_VALUEs). Also for non-free-text attributes check if the first and last characters are 
+;sub-value separators, or if there are consecutive sub-value separators in the attribute value.
+;Also check for values with excessive length.
 IF qa_yes THEN wstxt='are not permitted' ELSE wstxt='removed' ;used when reporting white space in sub-values
 
 FOR i=0L,nrline-1L DO BEGIN
@@ -2270,11 +2306,11 @@ FOR i=0L,nrline-1L DO BEGIN
   ;IF scnt EQ 0 THEN si=WHERE(m_v0[i] EQ attr_free,scnt)
   si=WHERE(m_v0[i] EQ attr_free,scnt)
   IF scnt EQ 0 THEN BEGIN ;not a free text attribute
+    nmi=WHERE(STRUPCASE(m_v0[i]) EQ STRUPCASE(num_atts),nmcnt) ;is it an attribute that should have a numeric value
     res=STRSPLIT(m_v1[i],';',/Extract,COUNT=svcnt)
-    spi=WHERE(STRTRIM(res,2) NE res, spcnt)
+    spi=WHERE(STRTRIM(res,2) NE res,spcnt)
     IF spcnt NE 0 THEN BEGIN
       IF (svcnt EQ 1) AND (qa_yes) THEN BEGIN
-        nmi=WHERE(STRUPCASE(m_v0[i]) EQ STRUPCASE(num_atts),nmcnt) ;is it an attribute that should have a numeric value
         IF (nmcnt NE 0) AND (STRTRIM(res[0],2) NE '') THEN infotxt='' $ value should be a number not a string so do not check for spaces
         ELSE IF res[0] EQ ' ' THEN infotxt='' $ there could be a single space in the HDF file if there is no entry
         ELSE infotxt='2 Leading or trailing spaces in '+m_v0[i]+'='+m_v1[i]+' value '+wstxt
@@ -2286,6 +2322,31 @@ FOR i=0L,nrline-1L DO BEGIN
       m_v1[i]=res[0]
       IF svcnt GT 1 THEN FOR j=1,svcnt-1 DO m_v1[i]=m_v1[i]+';'+res[j]
     ENDIF
+    
+    IF nmcnt EQ 0 THEN BEGIN
+      ;Check that first and last characters of the metadata value aren't sub-value separators
+      ;and that there are no consecutive sub-value separators
+      mv1len=STRLEN(m_v1[i]) ;string length of attribute value
+      mv1start=STRMID(m_v1[i],0,1) ;first character
+      mv1end=STRMID(m_v1[i],mv1len-1L,1) ;last character
+      FOR k=0,1 DO BEGIN
+        writeonce=1B ;so that error message is only written once for each attribute
+        IF k EQ 0 THEN $
+          infotxt='3 Sub-value separators at the start or end of the attribute value are not permitted' $
+        ELSE infotxt='3 Consecutive sub-value separators are not permitted'
+        FOR j=0,N_ELEMENTS(sv_sep)-1 DO BEGIN
+          spchk=sv_sep[j]+sv_sep[j]
+          IF k EQ 0 THEN test=(mv1start EQ sv_sep[j]) OR (mv1end EQ sv_sep[j]) $
+          ELSE test=STRPOS(m_v1[i],spchk) NE -1
+          IF (test) AND (writeonce) THEN BEGIN
+            IF mv1len GT 50 THEN infotxt=infotxt+' in '+m_v0[i]+' attribute' $
+            ELSE infotxt=infotxt+': '+m_v0[i]+'='+m_v1[i]
+            INFOTXT_OUTPUT,infotxt
+            writeonce=0B
+          ENDIF
+        ENDFOR
+      ENDFOR
+    ENDIF
   ENDIF
   ;check for string length of attribute value
   IF STRLEN(m_v1[i]) GT nchar THEN BEGIN
@@ -2698,6 +2759,8 @@ PRO extract_and_test, dentry, cpos, nel, ta1, ta2, taname, aname, ti
 ;              message - Version 4.0b41
 ;    20171121: Fix error caused by incorrectly referring to the variable infoerr as inforerr
 ;              - Version 4.0b43
+;    20191205: Account for DATA_SOURCE being able to have 2 or 3 sub-values, where the third sub-
+;              value is a VERSION_NAME - Version 4.0b53
 ;
 ;  Inputs: dentry - the Metadata entry to be checked (everything after the '=')
 ;          cpos - a pattern used by StrSplit on DEntry to split it into its component parts e.g. ';' or '_'
@@ -2731,8 +2794,9 @@ infoerr=['COUNTRY','AFFILIATION','FILE_ACCESS','FILE_META_VERSION','DATA_LOCATIO
          'DATA_DISCIPLINE','DATA_GROUP'] ;do not stop the program for errors in these attributes
 
 ;Check number of Metadata entry components is as expected
-achk=STRSPLIT(dentry,cpos,/Extract)
-IF N_ELEMENTS(achk) NE nel THEN BEGIN
+achk=STRSPLIT(dentry,cpos,/Extract,COUNT=n_achk)
+oki=WHERE(n_achk EQ nel,okcnt) ;Note: DATA_SOURCE can have 2 or 3 sub-values (nel=[2,3] from 4.0b53)
+IF okcnt EQ 0 THEN BEGIN
   STOP_WITH_ERROR,o3[3]+proname+taname+'='+dentry+': ',errtxt2[0]+STRTRIM(nel,2)+'.',lu
   RETURN
 ENDIF
@@ -2858,6 +2922,8 @@ PRO check_metadata
 ;    20170331: Allow for COUNTRY value in XX_ADDRESS to be all uppercase or have initial capitalization
 ;              (allow for special cases e.g. Cote d'Ivoire; words such as the, and, of also not
 ;              capitalized) - Version 4.0b42
+;    20191205: Add check for optional VERSION_NAME sub-value of DATA_SOURCE. If present then call
+;              GEOMS_RULE_CHANGES to do checks on the value - Version 4.0b53 
 ;
 ;  Inputs: meta_arr - a string array containing the Global and Variable Attributes
 ;          tab_arr - a string array containing the TAV or table.dat file attributes
@@ -3021,8 +3087,16 @@ FOR i=0,N_ELEMENTS(meta_arr)-1 DO BEGIN
       ENDIF
       ta2=tab_arr[ai[0],2:tab_arr[ai[0],1]+1] ;AFFILIATION/DATA_SOURCE_02 ENTRIES
       taname=res[0]+atx+'/'+tab_arr[ai[0],0] ;name combination for error message
-      EXTRACT_AND_TEST,res[1],'_',2,ta1,ta2,taname,res[1],ti
+      n_sv=[2,3] ;possible number of DATA_SOURCE sub-values
+      EXTRACT_AND_TEST,res[1],'_',n_sv,ta1,ta2,taname,res[1],ti
       IF STRLEN(rerr[0]) GT 2 THEN RETURN
+      
+      achk=STRSPLIT(res[1],'_',/EXTRACT,COUNT=n_achk)
+      IF n_achk EQ 3 THEN BEGIN
+        ;Check DATA_VERSION_NAME if there are 3 sub-values in DATA_SOURCE (introduced v4.0b53)
+        GEOMS_RULE_CHANGES,11,achk[2],meta_arr
+      ENDIF
+      
     ENDIF ELSE IF res[0] EQ 'DATA_VARIABLES' THEN BEGIN
       ;make into 2-D array and separate out the components of the VAR_NAMES
       achk=STRSPLIT(res[1],';',/Extract,COUNT=nel)
@@ -4916,8 +4990,8 @@ FOR i=0,N_ELEMENTS(dt)-1 DO BEGIN
   IF i EQ 5 THEN ai=WHERE(attr_arr_glob EQ dt[i]) $
   ELSE ai=WHERE(attr_arr_glob EQ 'DATA_'+dt[i])
 
-  res=STRSPLIT(meta_arr[ai[0]],' =;',/Extract)
-  IF (N_ELEMENTS(res) NE nei) AND ((i LE 2) OR (i GE 6)) THEN BEGIN
+  res=STRSPLIT(meta_arr[ai[0]],' =;',/Extract,COUNT=cres)
+  IF (cres NE nei) AND ((i LE 2) OR (i GE 6)) THEN BEGIN
     ;infotxt='3 '+errtxt[0]+dt[i]+errtxt[1]+nes
     ;INFOTXT_OUTPUT,infotxt
     STOP_WITH_ERROR,o3[3]+proname+meta_arr[ai[0]]+': ',errtxt[0]+dt[i]+errtxt[1]+STRTRIM(nei-1,2),lu
@@ -5785,7 +5859,7 @@ PRO idlcr8hdf, ga, sds, tav, odir, reterr, H5=o1, AVK=o2, LOG=o4, POPUP=o5, QA=o
 ;
 ;Program documentation, idlcr8hdf-v4.0_Readme.pdf, available from http://avdc.gsfc.nasa.gov.
 ;
-;Program sub-version 4.0b52 (20190824)
+;Program sub-version 4.0b53 (20191205)
 ; ----------
 ;Written by Ian Boyd for the AVDC - iboyd@astro.umass.edu
 ;