From 86e4b8b001eaa5a1ad297765cbc152872d9ee2ce Mon Sep 17 00:00:00 2001
From: Ian Boyd <iboyd@astro.umass.edu>
Date: Tue, 29 Oct 2024 02:00:33 +0000
Subject: [PATCH] Replace geoms_tctools.pro
---
geoms_tctools.pro | 1452 +++++++++++++++++++++++----------------------
1 file changed, 727 insertions(+), 725 deletions(-)
diff --git a/geoms_tctools.pro b/geoms_tctools.pro
index 60d9e97..fc1a980 100644
--- a/geoms_tctools.pro
+++ b/geoms_tctools.pro
@@ -1,725 +1,727 @@
-;---------------------------------------------------------------------------------------------------
-;
-; name: geoms_tctools.pro
-; description: AVDC GEOMS template checker tool
-;
-; 2011-02-03, v0.1 Christian Retscher initial implementation
-; 2011-04-11, v0.2 Christian Retscher MWR support addded
-; concept: 1) GEOMS-VA -> HDF; 2) HDF -> GEOMS-VA checks
-; 2011-04-17, v0.3 Christian Retscher FTIR support start
-; filter for [SOLAR|LUNAR], [ppmv|ppbv|pptv], "...()"
-; 2011-04-23, v0.4 Christian Retscher FTIR support finalized, LIDAR support tested
-; 2011-04-25, v0.5 Christian Retscher Full LIDAR support added
-; 2011-04-26, v0.6 Christian Retscher Split in wrapper and TC tools (keep versioning)
-; 2011-05-13, v0.7 Christian Retscher Integrate UVVIS.DOAS
-; 2011-05-26, v0.8 Christian Retscher check VAR_DEPEND
-; 2011-09-07, v0.9 Christian Retscher adapted filter for FTIR "(or a unit scaled"
-; 2012-04-25, v1.0 Ian Boyd fixed bug when checking for conditional mandatory variables by
-; adding extra checks; for optional checks add actual variable name rather
-; than template name e.g. [GAS] replaced by O3
-; 2013-12-19, v1.3 Ian Boyd fixed bug when checking the template against the file - errors
-; were being generated but the iCheckTotal was not being assigned correctly
-; meaning that final result was showing that template was OK
-; 2014-01-28, v1.34 Ian Boyd GEOMS group introduced rule allowing for conditional variables
-; to be treated as optional when they do not otherwise meet the condition -
-; previously this generated an error
-; 2015-02-17, v1.43 Ian Boyd When checking VAR_UNITS, strip away any white space from the
-; VAR_UNITS value
-; 2016-11-30, v1.57 Ian Boyd Account for non-GEOMS variable attributes (ptr_valid check)
-; 2019-12-05, v2.00 Ian Boyd Add check_for_version_name function to check for optional Version
-; name field in DATA_SOURCE. This determines whether to write out an end of
-; check message
-; --------------------------------------------------------------------------------------------------
-
-pro STRREPLACE, Strings, Find1, Replacement1
-
-; Check integrity of input parameter
-
- NP = N_PARAMS()
- if (NP ne 3) then message,'Must be called with 3 parameters, '+$
- 'Strings, Find, Replacement'
-
- sz = SIZE(Strings)
- ns = n_elements(sz)
- if (sz(ns-2) ne 7) then message,'Parameter must be of string type.'
-
- Find = STRING(Find1)
- pos = STRPOS(Strings,Find)
- here = WHERE(pos ne -1, nreplace)
-
- if (nreplace eq 0) then return
-
- Replacement=STRING(Replacement1)
- Flen = strlen(Find)
- for i=0,nreplace-1 do begin
-
- j = here(i)
- prefix = STRMID(Strings(j),0,pos(j))
- suffix = STRMID(Strings(j),pos(j)+Flen,$
- strlen(Strings(j))-(pos(j)+Flen))
- Strings(j) = prefix + replacement + suffix
- endfor
-end
-
-
-;---------------------------------------------------------------------------------------------------
-FUNCTION read_GEOMS_template, templatefile
-
- ; read template file
- openr, lu, templatefile, /GET_LUN
-
- iSizeDum1 = 10000
- chGEOMSTempDum = strarr(iSizeDum1)
-
- iC1=0
- chTempLine=''
- repeat BEGIN
- readf, lu, chTempLine
- chGEOMSTempDum( iC1 ) = chTempLine
- iC1 += 1
- ENDREP until eof(lu)
-
- ;resize input file array
- chGEOMSTemp = strarr(iC1)
- chGEOMSTemp(*) = chGEOMSTempDum(0:iC1-1)
-
- close,lu
-
- return, chGEOMSTemp
-END
-;---------------------------------------------------------------------------------------------------
-FUNCTION check_for_version_name, ga
-;determine if DATA_SOURCE has a VERSION_NAME field
-
- ga=STRTRIM(ga,2)
- dsi = WHERE(STRMID(STRUPCASE(ga),0,11) eq 'DATA_SOURCE',dscnt)
- if dscnt eq 1 then begin
- dsv = strmid(ga[dsi[0]], strpos(ga[dsi[0]],'=')+1) ;everything after the '=' sign
- dsx = strsplit(strtrim(dsv,2),'_',/Extract,count=cdsx)
- if cdsx eq 3 then return, dsx[2] else return, ''
- endif else return, ''
-END
-
-
-;---------------------------------------------------------------------------------------------------
-FUNCTION checkgas, chVarCheck, chSearch, chHDFGas
-
- if (strpos(chVarCheck, chSearch) ge 0) then begin
- STRREPLACE, chVarCheck, chSearch, chHDFGas
- end
-
- return, chVarCheck
-END
-
-;---------------------------------------------------------------------------------------------------
-FUNCTION check_data_source, ga
- ; get GA and extract DATA_SOURCE
- for iR1 = 0, size(ga, /N_ELEMENTS) - 1 do begin
- chGATmp = strsplit(ga(iR1), '=', /EXTRACT)
-
- if (chGATmp(0) eq 'DATA_SOURCE') then begin
- chHDFDS = strsplit(chGATmp(1), ';', /EXTRACT)
- end
- end
-
- ; extract name and gas of data source
- chHDFDSDum1 = strsplit(chHDFDS, '_', /EXTRACT)
- chHDFDSDum = chHDFDSDum1
-
- ; check if data source contains '.', then split
- if ( strpos( chHDFDSDum1(0), '.' ) ge 0 ) then begin
- chHDFDSDum2 = strsplit(chHDFDSDum1(0), '.', /EXTRACT)
- chHDFDSDum = chHDFDSDum2
- end
-
- iSize1 = size( chHDFDSDum, /N_ELEMENTS )
-
- chHDFName = chHDFDSDum(0)
- chHDFGas = chHDFDSDum(iSize1 - 1)
-
- return, chHDFGas
-end
-
-;---------------------------------------------------------------------------------------------------
-FUNCTION find_optional_value1, chGEOMSTempVAVal, chHDFVAVal
- chDelim2 = '|'
- chVarOptTmp1 = chGEOMSTempVAVal
-
- ; remove []
- STRREPLACE, chVarOptTmp1, '[', ''
- STRREPLACE, chVarOptTmp1, ']', ''
-
- chGEOMSTempVAValArr = strsplit(chVarOptTmp1, chDelim2, /EXTRACT)
-
- iPos = where( chGEOMSTempVAValArr eq chHDFVAVal)
-
- if ( iPos ge 0 ) then begin
- iOutput = 1
- end else begin
- iOutput = 0
- end
-
- return, iOutput
-end
-
-;---------------------------------------------------------------------------------------------------
-FUNCTION find_optional_value2, chGEOMSTempVAVal, chHDFVAVal
- chDelim2 = '('
- chVarOptTmp1 = chGEOMSTempVAVal
-
- ; remove []
- STRREPLACE, chVarOptTmp1, '"', ''
-
- chGEOMSTempVAValArr = strsplit(chVarOptTmp1, chDelim2, /EXTRACT)
-
-; iPos = where( STRTRIM(chGEOMSTempVAValArr(0)) eq STRTRIM(chHDFVAVal))
-
- ; split first letter
- chHDFVAVal2 = strmid(STRTRIM(chHDFVAVal), 1, strlen(STRTRIM(chHDFVAVal)))
-
- iPos1 = where(STRTRIM(chGEOMSTempVAValArr(0)) eq STRTRIM(chHDFVAVal2))
- iPos2 = where(STRTRIM(chGEOMSTempVAValArr(0)) eq STRTRIM(chHDFVAVal))
-
- if ((iPos1 ge 0) or (iPos2 ge 0)) then begin
- iOutput = 1
- end else begin
- iOutput = 0
- end
-
- return, iOutput
-end
-
-;---------------------------------------------------------------------------------------------------
-FUNCTION find_optional_value3, chGEOMSTempVAVal
- chDelim2 = '|'
- chVarOptTmp1 = chGEOMSTempVAVal
-
- ; remove []
- STRREPLACE, chVarOptTmp1, '[', ''
- STRREPLACE, chVarOptTmp1, ']', ''
-
- chGEOMSTempVAValArr = strsplit(chVarOptTmp1, chDelim2, /EXTRACT)
-
- return, chGEOMSTempVAValArr
-end
-
-;---------------------------------------------------------------------------------------------------
-FUNCTION check_opt_var, chGEOMSTempVARed, chSearch
-
- iSize = size(chGEOMSTempVARed, /N_ELEMENTS)
-
- iPos = -1
- for iR2 = 0, iSize - 1 do begin
- iPosTmp = strpos(chGEOMSTempVARed(iR2), chSearch)
- if (iPosTmp ge 0) then begin
- iPos = iPosTmp
- end
- end
-
- return, iPos
-end
-
-;---------------------------------------------------------------------------------------------------
-FUNCTION get_optional_vars, chHDFVARedName, chVarOpt
- chDelim1 = '.'
- chDelim2 = '_'
- chDelim3 = '|'
- chDelim4 = '['
- chDelim5 = ']'
-
- chVarOptTmp1 = chVarOpt
-
- ; split after []
- chGEOMSTempVAEle = strsplit(chVarOptTmp1, chDelim5, /EXTRACT)
-
- chVarOptTmp1 = chGEOMSTempVAEle(0)
-
- ; remove []
- STRREPLACE, chVarOptTmp1, chDelim1, ''
- STRREPLACE, chVarOptTmp1, chDelim4, ''
- STRREPLACE, chVarOptTmp1, chDelim5, ''
-
- chGEOMSTempVAOpt = strsplit(chVarOptTmp1, chDelim3, /EXTRACT)
-
- ; size of HDF variables
- iSize3 = size(chHDFVARedName(*,0), /N_ELEMENTS)
- ; size of HDF variable options
- iSize6 = size(chGEOMSTempVAOpt, /N_ELEMENTS)
- ; size of HDF variable split
- iSize7 = size(chGEOMSTempVAEle, /N_ELEMENTS)
-
- chGEOMSTempVAOptCount = INTARR(iSize6)
-
- for iR6 = 0, iSize6 - 1 do begin
- iC6 = 0
-
- for iR3 = 0, iSize3 - 1 do begin
- ;print, chHDFVARedName( iR3, 0 )
-
- if (iSize7 gt 1 ) then begin
- iFound = strpos(chHDFVARedName(iR3, 0), chGEOMSTempVAOpt(iR6)+chGEOMSTempVAEle(1))
- end else begin
- iFound = strpos(chHDFVARedName(iR3, 0), chDelim1+chGEOMSTempVAOpt(iR6))
- end
-
- if ( iFound ge 0 ) then begin
- iC6+=1
- end
- end
-
- chGEOMSTempVAOptCount(iR6) = iC6
- end
-
- iPosMax = where( chGEOMSTempVAOptCount eq max(chGEOMSTempVAOptCount))
-
- chGEOMSTempVAOptOut = chGEOMSTempVAOpt(iPosMax(0))
-
- if (iSize7 gt 1 ) then begin
- chGEOMSTempVAOptOut = chGEOMSTempVAOptOut+chGEOMSTempVAEle(1)
- end
-
- return, chGEOMSTempVAOptOut
-end
-
-;---------------------------------------------------------------------------------------------------
-FUNCTION analyze_GEOMS_VA, ga, sds, chGEOMSTempVA, lu
-
- tccode = ['0 (Passed template checks)', '1 (Failed template checks)']
-
- ; get [GAS] form data source
- chHDFGas = check_data_source( ga )
-
- ; size of GEOMS-TE VA
- iSize1 = size(STRSPLIT(chGEOMSTempVA(0), ',',/EXTRACT), /N_ELEMENTS)
- ; size of GEOMS-TE VA
- iSize2 = size(chGEOMSTempVA, /N_ELEMENTS)
- ; size of HDF variables
- iSize3 = size(sds(*,0), /N_ELEMENTS )
- ; size of HDF VA
- iSize4 = size(sds(0,*), /N_ELEMENTS )
- ; size of predefined VA
- iSize5 = iSize1 - 3 ;size(chVA, /N_ELEMENTS )
-
- ; decide on type of data template
- if (iSize1 eq 5) then begin
- chVA = [ 'VAR_NAME', 'VAR_UNITS' ]
- end
- if (iSize1 eq 6) then begin
- chVA = [ 'VAR_NAME', 'VAR_UNITS', 'VAR_DATA_TYPE' ]
- end
- if (iSize1 eq 7) then begin
- chVA = [ 'VAR_NAME', 'VAR_UNITS', 'VAR_DATA_TYPE', 'VAR_DEPEND' ]
- end
-
- ; 4 elements read from HDF file, max 5 elements read from template
- chHDFVARed = strarr(iSize3,4)
- chGEOMSTempVARed = strarr(iSize2,5)
- chGEOMSTempVARedReplace = strarr(iSize2,5)
-
- ; read HDF VA and assign values
- ; dynamic check on list of predefined variables attributes chVA
- for iR3 = 0, iSize3 - 1 do begin
- ; loop trough lookup VA
- for iR5 = 0, iSize5 - 1 do begin
- ; loop trough all HDF VA
-; for iR4 = 0, iSize4 - 1 do begin
-; ; look for matching VA values
-; print, *sds(iR3,iR4).va_l+', '+*sds(0,iR4).va_l+', '+chVA(iR5)
-; if (*sds(iR3,iR4).va_l eq chVA(iR5)) then begin
-; ; assign variable attributes
-; chHDFVARed(iR3,iR5) = *sds(iR3,iR4).va_v
-;
-; print, iR3,iR5, iR4, chHDFVARed(iR3,iR5)
-; end
-
- ;ptr_valid check added v1.57
- iC4 = 0 & goodexit = 0
- while goodexit eq 0 do begin
- if ptr_valid(sds(iR3,iC4).va_l) then begin
- if *sds(iR3,iC4).va_l eq chVA(iR5) then goodexit = 1 else iC4++
- endif else goodexit = -1 ;non-valid entry
- end
-
- if goodexit eq 1 then chHDFVARed(iR3,iR5) = *sds(iR3,iC4).va_v $
- else if iR5 eq 0 then chHDFVARed(iR3,iR5) = 'Invalid/Unknown' ;*sds(iR3,0).va_v
- end
- ;print, STRING(iR3)+','+chHDFVARed(iR3, 0)+','+ chHDFVARed(iR3, 1)+','+ chHDFVARed(iR3, 2)+','+ chHDFVARed(iR3, 3)
- end
-
- ; read GEOMS-TE VA and assign values
- ; The reading of the tempate CSV files strongly depends on its format.
- for iR2 = 0, iSize2 - 1 do begin
- chGEOMSTempVATmp = strsplit(chGEOMSTempVA(iR2), ',', /EXTRACT)
-
- ; template variable name
- chGEOMSTempVARed(iR2,0) = strtrim(chGEOMSTempVATmp(1))
-
- ; LIDAR 2011-04-23
- if ( iSize1 ge 5 ) then begin
- ; template variable units
- chGEOMSTempVARed(iR2,1) = strtrim(chGEOMSTempVATmp(2))
- ; template variable requirement
- chGEOMSTempVARed(iR2,3) = strtrim(chGEOMSTempVATmp(size(chGEOMSTempVATmp, /N_ELEMENTS) - 1))
- end
-
- ; MWR, FTIR 2011-04-23
- if ( iSize1 ge 6 ) then begin
- ; template variable units
- chGEOMSTempVARed(iR2,1) = strtrim(chGEOMSTempVATmp(2))
- ; template variable data type
- chGEOMSTempVARed(iR2,2) = strtrim(chGEOMSTempVATmp(3))
- ; template variable requirement
- chGEOMSTempVARed(iR2,3) = strtrim(chGEOMSTempVATmp(size(chGEOMSTempVATmp, /N_ELEMENTS) - 1))
- end
-
- ; FTIR 2011-05-18
- if ( iSize1 ge 7 ) then begin
- ; template variable units
- chGEOMSTempVARed(iR2,1) = strtrim(chGEOMSTempVATmp(3))
- ; template variable data type
- chGEOMSTempVARed(iR2,2) = strtrim(chGEOMSTempVATmp(4))
- ; template variable requirement
- chGEOMSTempVARed(iR2,3) = strtrim(chGEOMSTempVATmp(size(chGEOMSTempVATmp, /N_ELEMENTS) - 1))
- ; template variable dependencies (dimensions)
- chGEOMSTempVARed(iR2,4) = strtrim(chGEOMSTempVATmp(2))
- end
-
- ; template variable dependencies
- ;chGEOMSTempVARed(iR2,1) = strtrim(chGEOMSTempVATmp(size(chGEOMSTempVATmp, /N_ELEMENTS) - 1))
-
- ;print, STRING(iR2)+' '+chGEOMSTempVARed(iR2, 0)+' '+ chGEOMSTempVARed(iR2, 1)+' '+ chGEOMSTempVARed(iR2, 2)+' '+ chGEOMSTempVARed(iR2, 3)
- end
-
- ;---------------------------------------------------------------------------
- ; check if search string found in template and count optional attributes
-
- ; var1
- chVarOpt1 = ''
- chSearch1 = '[SOLAR|LUNAR]'
- iPos1 = check_opt_var(chGEOMSTempVARed(*,0), chSearch1)
-
- if (iPos1 ge 0) then begin
- chVarOpt1 = get_optional_vars(chHDFVARed(*,0), chSearch1)
- printf, lu, ' INFORMATION: Variable option '+chSearch1+' found and analyzed.'
- end
-
- ; var2
- chVarOpt2 = ''
- chSearch2 = '[PRESSURE|TEMPERATURE]_INDEPENDENT_INITIALIZATION'
- iPos2 = check_opt_var(chGEOMSTempVARed(*,0), chSearch2)
-
- if (iPos2 ge 0) then begin
- chVarOpt2 = get_optional_vars(chHDFVARed(*,0), chSearch2)
- printf, lu, ' INFORMATION: Variable option '+chSearch2+' found and analyzed.'
- end
-
- ; go through all GEOMS-TE and replace [GAS] and optional variables e.g. [SOLAR|LUNAR]; create new array
- for iR2 = 0, iSize2 - 1 do begin
- ; replacement checks
- ; 1) check if [GAS] is reported, that data source [GAS] matches gases in VA
- chVarCheck = checkgas( chGEOMSTempVARed(iR2,0), '[GAS]', chHDFGas )
- ; 2) replace/substitute optional attributes
- STRREPLACE, chVarCheck, chSearch1, chVarOpt1
- STRREPLACE, chVarCheck, chSearch2, chVarOpt2
-
- chGEOMSTempVARedReplace(iR2,0) = chVarCheck
- chGEOMSTempVARedReplace(iR2,1) = chGEOMSTempVARed(iR2,1)
- chGEOMSTempVARedReplace(iR2,2) = chGEOMSTempVARed(iR2,2)
- chGEOMSTempVARedReplace(iR2,3) = chGEOMSTempVARed(iR2,3)
- chGEOMSTempVARedReplace(iR2,4) = chGEOMSTempVARed(iR2,4)
- end
-
- ;---------------------------------------------------------------------------
- ; check diffs GEOMS-TE -> HDF
- ; do limited check: check if variable is present; all other checks done by HDF -> GEOMS-TE
- iCheckTotal = 1
- for iR2 = 0, iSize2 - 1 do begin
- iCheck = 0
-
- for iR3 = 0, iSize3 - 1 do begin
- ; check for plain existence of data variable
- if ( chGEOMSTempVARedReplace(iR2,0) eq chHDFVARed(iR3, 0) ) then begin
- iCheck = 1
- iVarNr = iR3
- end
- end
-
- ; check if mandatory
- if (iCheck eq 0) then begin
- if ( chGEOMSTempVARed(iR2,3) eq 'o' ) then begin
- ;printf, lu, ' INFORMATION: '+chGEOMSTempVARed(iR2,0)+' optional data variable is not found in HDF file.'
- printf, lu, ' INFORMATION: '+chGEOMSTempVARedReplace(iR2,0)+' optional data variable is not found in HDF file.'
- iCheck = 1
- end
- end
-
- ; check if conditional
- if ( STRMID(chGEOMSTempVARed(iR2,3), 0, 4) eq 'x if' ) then begin
- chCondition = STRSPLIT(chGEOMSTempVARed(iR2,3), ' ',/EXTRACT)
-
-; ; replacement checks
-; ; 1) check if [GAS] is reported, that data source [GAS] matches gases in VA
- chConditionCheck = checkgas( chCondition(2), '[GAS]', chHDFGas )
-; ; 2) replace/substitute optional attributes
- STRREPLACE, chConditionCheck, chSearch1, chVarOpt1
- STRREPLACE, chConditionCheck, chSearch2, chVarOpt2
-
- ; check if variable condition found in variable list
- iPos = where(chConditionCheck eq chHDFVARed(*,0))
-
- ; DEBUG
- ;print, chCondition(2)+': '+chConditionCheck+': '+chHDFVARed(*,0)
-
- if ( ( chCondition(2) eq chHDFGas ) or ( iPos ge 0 ) ) then begin
- if (iCheck eq 0) then begin
-; print, ' ERROR: '+chGEOMSTempVARed(iR2,0)+' conditional data variable is not found in HDF file.'
- chError = ' ERROR: '+chGEOMSTempVARedReplace(iR2,0)+' conditional data variable is not found in HDF file, and required by template.'
- print, chError
- printf, lu, chError
- iCheck = 2 ;1 (changed v1.3)
- end
- end else if (iPos eq -1) and (iCheck eq 1) then begin
- ;printf, lu, ' ERROR: '+chGEOMSTempVARedReplace(iR2,0)+' conditional data variable is found in HDF file, but does not satisfy the condition in the template.'
- ;iCheck = 2 ;1 (changed v1.3)
- printf, lu, ' INFORMATION: '+chGEOMSTempVARedReplace(iR2,0)+' conditional data variable is found in HDF file, and treated as an optional variable.'
- iCheck = 1 ;(changed v1.34 due to GEOMS ruling allowing conditional variables to be treated as optional when they do not meet the condition)
- end else begin
- printf, lu, ' INFORMATION: '+chGEOMSTempVARedReplace(iR2,0)+' conditional data variable is not found in HDF file, and not required by template.'
- iCheck = 1
- end
- end
-
- ; check if variable exists
- if ( iCheck eq 0 ) then begin
- chError = ' ERROR: '+chGEOMSTempVARedReplace(iR2,0)+' data variable is not found in HDF file.'
- print, chError
- printf, lu, chError
- end else begin
- if iCheck eq 2 then iCheck = 0 ;added v1.3
- end
- ; set total error
- iCheckTotal *= iCheck
- end
-
- ;---------------------------------------------------------------------------
- ; check diffs HDF -> GEOMS-TE
- ;iCheckTotal = 0
- for iR3 = 0, iSize3 - 1 do begin
- iCheck = 0
-
- for iR2 = 0, iSize2 - 1 do begin
- ; replacement checks
- ; 1) check if [GAS] is reported, that data source [GAS] matches gases in VA
- chVarCheck = checkgas( chGEOMSTempVARed(iR2,0), '[GAS]', chHDFGas )
- ; 2) replace/substitute optional attributes
- STRREPLACE, chVarCheck, chSearch1, chVarOpt1
- STRREPLACE, chVarCheck, chSearch2, chVarOpt2
-
- ; check for plain existence of data variable
- if (chVarCheck eq chHDFVARed(iR3, 0)) then begin
- iCheck = 1
- iStor2 = iR2
- end
- end
-
- ; check if variable exists
- if ( iCheck eq 1 ) then begin
- ; check validity of units, data type and dependencies
-
- ; VAR_UNITS
- iVANr = 1
- STRREPLACE, chGEOMSTempVARed(iStor2,iVANr), '"', ''
- ; check for 'NONE'
- if ( chGEOMSTempVARed(iStor2,iVANr) eq "[empty]" ) then begin
- chGEOMSTempVARed(iStor2,iVANr) = ''
- end
-
- if ( chGEOMSTempVARed(iStor2,iVANr) ne STRTRIM(chHDFVARed(iR3,iVANr),2) ) then begin
- iCheck = 0
- ; check for optional elements [a|b]
- if ( STRMID( chGEOMSTempVARed(iStor2,iVANr), 0, 1 ) eq '[' ) then begin
- iCheck = find_optional_value1(chGEOMSTempVARed(iStor2,iVANr), chHDFVARed(iR3,iVANr))
- end
-
-; iCheckTotal *= iCheck
-
- ; check for optional elements '(or'
- if ( STRPOS( chGEOMSTempVARed(iStor2,iVANr), '(or' ) ge 0 ) then begin
- iCheck = find_optional_value2(chGEOMSTempVARed(iStor2,iVANr), chHDFVARed(iR3,iVANr))
- end
-
-; iCheckTotal *= iCheck
-
- if (iCheck eq 0) then begin
- chError = ' ERROR: '+STRING(chHDFVARed(iR3,0))+' has '+chVA(iVANr)+'='+STRING(chHDFVARed(iR3,iVANr))+', but '+chVA(iVANr)+'='+STRING(chGEOMSTempVARed(iStor2,iVANr))+' is required by template.'
- print, chError
- printf, lu, chError
- end
- end
- ; set total error
- iCheckTotal *= iCheck
-
- ; VAR_DATA_TYPE
- iVANr = 2
- if ( chGEOMSTempVARed(iStor2,iVANr) ne chHDFVARed(iR3,iVANr) ) then begin
- chError = ' ERROR: '+STRING(chHDFVARed(iR3,0))+' has '+chVA(iVANr)+'='+STRING(chHDFVARed(iR3,iVANr))+', but '+chVA(iVANr)+'='+STRING(chGEOMSTempVARed(iStor2,iVANr))+' is required by template.'
- print, chError
- printf, lu, chError
- iCheck = 0
- end
- ; set total error
- iCheckTotal *= iCheck
-
- ; VAR_DEPEND
- iVANr = 3
-
- ; check if VAR_DEPEND has options
- chVARDEPENDTemp_arr = find_optional_value3( chGEOMSTempVARed(iStor2,iVANr + 1) )
-
- iSize9 = SIZE(chVARDEPENDTemp_arr, /N_ELEMENTS)
-
- iCheck = 0
- iC9 = 0
-; while ((iC9 < iSize9 - 1) and (iCheck ne 1)) do begin
- for iR9 = iSize9 - 1, 0, -1 do begin
-
- ;chVARDEPENDTemp_h = STRSPLIT(chGEOMSTempVARed(iStor2,iVANr + 1), ';',/EXTRACT)
- chVARDEPENDTemp_h = chVARDEPENDTemp_arr(iC9)
-
- ; transpose template VAR_DEPEND
- chVARDEPENDTemp_check = ''
- iSize8 = SIZE(chVARDEPENDTemp_h, /N_ELEMENTS)
-
- chVARDEPENDHDF_h = STRSPLIT(chHDFVARed(iR3,iVANr), ';',/EXTRACT)
- chVARDEPENDHDF_check = ''
- iSize7 = SIZE(chVARDEPENDHDF_h, /N_ELEMENTS)
-
- for iR8 = 0, iSize8 - 1 do begin
- chVARDEPENDTemp_check += strtrim(chVARDEPENDTemp_h(iR8))
-
- if (( iSize8 gt 0 ) and ( iR8 lt iSize8 - 1 )) then begin
- chVARDEPENDTemp_check += ';'
- end
- end
-
- for iR7 = iSize7 - 1, 0, -1 do begin
- chVARDEPENDHDF_check += strtrim(chVARDEPENDHDF_h(iR7))
-
- if (( iSize7 gt 0 ) and ( iR7 gt 0 )) then begin
- chVARDEPENDHDF_check += ';'
- end
- end
-
- if (chVARDEPENDTemp_check eq chVARDEPENDHDF_check) then begin
- iCheck = 1
- break
- end else begin
- iCheck = 0
- end
-
- iC9 += 1
- end
- ; set total error
- iCheckTotal *= iCheck
-
- ; DEBUG
- ;print, string(iR3)+': '+chVARDEPENDTemp_check+' -> '+chVARDEPENDHDF_check
- if (iCheck eq 0) then begin
- chError = ' ERROR: '+STRING(chHDFVARed(iR3,0))+' has '+chVA(iVANr)+'='+chVARDEPENDHDF_check+', but '+chVA(iVANr)+'='+chGEOMSTempVARed(iStor2,iVANr + 1)+' is required by template.'
- print, chError
- printf, lu, chError
- iCheck = 0
- end
-
- end else begin
- chError = ' ERROR: '+STRING(chHDFVARed(iR3,0))+' data variable is not found in template.'
- print, chError
- printf, lu, chError
- end
-
- iCheckTotal *= iCheck
-
- end
-
- if ( iCheckTotal eq 1 ) then begin
- iCheckOut = 0
- end else begin
- iCheckOut = 1
- end
-
- printf, lu, ''
- printf, lu, tccode(iCheckOut)
-
- return, iCheckOut
-END
-
-
-;---------------------------------------------------------------------------------------------------
-PRO analyze_GEOMS_GA
-
- ; The reading of the tempate CSV files strongly depends on its format. Currently only
- ; DATA_VARIABLES is allowed to have empty first columns
-
- ; not used, incomplete
-
- repeat BEGIN
- readf, lu, chTempLine
-
- chTempLine1 = STRSPLIT(chTempLine, ',', ESCAPE=' ',/EXTRACT)
-
- if (size(chTempLine1, /N_ELEMENTS) eq 1) then begin
- chTempDATAVARDum(iC1) = chTempLine1(0)
- iC1 += 1
- end else begin
- chTempGADum(iC2) = chTempLine1(1)
- iC2 += 1
- end
- ENDREP until eof(lu)
-
- chTempDATAVAR=strarr(iC1)
- chTempGA=strarr(iC2)
-
- print, iC1, iC2
-
- chTempDATAVAR(*) = chTempDATAVARDum(0:iC1 - 1)
- chTempGA(*) = chTempGADum(0:iC2 - 1)
-
- for iR1 = 0, size(chTempDATAVAR, /N_ELEMENTS) - 1 do begin
- print, iR1, chTempDATAVAR(iR1)
- end
-
- for iR1 = 0, size(chTempGADum, /N_ELEMENTS) - 1 do begin
- print, iR1, chTempGA(iR1)
- end
-END
-
-;---------------------------------------------------------------------------------------------------
-FUNCTION geoms_tctools, templatefile, hdffile, QA_error_code
-
- reterr=''
- idlcr8ascii, hdffile, ga, sds, reterr
-
- chVN_value = check_for_version_name( ga ) ;check for existence of version name in DATA_SOURCE
-
- ; find log file (see geoms_tools.pro)
- logfile=STRMID(hdffile,0,STRPOS(hdffile,'.',/REVERSE_SEARCH))+'.log'
-
- openw,lu,logfile,/GET_LUN,/APPEND
-
- chGEOMSTemp = read_GEOMS_template( templatefile )
- chMessage = 'Analyzing compliance to GEOMS template: '+FILE_BASENAME(templatefile)
- print, chMessage
- printf, lu, ''
- printf, lu, chMessage
-
- TC_error_code = analyze_GEOMS_VA( ga, sds, chGEOMSTemp, lu )
-
- if chVN_value eq '' then begin ;print out QA/TC result because no need to do version name check
- chMessage = STRTRIM(STRING( QA_error_code + 3*TC_error_code + 4 ), 1)+' (Total QA/TC error)'
- printf, lu, ''
- printf, lu, chMessage
- Free_Lun, lu
- endif else Free_Lun, lu
-
- return, TC_error_code
-END
+;---------------------------------------------------------------------------------------------------
+;
+; name: geoms_tctools.pro
+; description: AVDC GEOMS template checker tool
+;
+; 2011-02-03, v0.1 Christian Retscher initial implementation
+; 2011-04-11, v0.2 Christian Retscher MWR support addded
+; concept: 1) GEOMS-VA -> HDF; 2) HDF -> GEOMS-VA checks
+; 2011-04-17, v0.3 Christian Retscher FTIR support start
+; filter for [SOLAR|LUNAR], [ppmv|ppbv|pptv], "...()"
+; 2011-04-23, v0.4 Christian Retscher FTIR support finalized, LIDAR support tested
+; 2011-04-25, v0.5 Christian Retscher Full LIDAR support added
+; 2011-04-26, v0.6 Christian Retscher Split in wrapper and TC tools (keep versioning)
+; 2011-05-13, v0.7 Christian Retscher Integrate UVVIS.DOAS
+; 2011-05-26, v0.8 Christian Retscher check VAR_DEPEND
+; 2011-09-07, v0.9 Christian Retscher adapted filter for FTIR "(or a unit scaled"
+; 2012-04-25, v1.0 Ian Boyd fixed bug when checking for conditional mandatory variables by
+; adding extra checks; for optional checks add actual variable name rather
+; than template name e.g. [GAS] replaced by O3
+; 2013-12-19, v1.3 Ian Boyd fixed bug when checking the template against the file - errors
+; were being generated but the iCheckTotal was not being assigned correctly
+; meaning that final result was showing that template was OK
+; 2014-01-28, v1.34 Ian Boyd GEOMS group introduced rule allowing for conditional variables
+; to be treated as optional when they do not otherwise meet the condition -
+; previously this generated an error
+; 2015-02-17, v1.43 Ian Boyd When checking VAR_UNITS, strip away any white space from the
+; VAR_UNITS value
+; 2016-11-30, v1.57 Ian Boyd Account for non-GEOMS variable attributes (ptr_valid check)
+; 2019-12-05, v2.00 Ian Boyd Add check_for_version_name function to check for optional Version
+; name field in DATA_SOURCE. This determines whether to write out an end of
+; check message
+; 2024-10-29, v2.16 Ian Boyd Fix bug for when there are more than one index values calculated
+; from WHERE statements that only expect one value to be returned (e.g. iPos)
+; --------------------------------------------------------------------------------------------------
+
+pro STRREPLACE, Strings, Find1, Replacement1
+
+; Check integrity of input parameter
+
+ NP = N_PARAMS()
+ if (NP ne 3) then message,'Must be called with 3 parameters, '+$
+ 'Strings, Find, Replacement'
+
+ sz = SIZE(Strings)
+ ns = n_elements(sz)
+ if (sz(ns-2) ne 7) then message,'Parameter must be of string type.'
+
+ Find = STRING(Find1)
+ pos = STRPOS(Strings,Find)
+ here = WHERE(pos ne -1, nreplace)
+
+ if (nreplace eq 0) then return
+
+ Replacement=STRING(Replacement1)
+ Flen = strlen(Find)
+ for i=0,nreplace-1 do begin
+
+ j = here(i)
+ prefix = STRMID(Strings(j),0,pos(j))
+ suffix = STRMID(Strings(j),pos(j)+Flen,$
+ strlen(Strings(j))-(pos(j)+Flen))
+ Strings(j) = prefix + replacement + suffix
+ endfor
+end
+
+
+;---------------------------------------------------------------------------------------------------
+FUNCTION read_GEOMS_template, templatefile
+
+ ; read template file
+ openr, lu, templatefile, /GET_LUN
+
+ iSizeDum1 = 10000
+ chGEOMSTempDum = strarr(iSizeDum1)
+
+ iC1=0
+ chTempLine=''
+ repeat BEGIN
+ readf, lu, chTempLine
+ chGEOMSTempDum( iC1 ) = chTempLine
+ iC1 += 1
+ ENDREP until eof(lu)
+
+ ;resize input file array
+ chGEOMSTemp = strarr(iC1)
+ chGEOMSTemp(*) = chGEOMSTempDum(0:iC1-1)
+
+ close,lu
+
+ return, chGEOMSTemp
+END
+;---------------------------------------------------------------------------------------------------
+FUNCTION check_for_version_name, ga
+;determine if DATA_SOURCE has a VERSION_NAME field
+
+ ga=STRTRIM(ga,2)
+ dsi = WHERE(STRMID(STRUPCASE(ga),0,11) eq 'DATA_SOURCE',dscnt)
+ if dscnt eq 1 then begin
+ dsv = strmid(ga[dsi[0]], strpos(ga[dsi[0]],'=')+1) ;everything after the '=' sign
+ dsx = strsplit(strtrim(dsv,2),'_',/Extract,count=cdsx)
+ if cdsx eq 3 then return, dsx[2] else return, ''
+ endif else return, ''
+END
+
+
+;---------------------------------------------------------------------------------------------------
+FUNCTION checkgas, chVarCheck, chSearch, chHDFGas
+
+ if (strpos(chVarCheck, chSearch) ge 0) then begin
+ STRREPLACE, chVarCheck, chSearch, chHDFGas
+ end
+
+ return, chVarCheck
+END
+
+;---------------------------------------------------------------------------------------------------
+FUNCTION check_data_source, ga
+ ; get GA and extract DATA_SOURCE
+ for iR1 = 0, size(ga, /N_ELEMENTS) - 1 do begin
+ chGATmp = strsplit(ga(iR1), '=', /EXTRACT)
+
+ if (chGATmp(0) eq 'DATA_SOURCE') then begin
+ chHDFDS = strsplit(chGATmp(1), ';', /EXTRACT)
+ end
+ end
+
+ ; extract name and gas of data source
+ chHDFDSDum1 = strsplit(chHDFDS, '_', /EXTRACT)
+ chHDFDSDum = chHDFDSDum1
+
+ ; check if data source contains '.', then split
+ if ( strpos( chHDFDSDum1(0), '.' ) ge 0 ) then begin
+ chHDFDSDum2 = strsplit(chHDFDSDum1(0), '.', /EXTRACT)
+ chHDFDSDum = chHDFDSDum2
+ end
+
+ iSize1 = size( chHDFDSDum, /N_ELEMENTS )
+
+ chHDFName = chHDFDSDum(0)
+ chHDFGas = chHDFDSDum(iSize1 - 1)
+
+ return, chHDFGas
+end
+
+;---------------------------------------------------------------------------------------------------
+FUNCTION find_optional_value1, chGEOMSTempVAVal, chHDFVAVal
+ chDelim2 = '|'
+ chVarOptTmp1 = chGEOMSTempVAVal
+
+ ; remove []
+ STRREPLACE, chVarOptTmp1, '[', ''
+ STRREPLACE, chVarOptTmp1, ']', ''
+
+ chGEOMSTempVAValArr = strsplit(chVarOptTmp1, chDelim2, /EXTRACT)
+
+ iPos = where( chGEOMSTempVAValArr eq chHDFVAVal)
+
+ if ( iPos[0] ge 0 ) then begin
+ iOutput = 1
+ end else begin
+ iOutput = 0
+ end
+
+ return, iOutput
+end
+
+;---------------------------------------------------------------------------------------------------
+FUNCTION find_optional_value2, chGEOMSTempVAVal, chHDFVAVal
+ chDelim2 = '('
+ chVarOptTmp1 = chGEOMSTempVAVal
+
+ ; remove []
+ STRREPLACE, chVarOptTmp1, '"', ''
+
+ chGEOMSTempVAValArr = strsplit(chVarOptTmp1, chDelim2, /EXTRACT)
+
+; iPos = where( STRTRIM(chGEOMSTempVAValArr(0)) eq STRTRIM(chHDFVAVal))
+
+ ; split first letter
+ chHDFVAVal2 = strmid(STRTRIM(chHDFVAVal), 1, strlen(STRTRIM(chHDFVAVal)))
+
+ iPos1 = where(STRTRIM(chGEOMSTempVAValArr(0)) eq STRTRIM(chHDFVAVal2))
+ iPos2 = where(STRTRIM(chGEOMSTempVAValArr(0)) eq STRTRIM(chHDFVAVal))
+
+ if ((iPos1[0] ge 0) or (iPos2[0] ge 0)) then begin
+ iOutput = 1
+ end else begin
+ iOutput = 0
+ end
+
+ return, iOutput
+end
+
+;---------------------------------------------------------------------------------------------------
+FUNCTION find_optional_value3, chGEOMSTempVAVal
+ chDelim2 = '|'
+ chVarOptTmp1 = chGEOMSTempVAVal
+
+ ; remove []
+ STRREPLACE, chVarOptTmp1, '[', ''
+ STRREPLACE, chVarOptTmp1, ']', ''
+
+ chGEOMSTempVAValArr = strsplit(chVarOptTmp1, chDelim2, /EXTRACT)
+
+ return, chGEOMSTempVAValArr
+end
+
+;---------------------------------------------------------------------------------------------------
+FUNCTION check_opt_var, chGEOMSTempVARed, chSearch
+
+ iSize = size(chGEOMSTempVARed, /N_ELEMENTS)
+
+ iPos = -1
+ for iR2 = 0, iSize - 1 do begin
+ iPosTmp = strpos(chGEOMSTempVARed(iR2), chSearch)
+ if (iPosTmp ge 0) then begin
+ iPos = iPosTmp
+ end
+ end
+
+ return, iPos
+end
+
+;---------------------------------------------------------------------------------------------------
+FUNCTION get_optional_vars, chHDFVARedName, chVarOpt
+ chDelim1 = '.'
+ chDelim2 = '_'
+ chDelim3 = '|'
+ chDelim4 = '['
+ chDelim5 = ']'
+
+ chVarOptTmp1 = chVarOpt
+
+ ; split after []
+ chGEOMSTempVAEle = strsplit(chVarOptTmp1, chDelim5, /EXTRACT)
+
+ chVarOptTmp1 = chGEOMSTempVAEle(0)
+
+ ; remove []
+ STRREPLACE, chVarOptTmp1, chDelim1, ''
+ STRREPLACE, chVarOptTmp1, chDelim4, ''
+ STRREPLACE, chVarOptTmp1, chDelim5, ''
+
+ chGEOMSTempVAOpt = strsplit(chVarOptTmp1, chDelim3, /EXTRACT)
+
+ ; size of HDF variables
+ iSize3 = size(chHDFVARedName(*,0), /N_ELEMENTS)
+ ; size of HDF variable options
+ iSize6 = size(chGEOMSTempVAOpt, /N_ELEMENTS)
+ ; size of HDF variable split
+ iSize7 = size(chGEOMSTempVAEle, /N_ELEMENTS)
+
+ chGEOMSTempVAOptCount = INTARR(iSize6)
+
+ for iR6 = 0, iSize6 - 1 do begin
+ iC6 = 0
+
+ for iR3 = 0, iSize3 - 1 do begin
+ ;print, chHDFVARedName( iR3, 0 )
+
+ if (iSize7 gt 1 ) then begin
+ iFound = strpos(chHDFVARedName(iR3, 0), chGEOMSTempVAOpt(iR6)+chGEOMSTempVAEle(1))
+ end else begin
+ iFound = strpos(chHDFVARedName(iR3, 0), chDelim1+chGEOMSTempVAOpt(iR6))
+ end
+
+ if ( iFound ge 0 ) then begin
+ iC6+=1
+ end
+ end
+
+ chGEOMSTempVAOptCount(iR6) = iC6
+ end
+
+ iPosMax = where( chGEOMSTempVAOptCount eq max(chGEOMSTempVAOptCount))
+
+ chGEOMSTempVAOptOut = chGEOMSTempVAOpt(iPosMax(0))
+
+ if (iSize7 gt 1 ) then begin
+ chGEOMSTempVAOptOut = chGEOMSTempVAOptOut+chGEOMSTempVAEle(1)
+ end
+
+ return, chGEOMSTempVAOptOut
+end
+
+;---------------------------------------------------------------------------------------------------
+FUNCTION analyze_GEOMS_VA, ga, sds, chGEOMSTempVA, lu
+
+ tccode = ['0 (Passed template checks)', '1 (Failed template checks)']
+
+ ; get [GAS] form data source
+ chHDFGas = check_data_source( ga )
+
+ ; size of GEOMS-TE VA
+ iSize1 = size(STRSPLIT(chGEOMSTempVA(0), ',',/EXTRACT), /N_ELEMENTS)
+ ; size of GEOMS-TE VA
+ iSize2 = size(chGEOMSTempVA, /N_ELEMENTS)
+ ; size of HDF variables
+ iSize3 = size(sds(*,0), /N_ELEMENTS )
+ ; size of HDF VA
+ iSize4 = size(sds(0,*), /N_ELEMENTS )
+ ; size of predefined VA
+ iSize5 = iSize1 - 3 ;size(chVA, /N_ELEMENTS )
+
+ ; decide on type of data template
+ if (iSize1 eq 5) then begin
+ chVA = [ 'VAR_NAME', 'VAR_UNITS' ]
+ end
+ if (iSize1 eq 6) then begin
+ chVA = [ 'VAR_NAME', 'VAR_UNITS', 'VAR_DATA_TYPE' ]
+ end
+ if (iSize1 eq 7) then begin
+ chVA = [ 'VAR_NAME', 'VAR_UNITS', 'VAR_DATA_TYPE', 'VAR_DEPEND' ]
+ end
+
+ ; 4 elements read from HDF file, max 5 elements read from template
+ chHDFVARed = strarr(iSize3,4)
+ chGEOMSTempVARed = strarr(iSize2,5)
+ chGEOMSTempVARedReplace = strarr(iSize2,5)
+
+ ; read HDF VA and assign values
+ ; dynamic check on list of predefined variables attributes chVA
+ for iR3 = 0, iSize3 - 1 do begin
+ ; loop trough lookup VA
+ for iR5 = 0, iSize5 - 1 do begin
+ ; loop trough all HDF VA
+; for iR4 = 0, iSize4 - 1 do begin
+; ; look for matching VA values
+; print, *sds(iR3,iR4).va_l+', '+*sds(0,iR4).va_l+', '+chVA(iR5)
+; if (*sds(iR3,iR4).va_l eq chVA(iR5)) then begin
+; ; assign variable attributes
+; chHDFVARed(iR3,iR5) = *sds(iR3,iR4).va_v
+;
+; print, iR3,iR5, iR4, chHDFVARed(iR3,iR5)
+; end
+
+ ;ptr_valid check added v1.57
+ iC4 = 0 & goodexit = 0
+ while goodexit eq 0 do begin
+ if ptr_valid(sds(iR3,iC4).va_l) then begin
+ if *sds(iR3,iC4).va_l eq chVA(iR5) then goodexit = 1 else iC4++
+ endif else goodexit = -1 ;non-valid entry
+ end
+
+ if goodexit eq 1 then chHDFVARed(iR3,iR5) = *sds(iR3,iC4).va_v $
+ else if iR5 eq 0 then chHDFVARed(iR3,iR5) = 'Invalid/Unknown' ;*sds(iR3,0).va_v
+ end
+ ;print, STRING(iR3)+','+chHDFVARed(iR3, 0)+','+ chHDFVARed(iR3, 1)+','+ chHDFVARed(iR3, 2)+','+ chHDFVARed(iR3, 3)
+ end
+
+ ; read GEOMS-TE VA and assign values
+ ; The reading of the tempate CSV files strongly depends on its format.
+ for iR2 = 0, iSize2 - 1 do begin
+ chGEOMSTempVATmp = strsplit(chGEOMSTempVA(iR2), ',', /EXTRACT)
+
+ ; template variable name
+ chGEOMSTempVARed(iR2,0) = strtrim(chGEOMSTempVATmp(1))
+
+ ; LIDAR 2011-04-23
+ if ( iSize1 ge 5 ) then begin
+ ; template variable units
+ chGEOMSTempVARed(iR2,1) = strtrim(chGEOMSTempVATmp(2))
+ ; template variable requirement
+ chGEOMSTempVARed(iR2,3) = strtrim(chGEOMSTempVATmp(size(chGEOMSTempVATmp, /N_ELEMENTS) - 1))
+ end
+
+ ; MWR, FTIR 2011-04-23
+ if ( iSize1 ge 6 ) then begin
+ ; template variable units
+ chGEOMSTempVARed(iR2,1) = strtrim(chGEOMSTempVATmp(2))
+ ; template variable data type
+ chGEOMSTempVARed(iR2,2) = strtrim(chGEOMSTempVATmp(3))
+ ; template variable requirement
+ chGEOMSTempVARed(iR2,3) = strtrim(chGEOMSTempVATmp(size(chGEOMSTempVATmp, /N_ELEMENTS) - 1))
+ end
+
+ ; FTIR 2011-05-18
+ if ( iSize1 ge 7 ) then begin
+ ; template variable units
+ chGEOMSTempVARed(iR2,1) = strtrim(chGEOMSTempVATmp(3))
+ ; template variable data type
+ chGEOMSTempVARed(iR2,2) = strtrim(chGEOMSTempVATmp(4))
+ ; template variable requirement
+ chGEOMSTempVARed(iR2,3) = strtrim(chGEOMSTempVATmp(size(chGEOMSTempVATmp, /N_ELEMENTS) - 1))
+ ; template variable dependencies (dimensions)
+ chGEOMSTempVARed(iR2,4) = strtrim(chGEOMSTempVATmp(2))
+ end
+
+ ; template variable dependencies
+ ;chGEOMSTempVARed(iR2,1) = strtrim(chGEOMSTempVATmp(size(chGEOMSTempVATmp, /N_ELEMENTS) - 1))
+
+ ;print, STRING(iR2)+' '+chGEOMSTempVARed(iR2, 0)+' '+ chGEOMSTempVARed(iR2, 1)+' '+ chGEOMSTempVARed(iR2, 2)+' '+ chGEOMSTempVARed(iR2, 3)
+ end
+
+ ;---------------------------------------------------------------------------
+ ; check if search string found in template and count optional attributes
+
+ ; var1
+ chVarOpt1 = ''
+ chSearch1 = '[SOLAR|LUNAR]'
+ iPos1 = check_opt_var(chGEOMSTempVARed(*,0), chSearch1)
+
+ if (iPos1 ge 0) then begin
+ chVarOpt1 = get_optional_vars(chHDFVARed(*,0), chSearch1)
+ printf, lu, ' INFORMATION: Variable option '+chSearch1+' found and analyzed.'
+ end
+
+ ; var2
+ chVarOpt2 = ''
+ chSearch2 = '[PRESSURE|TEMPERATURE]_INDEPENDENT_INITIALIZATION'
+ iPos2 = check_opt_var(chGEOMSTempVARed(*,0), chSearch2)
+
+ if (iPos2 ge 0) then begin
+ chVarOpt2 = get_optional_vars(chHDFVARed(*,0), chSearch2)
+ printf, lu, ' INFORMATION: Variable option '+chSearch2+' found and analyzed.'
+ end
+
+ ; go through all GEOMS-TE and replace [GAS] and optional variables e.g. [SOLAR|LUNAR]; create new array
+ for iR2 = 0, iSize2 - 1 do begin
+ ; replacement checks
+ ; 1) check if [GAS] is reported, that data source [GAS] matches gases in VA
+ chVarCheck = checkgas( chGEOMSTempVARed(iR2,0), '[GAS]', chHDFGas )
+ ; 2) replace/substitute optional attributes
+ STRREPLACE, chVarCheck, chSearch1, chVarOpt1
+ STRREPLACE, chVarCheck, chSearch2, chVarOpt2
+
+ chGEOMSTempVARedReplace(iR2,0) = chVarCheck
+ chGEOMSTempVARedReplace(iR2,1) = chGEOMSTempVARed(iR2,1)
+ chGEOMSTempVARedReplace(iR2,2) = chGEOMSTempVARed(iR2,2)
+ chGEOMSTempVARedReplace(iR2,3) = chGEOMSTempVARed(iR2,3)
+ chGEOMSTempVARedReplace(iR2,4) = chGEOMSTempVARed(iR2,4)
+ end
+
+ ;---------------------------------------------------------------------------
+ ; check diffs GEOMS-TE -> HDF
+ ; do limited check: check if variable is present; all other checks done by HDF -> GEOMS-TE
+ iCheckTotal = 1
+ for iR2 = 0, iSize2 - 1 do begin
+ iCheck = 0
+
+ for iR3 = 0, iSize3 - 1 do begin
+ ; check for plain existence of data variable
+ if ( chGEOMSTempVARedReplace(iR2,0) eq chHDFVARed(iR3, 0) ) then begin
+ iCheck = 1
+ iVarNr = iR3
+ end
+ end
+
+ ; check if mandatory
+ if (iCheck eq 0) then begin
+ if ( chGEOMSTempVARed(iR2,3) eq 'o' ) then begin
+ ;printf, lu, ' INFORMATION: '+chGEOMSTempVARed(iR2,0)+' optional data variable is not found in HDF file.'
+ printf, lu, ' INFORMATION: '+chGEOMSTempVARedReplace(iR2,0)+' optional data variable is not found in HDF file.'
+ iCheck = 1
+ end
+ end
+
+ ; check if conditional
+ if ( STRMID(chGEOMSTempVARed(iR2,3), 0, 4) eq 'x if' ) then begin
+ chCondition = STRSPLIT(chGEOMSTempVARed(iR2,3), ' ',/EXTRACT)
+
+; ; replacement checks
+; ; 1) check if [GAS] is reported, that data source [GAS] matches gases in VA
+ chConditionCheck = checkgas( chCondition(2), '[GAS]', chHDFGas )
+; ; 2) replace/substitute optional attributes
+ STRREPLACE, chConditionCheck, chSearch1, chVarOpt1
+ STRREPLACE, chConditionCheck, chSearch2, chVarOpt2
+
+ ; check if variable condition found in variable list
+ iPos = where(chConditionCheck eq chHDFVARed(*,0))
+
+ ; DEBUG
+ ;print, chCondition(2)+': '+chConditionCheck+': '+chHDFVARed(*,0)
+
+ if ( ( chCondition(2) eq chHDFGas ) or ( iPos[0] ge 0 ) ) then begin
+ if (iCheck eq 0) then begin
+; print, ' ERROR: '+chGEOMSTempVARed(iR2,0)+' conditional data variable is not found in HDF file.'
+ chError = ' ERROR: '+chGEOMSTempVARedReplace(iR2,0)+' conditional data variable is not found in HDF file, and required by template.'
+ print, chError
+ printf, lu, chError
+ iCheck = 2 ;1 (changed v1.3)
+ end
+ end else if (iPos[0] eq -1) and (iCheck eq 1) then begin
+ ;printf, lu, ' ERROR: '+chGEOMSTempVARedReplace(iR2,0)+' conditional data variable is found in HDF file, but does not satisfy the condition in the template.'
+ ;iCheck = 2 ;1 (changed v1.3)
+ printf, lu, ' INFORMATION: '+chGEOMSTempVARedReplace(iR2,0)+' conditional data variable is found in HDF file, and treated as an optional variable.'
+ iCheck = 1 ;(changed v1.34 due to GEOMS ruling allowing conditional variables to be treated as optional when they do not meet the condition)
+ end else begin
+ printf, lu, ' INFORMATION: '+chGEOMSTempVARedReplace(iR2,0)+' conditional data variable is not found in HDF file, and not required by template.'
+ iCheck = 1
+ end
+ end
+
+ ; check if variable exists
+ if ( iCheck eq 0 ) then begin
+ chError = ' ERROR: '+chGEOMSTempVARedReplace(iR2,0)+' data variable is not found in HDF file.'
+ print, chError
+ printf, lu, chError
+ end else begin
+ if iCheck eq 2 then iCheck = 0 ;added v1.3
+ end
+ ; set total error
+ iCheckTotal *= iCheck
+ end
+
+ ;---------------------------------------------------------------------------
+ ; check diffs HDF -> GEOMS-TE
+ ;iCheckTotal = 0
+ for iR3 = 0, iSize3 - 1 do begin
+ iCheck = 0
+
+ for iR2 = 0, iSize2 - 1 do begin
+ ; replacement checks
+ ; 1) check if [GAS] is reported, that data source [GAS] matches gases in VA
+ chVarCheck = checkgas( chGEOMSTempVARed(iR2,0), '[GAS]', chHDFGas )
+ ; 2) replace/substitute optional attributes
+ STRREPLACE, chVarCheck, chSearch1, chVarOpt1
+ STRREPLACE, chVarCheck, chSearch2, chVarOpt2
+
+ ; check for plain existence of data variable
+ if (chVarCheck eq chHDFVARed(iR3, 0)) then begin
+ iCheck = 1
+ iStor2 = iR2
+ end
+ end
+
+ ; check if variable exists
+ if ( iCheck eq 1 ) then begin
+ ; check validity of units, data type and dependencies
+
+ ; VAR_UNITS
+ iVANr = 1
+ STRREPLACE, chGEOMSTempVARed(iStor2,iVANr), '"', ''
+ ; check for 'NONE'
+ if ( chGEOMSTempVARed(iStor2,iVANr) eq "[empty]" ) then begin
+ chGEOMSTempVARed(iStor2,iVANr) = ''
+ end
+
+ if ( chGEOMSTempVARed(iStor2,iVANr) ne STRTRIM(chHDFVARed(iR3,iVANr),2) ) then begin
+ iCheck = 0
+ ; check for optional elements [a|b]
+ if ( STRMID( chGEOMSTempVARed(iStor2,iVANr), 0, 1 ) eq '[' ) then begin
+ iCheck = find_optional_value1(chGEOMSTempVARed(iStor2,iVANr), chHDFVARed(iR3,iVANr))
+ end
+
+; iCheckTotal *= iCheck
+
+ ; check for optional elements '(or'
+ if ( STRPOS( chGEOMSTempVARed(iStor2,iVANr), '(or' ) ge 0 ) then begin
+ iCheck = find_optional_value2(chGEOMSTempVARed(iStor2,iVANr), chHDFVARed(iR3,iVANr))
+ end
+
+; iCheckTotal *= iCheck
+
+ if (iCheck eq 0) then begin
+ chError = ' ERROR: '+STRING(chHDFVARed(iR3,0))+' has '+chVA(iVANr)+'='+STRING(chHDFVARed(iR3,iVANr))+', but '+chVA(iVANr)+'='+STRING(chGEOMSTempVARed(iStor2,iVANr))+' is required by template.'
+ print, chError
+ printf, lu, chError
+ end
+ end
+ ; set total error
+ iCheckTotal *= iCheck
+
+ ; VAR_DATA_TYPE
+ iVANr = 2
+ if ( chGEOMSTempVARed(iStor2,iVANr) ne chHDFVARed(iR3,iVANr) ) then begin
+ chError = ' ERROR: '+STRING(chHDFVARed(iR3,0))+' has '+chVA(iVANr)+'='+STRING(chHDFVARed(iR3,iVANr))+', but '+chVA(iVANr)+'='+STRING(chGEOMSTempVARed(iStor2,iVANr))+' is required by template.'
+ print, chError
+ printf, lu, chError
+ iCheck = 0
+ end
+ ; set total error
+ iCheckTotal *= iCheck
+
+ ; VAR_DEPEND
+ iVANr = 3
+
+ ; check if VAR_DEPEND has options
+ chVARDEPENDTemp_arr = find_optional_value3( chGEOMSTempVARed(iStor2,iVANr + 1) )
+
+ iSize9 = SIZE(chVARDEPENDTemp_arr, /N_ELEMENTS)
+
+ iCheck = 0
+ iC9 = 0
+; while ((iC9 < iSize9 - 1) and (iCheck ne 1)) do begin
+ for iR9 = iSize9 - 1, 0, -1 do begin
+
+ ;chVARDEPENDTemp_h = STRSPLIT(chGEOMSTempVARed(iStor2,iVANr + 1), ';',/EXTRACT)
+ chVARDEPENDTemp_h = chVARDEPENDTemp_arr(iC9)
+
+ ; transpose template VAR_DEPEND
+ chVARDEPENDTemp_check = ''
+ iSize8 = SIZE(chVARDEPENDTemp_h, /N_ELEMENTS)
+
+ chVARDEPENDHDF_h = STRSPLIT(chHDFVARed(iR3,iVANr), ';',/EXTRACT)
+ chVARDEPENDHDF_check = ''
+ iSize7 = SIZE(chVARDEPENDHDF_h, /N_ELEMENTS)
+
+ for iR8 = 0, iSize8 - 1 do begin
+ chVARDEPENDTemp_check += strtrim(chVARDEPENDTemp_h(iR8))
+
+ if (( iSize8 gt 0 ) and ( iR8 lt iSize8 - 1 )) then begin
+ chVARDEPENDTemp_check += ';'
+ end
+ end
+
+ for iR7 = iSize7 - 1, 0, -1 do begin
+ chVARDEPENDHDF_check += strtrim(chVARDEPENDHDF_h(iR7))
+
+ if (( iSize7 gt 0 ) and ( iR7 gt 0 )) then begin
+ chVARDEPENDHDF_check += ';'
+ end
+ end
+
+ if (chVARDEPENDTemp_check eq chVARDEPENDHDF_check) then begin
+ iCheck = 1
+ break
+ end else begin
+ iCheck = 0
+ end
+
+ iC9 += 1
+ end
+ ; set total error
+ iCheckTotal *= iCheck
+
+ ; DEBUG
+ ;print, string(iR3)+': '+chVARDEPENDTemp_check+' -> '+chVARDEPENDHDF_check
+ if (iCheck eq 0) then begin
+ chError = ' ERROR: '+STRING(chHDFVARed(iR3,0))+' has '+chVA(iVANr)+'='+chVARDEPENDHDF_check+', but '+chVA(iVANr)+'='+chGEOMSTempVARed(iStor2,iVANr + 1)+' is required by template.'
+ print, chError
+ printf, lu, chError
+ iCheck = 0
+ end
+
+ end else begin
+ chError = ' ERROR: '+STRING(chHDFVARed(iR3,0))+' data variable is not found in template.'
+ print, chError
+ printf, lu, chError
+ end
+
+ iCheckTotal *= iCheck
+
+ end
+
+ if ( iCheckTotal eq 1 ) then begin
+ iCheckOut = 0
+ end else begin
+ iCheckOut = 1
+ end
+
+ printf, lu, ''
+ printf, lu, tccode(iCheckOut)
+
+ return, iCheckOut
+END
+
+
+;---------------------------------------------------------------------------------------------------
+PRO analyze_GEOMS_GA
+
+ ; The reading of the tempate CSV files strongly depends on its format. Currently only
+ ; DATA_VARIABLES is allowed to have empty first columns
+
+ ; not used, incomplete
+
+ repeat BEGIN
+ readf, lu, chTempLine
+
+ chTempLine1 = STRSPLIT(chTempLine, ',', ESCAPE=' ',/EXTRACT)
+
+ if (size(chTempLine1, /N_ELEMENTS) eq 1) then begin
+ chTempDATAVARDum(iC1) = chTempLine1(0)
+ iC1 += 1
+ end else begin
+ chTempGADum(iC2) = chTempLine1(1)
+ iC2 += 1
+ end
+ ENDREP until eof(lu)
+
+ chTempDATAVAR=strarr(iC1)
+ chTempGA=strarr(iC2)
+
+ print, iC1, iC2
+
+ chTempDATAVAR(*) = chTempDATAVARDum(0:iC1 - 1)
+ chTempGA(*) = chTempGADum(0:iC2 - 1)
+
+ for iR1 = 0, size(chTempDATAVAR, /N_ELEMENTS) - 1 do begin
+ print, iR1, chTempDATAVAR(iR1)
+ end
+
+ for iR1 = 0, size(chTempGADum, /N_ELEMENTS) - 1 do begin
+ print, iR1, chTempGA(iR1)
+ end
+END
+
+;---------------------------------------------------------------------------------------------------
+FUNCTION geoms_tctools, templatefile, hdffile, QA_error_code
+
+ reterr=''
+ idlcr8ascii, hdffile, ga, sds, reterr
+
+ chVN_value = check_for_version_name( ga ) ;check for existence of version name in DATA_SOURCE
+
+ ; find log file (see geoms_tools.pro)
+ logfile=STRMID(hdffile,0,STRPOS(hdffile,'.',/REVERSE_SEARCH))+'.log'
+
+ openw,lu,logfile,/GET_LUN,/APPEND
+
+ chGEOMSTemp = read_GEOMS_template( templatefile )
+ chMessage = 'Analyzing compliance to GEOMS template: '+FILE_BASENAME(templatefile)
+ print, chMessage
+ printf, lu, ''
+ printf, lu, chMessage
+
+ TC_error_code = analyze_GEOMS_VA( ga, sds, chGEOMSTemp, lu )
+
+ if chVN_value eq '' then begin ;print out QA/TC result because no need to do version name check
+ chMessage = STRTRIM(STRING( QA_error_code + 3*TC_error_code + 4 ), 1)+' (Total QA/TC error)'
+ printf, lu, ''
+ printf, lu, chMessage
+ Free_Lun, lu
+ endif else Free_Lun, lu
+
+ return, TC_error_code
+END
--
GitLab