From a611c4f4c29c7534569b07a408fbeb9ede788768 Mon Sep 17 00:00:00 2001
From: Ian Boyd <iboyd@astro.umass.edu>
Date: Tue, 29 Oct 2024 02:02:27 +0000
Subject: [PATCH] Replace idlcr8hdf.pro
---
idlcr8hdf.pro | 13584 ++++++++++++++++++++++++------------------------
1 file changed, 6794 insertions(+), 6790 deletions(-)
diff --git a/idlcr8hdf.pro b/idlcr8hdf.pro
index 01c0184..f596ef3 100644
--- a/idlcr8hdf.pro
+++ b/idlcr8hdf.pro
@@ -1,6790 +1,6794 @@
-;Main Program Version: idlcr8hdf.pro v4.0b64, 20240912
-; Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
-;
-;Sub-versions (refer to idlcr8hdf-v4.0_Readme.pdf for full history)
-
-PRO idlcr8hdf_common
-;Procedure to define the COMMON blocks for this program
-; ----------
-;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
-;
-;History:
-; 20050909: Introduced to IDLCR8HDF - Version 1.1
-; 20061012: Variable 'ncsa' added to METADATA; 'iarr', 'larr', 'rarr', 'darr' holding Arrays
-; removed from DATA and replaced with Structure 'ds'; Variable 'vfv' added to DATA;
-; WIDGET_WIN added for common variables associated with the Graphical User Interface
-; - Version 2.0
-; 20080302: tab_type integer added to TABLEDATA - Version 3.0
-; 20100205: rerr string added to WIDGET_WIN - Version 3.09
-; 20110401: vnchange added to DATA; mv_lng and mv_dbl added to METADATA; vserror added to DATA;
-; vfv removed from DATA; ncsa removed from METADATA - Version 4.0b1
-; 20150127: mv_str added to METADATA to hold maximum string length of string dataset entries -
-; Version 4.0b25
-; 20151012: Rename free_attr to attr_free and add qa_yes - Version 4.0b31
-;
-;Input: Nil
-;
-;Output: Nil
-;
-;Called by: N/A
-;
-;Subroutines Called: None
-
-COMMON TABLEDATA, tab_arr,tab_ver,tab_type
-COMMON METADATA, meta_arr,attr_arr_glob,attr_arr_data,attr_free,mv_lng,mv_dbl,mv_str
-COMMON DATA, ds,ndm,nvn,vn,vu,vnchange,vserror,qa_yes
-COMMON WIDGET_WIN, wtxt,b1,lineno,base,o3,dux,rerr,lvals
-
-END ;Procedure idlcr8hdf_common
-
-
-
-PRO intro_event, ev
-;Procedure to define how Events from the Start-up Introduction Window are handled
-; ----------
-;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
-;
-;History:
-; 20061012: Introduced to IDLCR8HDF - Version 2.0
-;
-;Input: ev - Selected widget event structure
-;
-;Output: o3 - Common string array defining the various options for program output
-;
-;Called by: XMANAGER in INTRO
-;
-;Subroutines Called: None
-
-COMMON WIDGET_WIN
-
-;The uservalue is retrieved from a widget when an event occurs
-WIDGET_CONTROL,ev.id,GET_UVALUE=uv
-;Assign/Remove AVK button event to/from a variable name
-IF uv EQ 'AVK' THEN IF o3[1] EQ uv THEN o3[1]='0' ELSE o3[1]=uv $
-;Assign/Remove Log Output button event to/from a variable name
-ELSE IF uv EQ 'idlcr8hdf.log' THEN IF o3[2] EQ uv THEN o3[2]='0' ELSE o3[2]=uv $
-;Assign/Remove Pop-up window for Log Output button event to/from a variable name
-ELSE IF uv EQ 'Pop' THEN IF o3[3] EQ uv THEN o3[3]='0' ELSE o3[3]=uv $
-;Assign button event to a variable name
-ELSE IF (uv EQ 'H4') OR (uv EQ 'H5') OR (uv EQ 'NC') THEN BEGIN
- o3[0]=uv
- IF uv EQ 'H5' THEN BEGIN
- WIDGET_CONTROL,ev.id+1,Sensitive=1
- WIDGET_CONTROL,ev.id+2,Sensitive=1,Set_Combobox_Select=0
- ENDIF
-ENDIF ELSE IF uv EQ '0' THEN IF ev.str EQ 'None' THEN o3[0]='H5' ELSE o3[0]='H5_'+STRTRIM(ev.str,2) $
-ELSE o3[0]='0' ;Cancel button chosen
-IF (uv NE 'AVK') AND (uv NE 'idlcr8hdf.log') AND (uv NE 'Pop') AND (uv NE 'H5') THEN WIDGET_CONTROL,ev.top,/DESTROY
-
-END ;Intro_Event
-
-
-
-PRO intro, intype
-;Procedure which creates an Introduction Window at start-up when IDLCR8HDF is called without parameters,
-;or invalid parameters, or is called by IDL Virtual Machine. The user has a choice of continuing with
-;the program after selecting input options, or stopping the program.
-; ----------
-;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
-;
-;History:
-; 20061012: Introduced to IDLCR8HDF - Version 2.0
-; 20080302: Swapped the order of the option windows, so that the options indicated by the check
-; boxes appear above the option boxes that close the Introduction Window and continue
-; with the program - Version 3.0
-; 20100205: Include text regarding the RETERR argument which can be accepted as an input
-; parameter when using the full version of idlcr8hdf - Version 3.09
-; 20110401: Add vertxt string array to hold text which changes between versions of the code;
-; Change text to reference GEOMS compliant files instead of AVDC/EVDC/NDACC; Change
-; text regarding the format of the structure required for input - Version 4.0b1
-; 20111220: Change text and options to include netCDF; change contact details - Version 4.0b7
-; 20130114: Make insensitive /LOG and /NC options if the program is called in IDL DEMO mode
-; -Version 4.0b14
-;
-;Input: intype - Integer set to -1 or -2: -1 indicates normal state; -2 indicates that input
-; parameters at the IDLCR8HDF call were invalid.
-;
-;Output: Nil
-;
-;Called by: IDLCR8HDF
-;
-;Subroutines Called: INTRO_EVENT (via XMANAGER)
-
-COMMON WIDGET_WIN
-
-nhdr=44 & errtxt=STRARR(nhdr)
-vertxt=['idlcr8hdf-v4.0_Readme.pdf','v4.0b64 September 2024']
-errtxt[1]='Welcome to IDLcr8HDF. This program creates GEOMS compliant HDF4, HDF5 and netCDF files'
-errtxt[2]='(also refer to '+vertxt[0]+').'
-errtxt[4]='Inputs to the program (IDL Virtual Machine (VM) and IDL Licensed (LIC) Versions):'
-errtxt[5]=' METADATA TEMPLATE FILE - Containing the Global and Variable Attribute information for the site.'
-errtxt[6]=' DATA FILE(s) - Multiple selection permitted. The file(s) must show the datasets in the single column'
-errtxt[7]=' format described in the documentation.'
-errtxt[8]=' TAV FILE - the current non-encrypted Table Attribute Values file.'
-errtxt[9]=' OUTPUT DIRECTORY - Directory for any HDF, netCDF and log files created. Shortcut values of ''M'' or ''D'''
-errtxt[10]=' will output files to the Metadata or Data file directories respectively.'
-errtxt[12]='Alternative Input Option for IDL LIC Versions:'
-errtxt[13]=' GLOBAL ATTRIBUTES ARRAY (GA) - a string array containing the Global Attributes in the form'
-errtxt[14]=' ''label=value''.'
-errtxt[15]=' DATA STRUCTURE (DS) - a heap structure using pointers containing the Variable Attribute Labels'
-errtxt[16]=' (DS.VA_L), the Variable Attribute Values (DS.VA_V), and the Data (DS.Data), for a single output file.'
-errtxt[17]=' TAV FILE - the current non-encrypted Table Attribute Values file.'
-errtxt[18]=' OUTPUT DIRECTORY - Directory for any HDF, netCDF and log files created.'
-errtxt[19]=' RETERR - String variable to which any error(s) are written.'
-errtxt[21]='For IDL VM, input is by ''DIALOG_BOXES''. For IDL LIC, input can be by ''DIALOG_BOXES'' or passed'
-errtxt[22]='by calling the program with one of the following command line options:'
-errtxt[23]=' 1. idlcr8hdf (Opens this box, and allows the user the option to continue with file inputs).'
-errtxt[24]=' 2. idlcr8hdf,METAFILE,DATAFILE(s),TAVFILE,OUTDIR[,RETERR][,/H5][,/Cn][,/NC][,/AVK][,/Log][,/Popup] or'
-errtxt[25]=' idlcr8hdf,'''','''','''',''''[,/H5][,/Cn][,/NC][,/AVK][,/Log][,/Popup] (For null string, DIALOG_BOXES will prompt for input).'
-errtxt[26]=' 3. idlcr8hdf,GA,DS,TAVFILE,OUTDIR[,RETERR][,/H5][,/Cn][,/NC][,/AVK][,/Log][,/Popup] or
-errtxt[27]=' idlcr8hdf,GA,DS[,/H5][,/Cn][,/NC][,/AVK][,/Log][,/Popup] (Inputs are from session memory, DIALOG_BOX(s) will'
-errtxt[28]=' prompt for input if TAVFILE or OUTDIR are not included).'
-errtxt[29]=' /H5, /Cn, /NC, /AVK, /Log and /Popup keywords are used in place of the options given below (HDF4 is default).'
-errtxt[30]='The /Cn keyword enables compression and shuffling of HDF5 files only (C1=low, C9=high). Default is no compression.'
-errtxt[32]='Contacts -'
-errtxt[33]=' Ian Boyd, Bryan Scientific Consulting LLC (iboyd@bryanscientific.org)'
-errtxt[34]=' 9 Cambridge Terrace'
-errtxt[35]=' Masterton 5810, New Zealand'
-errtxt[37]=' Ann Mari Fjaeraa, EVDC Project Manager (amf@nilu.no)'
-errtxt[38]=' Norwegian Institute for Air Research, Instituttveien 18'
-errtxt[39]=' Postbox 100, N-2027 KJELLER, NORWAY'
-errtxt[41]='EVDC Website: Tools and documentation available from http://evdc.esa.int/'
-errtxt[43]='To continue, please choose from the options below (Note: HDF5 only available on IDL6.2 or greater).'
-errtxt=' '+errtxt
-
-;Set-up text display widget
-IF intype EQ -2 THEN xtxt=' - Command Line Input Error' ELSE xtxt=''
-IF intype EQ -3 THEN optsens=0 ELSE optsens=1
-base=WIDGET_BASE(Title='idlcr8hdf '+vertxt[1]+xtxt,Tlb_Frame_Attr=1,/Column) ;,Tab_Mode=1)
-wtxt=WIDGET_TEXT(base,xsize=102,ysize=25,/Scroll)
-base3=WIDGET_BASE(base,/Nonexclusive)
-logtext='Append log output to the file ''idlcr8hdf.log'' '
-IF intype EQ -3 THEN logtext=logtext+'(No log or netCDF file output permitted in IDL DEMO Mode)' $
-ELSE logtext=logtext+'(will create the file IF it doesn''t exist)'
-poptext='Open a Pop-Up Window to display log output'
-avktext='If Avg. Kernel data are present, append sentence to VAR_NOTES giving first three values of '
-avktext=avktext+'the first kernel'
-avktip='Sentence reads ''First three values of the first kernel are: x.xx x.xx x.xx'''
-b4=WIDGET_BUTTON(base3,value=logtext,uvalue='idlcr8hdf.log',frame=3,SENSITIVE=optsens)
-b5=WIDGET_BUTTON(base3,value=poptext,uvalue='Pop',frame=3)
-b6=WIDGET_BUTTON(base3,value=avktext,uvalue='AVK',frame=3) ;,Tooltip=AVKTip)
-base2=WIDGET_BASE(base,/Row)
-tip='Left Mouse Click or Tab to entry and hit <Spacebar>'
-b1=WIDGET_BUTTON(base2,value='HDF4',uvalue='H4',frame=3) ;,Tooltip=Tip)
-b7=WIDGET_BUTTON(base2,value='netCDF3',uvalue='NC',frame=3,SENSITIVE=optsens)
-IF FLOAT(!Version.Release) GE 6.2 THEN $
- b2=WIDGET_BUTTON(base2,value='HDF5',uvalue='H5',frame=3) $;,Tooltip=Tip) $
-ELSE b2=WIDGET_BUTTON(base2,value='HDF5',Sensitive=0,frame=3)
-b8=Widget_Label(base2,Value=' Compression ',Sensitive=0)
-b9=Widget_Combobox(base2,Value=lvals,Sensitive=0,uvalue='0')
-
-b3=WIDGET_BUTTON(base2,value='Stop',uvalue='CANCEL',frame=3) ;,ToolTip=Tip)
-WIDGET_CONTROL,base,/Realize
-WIDGET_CONTROL,b4,/Input_Focus
-FOR i=0,N_ELEMENTS(errtxt)-1 DO $
- WIDGET_CONTROL,wtxt,set_value=errtxt[i],/Append
-XMANAGER,'intro',base
-
-END ;Intro
-
-
-
-PRO idlcr8hdf_event, ev
-;Procedure to close the pop-up logging window after user selects 'Finish'.
-; ----------
-;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
-;
-;History:
-; 20061012: Introduced to IDLCR8HDF - Version 2.0
-;
-;Input: Selected widget event structure
-;
-;Output: Nil
-;
-;Called by: XMANAGER in IDLCR8HDF and STOP_WITH_ERROR
-;
-;Subroutines Called: None
-
-WIDGET_CONTROL,ev.top,/DESTROY
-RETALL & HEAP_GC
-
-END ;IDLcr8HDF_Event
-
-
-
-FUNCTION is_a_number_hdf, value, ind
-
-IF N_PARAMS() EQ 2 THEN BEGIN
- ;Check each individual character is numeric
- nlen=STRLEN(value)
- retval=1B
- FOR i=0,nlen-1 DO BEGIN
- ah=STRMID(value,i,1)
- isan=(BYTE(ah) GE 48) AND (BYTE(ah) LE 57) ;0-9
- IF ~isan THEN retval=0B
- ENDFOR
- RETURN, retval
-ENDIF ELSE BEGIN
- ON_IOERROR, ConversionError
- IF STRTRIM(value,2) EQ '' THEN RETURN, 0B
- n=DOUBLE(value)
- RETURN, 1B
- ConversionError:
- RETURN, 0B
-ENDELSE
-END
-
-
-
-PRO stop_with_error, txt1, txt2, lu
-;Procedure called when an error in the program inputs is detected. An error message is displayed
-;and the program stopped and reset. If necessary, open files are closed, and memory associated
-;with a structure is freed. The error message is displayed in one or more of the following:
-;a Pop-up dialog window; the Pop-up logging window; the Output Logging window in the IDLDE;
-;a log file (idlcr8hdf.log)
-; ----------
-;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
-;
-;History:
-; 20050802: Original IDLCR8HDF Routine - Version 1.0
-; 20061012: Set-up so that the error output is displayed in the output window(s) dependent on
-; the options chosen by the user, the point in the program that the error is detected,
-; and the method that IDLCR8HDF is called. If txt1 is preceeded by 'D_' or is null,
-; the error output is to a Pop-up Dialog window. Other error output options are
-; determined by the values in the (Common) dux array - Version 2.0
-; 20100205: Allow routine to return to the calling routine, rather than stop the application, if
-; a 'reterr' argument is included in the call to idlcr8hdf - Version 3.09
-; 20110401: Change end text dependent on whether the program is creating a GEOMS file or doing QA
-; on a GEOMS file - Version 4.0b1
-; 20111220: Change text referring to HDF to GEOMS - Version 4.0b7
-; 20171121: Add comment advising that program stopped before all checks were completed if doing
-; QA; Add COMMON DATA to routine and remove ds from the list of input parameters
-; - Version 4.0b43
-;
-;Inputs: txt1 - the initial text line of the error message. Generally contains the name of the routine
-; where the error was detected.
-; txt2 - the second text line which generally describes the error state.
-; lu - Where applicable, the file unit that needs to be closed at the termination of the program,
-; otherwise set to -1.
-; ds - Where applicable, the name of the data structure set-up by the program (dependent on
-; the point in the program that the error is detected), so that memory associated with the
-; structure can be freed (COMMON variable).
-; dux - an integer array used to determine the output options for the error message (COMMON) variable).
-; qa_yes - a boolean to indicate whether idlcr8hdf called in QA mode or not (COMMON variable).
-;
-;Output: Error message displayed/output dependent on the requested output options.
-;
-;Called by: The routine in which the error was detected. The following routines call STOP_WITH_ERROR:
-; READ_TABLEFILE; TEST_FILE_INPUT; READ_METADATA; EXTRACT_AND_TEST; CHECK_METADATA;
-; SET_UP_STRUCTURE; CHECK_MIN_MAX_FILL; EXTRACT_DATA; FIND_HDF_FILENAME; IDLCR8HDF.
-;
-;Subroutines Called: IDLCR8HDF_EVENT (via XMANAGER)
-
-COMMON DATA
-COMMON WIDGET_WIN
-
-IF lu NE -1L THEN FREE_LUN,lu
-
-IF txt1 EQ '' THEN BEGIN ;<cancel> chosen on Intro box
- res=DIALOG_MESSAGE('IDLcr8HDF Stopped!',/Information,Title='EVDC IDLcr8HDF')
-ENDIF ELSE BEGIN
- ;If necessary free up memory by destroying the heap variables pointed at by its pointer arguments
- IF N_ELEMENTS(ds) NE 0 THEN PTR_FREE,ds.data
- ;Write error to file and/or IDL output window
- IF STRMID(txt1,0,2) EQ 'D_' THEN txtx=STRMID(txt1,2) ELSE txtx=txt1
- FOR i=dux[0],dux[1],dux[2] DO BEGIN
- IF i EQ -1 THEN BEGIN
- PRINT,' ERROR in '+txtx & PRINT,' '+txt2
- PRINT,'' & PRINT,'IDLcr8HDF stopped - Program Ended on '+SYSTIME(0)
- ENDIF ELSE BEGIN
- PRINTF,i,' ERROR in '+txtx & PRINTF,i,' '+txt2
- IF (i EQ dux[0]) OR ((i EQ dux[1]) AND (STRPOS(o3[2],'idlcr8hdf.log') NE -1)) THEN BEGIN
- PRINTF,i,'' & PRINTF,i,'IDLcr8HDF stopped - Program Ended on '+SYSTIME(0)
- ENDIF ELSE IF qa_yes THEN BEGIN
- PRINTF,i,'' & PRINTF,i,' INFORMATION: QA stopped before all checks could be completed'
- ENDIF
- ENDELSE
- ENDFOR
- IF dux[1] NE dux[0] THEN FREE_LUN,dux[1]
- IF (STRMID(txt1,0,2) EQ 'D_') AND (rerr[0] EQ 'NA') THEN BEGIN
- ;write error to DIALOG Box instead of Popup window
- errtxt2=STRARR(4)
- errtxt2[0]=STRMID(txt1,2) & errtxt2[1]=txt2 & errtxt2[3]='IDLcr8HDF Stopped!'
- res=DIALOG_MESSAGE(errtxt2,/Error,Title='EVDC IDLcr8HDF Error')
- ENDIF ELSE IF rerr[0] EQ 'NA' THEN BEGIN ;write error to Popup window
- IF o3[4] EQ '0' THEN endtxt='GEOMS file creation ' $
- ELSE endtxt='GEOMS file testing '
- lineno=lineno+4L
- WIDGET_CONTROL,wtxt,set_value=' ERROR in '+txt1,/Append
- WIDGET_CONTROL,wtxt,set_value=' '+txt2,/Append
- WIDGET_CONTROL,wtxt,set_value='',/Append,Set_text_top_line=lineno
- WIDGET_CONTROL,wtxt,set_value=endtxt+'stopped - hit <Finish> to close program',/Append
- WIDGET_CONTROL,b1,Sensitive=1,/Input_Focus
- XMANAGER,'stop_with_error',base,Event_Handler='idlcr8hdf_event'
- ENDIF
-ENDELSE
-HEAP_GC
-IF rerr[0] EQ 'NA' THEN RETALL $
-ELSE BEGIN
- IF o3[4] EQ '0' THEN endtxt='create GEOMS file' $
- ELSE endtxt='complete GEOMS file test'
- rerr[0]='Unable to '+endtxt+' - '+txtx+txt2
-ENDELSE
-
-END ;Procedure Stop_With_Error
-
-
-
-PRO infotxt_output, infotxt
-;Procedure called when the program makes a change to the input meta data or reports information
-;relevant to the creation of the HDF/netCDF file. This information can be reported to a Pop-up logging
-;window, IDLDE output log window, and/or the log file. Code for this output was originally
-;written directly in the affected procedures
-; ----------
-;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
-;
-;History:
-; 20090311: Introduced to IDLCR8HDF - Version 3.06
-; 20110401: infotxt INFORMATION header changed to a number, to allow changes to the comment
-; header depending on how idlcr8hdf has been called, as follows: 0 = INFORMATION;
-; 1 = NON-STANDARD COMPLIANCE NOTIFICATION (QA) o/w INFORMATION; 2 = ERROR (QA)
-; o/w INFORMATION; 3 = ERROR; 4 = DEBUG - Version 4.01
-;
-;Inputs: infotxt - the text line(s) of the information message. Can be either a scalar string
-; or string array
-;
-;Output: Message displayed/output dependent on the requested output options
-;
-;Called by: The routine in which the reported change was made. The following routines call
-; INFOTXT_OUTPUT: READ_TABLEFILE; GEOMS_RULE_CHANGES; PRE_DEFINED_ATT_CHECKS;
-; READ_METADATA; EXTRACT_AND_TEST; CHECK_METADATA; EXTRACT_DATA; SET_UP_STRUCTURE;
-; CHECK_STRING_DATATYPE; CHECK_MIN_MAX_FILL; READ_DATA; FIND_HDF_FILENAME;
-; AVDC_HDF_WRITE; IDLCR8HDF
-;
-;Subroutines Called: None
-
-COMMON DATA
-COMMON WIDGET_WIN
-
-dm=SIZE(infotxt,/N_ELEMENTS)
-qaval=FIX(STRMID(infotxt[0],0,1))
-write_rerr=0 ;Boolean to generate a return error message
-
-IF qa_yes THEN BEGIN ;program called in QA mode
- ;Add correct message title
- CASE 1 OF
- qaval EQ 0: infotxt[0]=' INFORMATION:'+STRMID(infotxt[0],1)
- qaval EQ 1: infotxt[0]=' NON-STANDARD COMPLIANCE NOTIFICATION:'+STRMID(infotxt[0],1)
- qaval EQ 4: infotxt[0]=' DEBUG:'+STRMID(infotxt[0],1)
- ELSE: infotxt[0]=' ERROR:'+STRMID(infotxt[0],1)
- ENDCASE
- ;truncate message from the '|'
- bs_found=0
- FOR n=0,dm-1 DO BEGIN
- IF bs_found EQ 1 THEN infotxt[n]='' $
- ELSE BEGIN
- bspos=STRPOS(infotxt[n],'|')
- IF bspos NE -1 THEN BEGIN
- infotxt[n]=STRMID(infotxt[n],0,bspos)
- bs_found=1
- ENDIF
- ENDELSE
- ENDFOR
- ;recalculate number of good infotxt values
- gi=WHERE(infotxt NE '',dm)
-ENDIF ELSE BEGIN ;program called in HDF file create mode
- CASE 1 OF
- qaval EQ 3: BEGIN
- infotxt[0]=' ERROR:'+STRMID(infotxt[0],1)
- IF rerr[1] NE 'NA' THEN BEGIN ;generate return error message
- IF rerr[1] EQ '' THEN BEGIN
- IF o3[4] EQ '0' THEN endtxt='create GEOMS file' $
- ELSE endtxt='complete GEOMS file test'
- ENDIF
- write_rerr=1
- ENDIF
- o3[4]='NOHDF' ;Error in Input so do not create HDF file
- END
- qaval EQ 4: infotxt[0]=' DEBUG:'+STRMID(infotxt[0],1)
- ELSE: infotxt[0]=' INFORMATION:'+STRMID(infotxt[0],1)
- ENDCASE
- ;remove '|' from the message
- FOR n=0,dm-1 DO BEGIN
- bspos=STRPOS(infotxt[n],'|')
- IF bspos NE -1 THEN BEGIN
- infotxt[n]=STRMID(infotxt[n],0,bspos)+STRMID(infotxt[n],bspos+1)
- ENDIF
- ENDFOR
-ENDELSE
-
-lineno=lineno+dm
-FOR n=0,dm-1 DO BEGIN
- IF o3[3] EQ '' THEN WIDGET_CONTROL,wtxt,set_value=infotxt[n],/Append,Set_text_top_line=lineno
- FOR m=dux[0],dux[1],dux[2] DO BEGIN
- IF m EQ -1 THEN PRINT,infotxt[n] ELSE PRINTF,m,infotxt[n] ;write out to log
- ENDFOR
-ENDFOR
-
-IF write_rerr EQ 1 THEN BEGIN
- IF rerr[1] EQ '' THEN rerr[1]='Unable to '+endtxt+' -'+STRMID(infotxt[0],8) $
- ELSE rerr[1]=rerr[1]+';'+STRMID(infotxt[0],8)
- IF dm GT 1 THEN FOR n=1,dm-1 DO rerr[1]=rerr[1]+STRMID(infotxt[n],3)
-ENDIF
-
-END ;Procedure InfoTxt_Output
-
-
-
-FUNCTION jdf_2_datetime, jdf, MJD2000=mjd2000, SHORTISO8601=iso, LONGISO8601=isoms
-;Computes the UT date/time from JDF/MJD2000.
-; ----------
-; Bojan R. Bojkov
-; bojan.r.bojkov@nasa.gov
-; 03/04/2004
-; 03/11/2004 bug fix
-; 05/27/2005 add ShortISO8601 and LongISO8601 switches,
-; and change seconds value to decimal seconds (Ian Boyd)
-;
-; References ----------
-; Explan. Supp. Astron. Almanac, p.604 (1992); through E. Celarier
-;
-; Caveats ----------
-; Valid for all YYYY >= -4712 (i.e. for all JD .ge. 0)
-; The true Gregorian calendar was only adopted on 15 October 1582,
-; so any dates before this are "virtual" Gregorian dates.
-;
-; Input ----------
-; jdf : double precision value
-; mjd2000 : flag if input is MJD2000
-; iso : flag if output is in ISO8601 (YYYYMMDDThhmmssZ)
-; isoms : flag if output is in ISO8601ms (YYYYMMDDThhmmss.sssZ)
-;
-; Output ---------
-; floating point array [YYYY, MM, DD, hh, mn, ss.sss],
-; or ISO8601 string format
-; External subroutines ---------
-; NONE
-
-jdf=DOUBLE(jdf)
-
-j0=2451544.5D
-IF KEYWORD_SET(mjd2000) THEN jdhold=jdf+j0 ELSE jdhold=jdf
-jdi=LONG(jdhold)
-df=jdhold-jdi
-
-;Determine hh, mm, ss, ms
-hh=(df+0.5)*24.D
-q=WHERE(hh GE 24.D)
-IF q[0] NE -1 THEN BEGIN
- hh[q]=hh[q]-24.D
- jdi[q]=jdi[q]+1
-ENDIF
-t1=hh
-hh=LONG(t1)
-t2=(t1-hh)*60.D
-mn=LONG(t2)
-t3=(t2-mn)*60.D
-ss=t3
-;ss=LONG(t3)
-;ms=LONG((t3-ss)*1.d3)
-
-;Determine YYYY, MM, DD
-t1=jdi+68569L
-t2=(4*t1)/146097L
-t1=t1-(146097L*t2+3L)/4L
-t3=(4000L*(t1+1L))/1461001L
-t1=t1-(1461L*t3)/4L + 31L
-t4=(80L*t1)/2447L
-
-dd=t1-(2447L*t4)/80L
-t1=t4/11L
-mm=t4+2L-12L*t1
-yyyy=100L*(t2-49L)+t3+t1
-
-dt=TRANSPOSE([[yyyy],[mm],[dd],[hh],[mn],[ss]])
-IF (KEYWORD_SET(iso)) OR (KEYWORD_SET(isoms)) THEN BEGIN
- dts=STRARR(6)
- IF (KEYWORD_SET(iso)) AND (ss-FIX(ss) GE 0.5) THEN BEGIN
- ;recalculate to get correct seconds value
- jdhold=jdf+0.000008D ;add ~0.7 secs
- IF KEYWORD_SET(mjd2000) THEN jdhold=jdhold+j0
- CALDAT,jdhold,mm,dd,yyyy,hh,mn,ss
- dt=TRANSPOSE([[yyyy],[mm],[dd],[hh],[mn],[ss]])
- ENDIF
- dt=LONG(dt)
- FOR i=1,5 DO $
- IF dt[i] LT 10L THEN dts[i]='0'+STRTRIM(dt[i],2) ELSE dts[i]=STRTRIM(dt[i],2)
- IF KEYWORD_SET(isoms) THEN BEGIN
- ssd=STRING(format='(f6.3)',ss)
- IF FLOAT(ssd) LT 10.0 THEN dts[5]='0'+STRTRIM(ssd,2) ELSE dts[5]=STRTRIM(ssd,2)
- ENDIF
- RETURN,STRTRIM(dt[0],2)+dts[1]+dts[2]+'T'+dts[3]+dts[4]+dts[5]+'Z'
-ENDIF ELSE RETURN, dt
-
-END ;Function jdf_2_datetime
-
-
-
-FUNCTION julian_date, date_ut, ISO8601=iso, MJD2000=mjd2000
-;Computes the Julian date (jd) or date in MJD2000 format in double precision.
-; ----------
-; Bojan R. Bojkov
-; bojan.r.bojkov@nasa.gov
-; 06/03/2001
-; 09/24/2001: Code cleanup - Function Version 1.0
-; 10/16/2001: Mode corrections - Function Version 1.01
-; 03/07/2002: Comment cleanup and variable change - Function Version 2.0
-; 10/08/2002: Added jdf option - Function Version 2.1
-; 07/21/2003: Converted to a function - Function Version 3.0
-; 05/25/2005: Added ISO8601 and MJD2000 keywords.
-; Input can either be a string or a numeric array
-; (see below for input details). Returns -99999.d
-; if the Input is invalid - Ian Boyd
-;
-; References ----------
-; Hughes, D.W., Yallop, B.D. and Hohenkerk, C.Y., "The Equation of Time",
-; Mon. Not. R. astr. Soc., 238, pp 1529-1535. 1989.
-;
-; Results verified using:
-; Meeus, J, "Astronomical Algorithms", William-Bell, Inc., Richmond VA,
-; p62, 1991.
-;
-; Caveats ----------
-; NONE
-;
-; Input ----------
-; date_ut: Numeric array (UT: YYYY, MM, DD, [hh, mm, ss]), or if ISO8601
-; keyword set, a string containing datetime as YYYYMMDDThhmmssZ
-;
-; Output ---------
-; jd: julian day, or if MJD2000 keyword set,
-; a double precision day in MJD2000 format
-;
-; External subroutines ---------
-; NONE
-
-ON_IOERROR,TypeConversionError
-valid=0 ;Type conversion check
-inputerror=0 ;Check that input is as expected
-;Check input format
-IF KEYWORD_SET(iso) THEN BEGIN ;input is YYYYMMDDThhmmssZ
- IF STRLEN(date_ut) EQ 16 THEN BEGIN
- ;test for numeric values (will return type conversion error value if not a number)
- FOR i=0,7 DO check=FIX(STRMID(date_ut,i,1))
- FOR i=9,14 DO check=FIX(STRMID(date_ut,i,1))
- dt=INTARR(6)
- dt[0]=FIX(STRMID(date_ut,0,4)) & dt[1]=FIX(STRMID(date_ut,4,2))
- dt[2]=FIX(STRMID(date_ut,6,2)) & dt[3]=FIX(STRMID(date_ut,9,2))
- dt[4]=FIX(STRMID(date_ut,11,2)) & dt[5]=FIX(STRMID(date_ut,13,2))
- date_ut=dt & achk=[0,6] ;to add hhmmss component
- ENDIF ELSE inputerror=1
-ENDIF ELSE BEGIN
- ;check that input contains at least YMD (and at most YMDhms) information and is either
- ;an integer, long, float or double array
- achk=SIZE(date_ut)
- IF (achk[1] LT 3) OR (achk[1] GT 6) OR (achk[2] LT 2) OR (achk[2] GT 5) THEN inputerror=1
-ENDELSE
-
-IF inputerror EQ 0 THEN BEGIN ;can perform JD calculation
- IF date_ut[1] GT 2 THEN BEGIN
- y=DOUBLE(date_ut[0])
- m=DOUBLE(date_ut[1]-3)
- d=DOUBLE(date_ut[2])
- ENDIF ELSE BEGIN
- y=DOUBLE(date_ut[0]-1)
- m=DOUBLE(date_ut[1]+9)
- d=DOUBLE(date_ut[2])
- ENDELSE
-
- ;Compute Julian date:
- j=LONG(365.25D0*(y+4712.D0))+LONG(30.6D0*m+0.5D0)+59.D0+d-0.5D0
-
- ;Check for Julian or Gregorian calendar:
- IF j LT 2299159.5D0 THEN jd=j $ ;If Julian calendar.
- ELSE BEGIN ;If Gregorian calendar.
- gn=38.D0-LONG(3.D0*LONG(49.D0+y/100.D0)/4.D0)
- jd=j+gn
- ENDELSE
-
- ;add hhmmss values if present
- CASE 1 OF
- achk[1] EQ 4: jd=jd+DOUBLE(date_ut[3])/24.D ;hh only
- achk[1] EQ 5: jd=jd+(DOUBLE(date_ut[3])*3600.D + DOUBLE(date_ut[4])*60.D)/86400.D ;hhmm only
- achk[1] EQ 6: jd=jd+(DOUBLE(date_ut[3])*3600.D + DOUBLE(date_ut[4])*60.D + $
- DOUBLE(date_ut[5]))/86400.D ;hhmmss
- ELSE:
- ENDCASE
-
- valid=1 ;Type conversion OK
-
- ;Set to MJD2000 if required
- IF KEYWORD_SET(mjd2000) THEN jd=jd-2451544.5D
-ENDIF
-
-TypeConversionError:
-IF valid EQ 0 THEN RETURN,-99999.D ELSE RETURN, jd
-
-END ;Function Julian_Date
-
-
-
-PRO var_units_test, vuvalue, rd, tab_type, var_si_conv, errstate
-;Procedure to perform checks on the VAR_UNITS value in the metadata and, based on the VAR_UNITS input,
-;calculate and return the VAR_SI_CONVERSION value.
-; ----------
-;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
-;
-; History:
-; 20061012: Introduced to IDLCR8HDF - Version 2.0
-; (Previously these checks were performed in the Extract_and_Test routine)
-; 20080302: var_unit_arr and unit_pre_arr added which hold, respectively, the valid VAR_UNITS and
-; corresponding VAR_SI_CONVERSION values, and the set of UNIT_PREFIXs (previously
-; values from the EVDC/AVDC TAV file have been used). This has been done so that the
-; routine can be stand-alone (i.e. not dependent on also having to read in a TAV file),
-; and also because the original Envisat table.dat file does not contain the
-; VAR_SI_CONVERSION values. The input parameters have been changed to reflect this: bu
-; and up arrays (previously containing the VAR_UNIT and UNIT_PREFIX values from the TAV
-; file), and nbu and nup (the number of elements in the bu and up arrays) are no longer
-; used. The integer flag tab_type has been added to account for the different handling of
-; some of the VAR_UNIT and VAR_SI_CONVERSION values between AVDC and original Envisat.
-; The STOP_WITH_ERROR routine is no longer called in the event of an error, but an Error
-; State string is returned instead. Bug fixed when testing for a UNIT_PREFIX - previously
-; only the first character of the VAR_UNIT value was checked for a possible UNIT_PREFIX,
-; thus 'deka' (da) was excluded. Bug Fixed when the calculated Power Value of the last
-; base SI unit shown in VAR_SI_CONVERSION is '1', this is now set so that the power value
-; does not appear - Version 3.0
-; 20090611: Galileo added to var_unit_arr (AVDC); Last VAR_SI_CONVERSION value for ppmv, ppbv,
-; pptv, and ppv changed to DIMENSIONLESS (was ppv) in var_unit_arr (AVDC);
-; VAR_SI_CONVERSION values for molec changed to 0;1.66054E-24;mol (was 0;1;molec) in
-; var_unit_arr (AVDC); VAR_SI_CONVERSION values for DU changed to 0;4.4615E-4;mol m-2
-; (was 0;2.6867E20;molec m-2) in var_unit_arr; Stop power units being added to
-; DIMENSIONLESS (e.g. for VAR_UNITS=ppmv2); Change EVDC VAR_SI_CONVERSION values in
-; var_unit_arr to match Envisat Metadata Guidelines values - Version 3.01
-; 20091117: EVDC VAR_SI_CONVERSION values set to the same as AVDC (only one var_unit_arr set);
-; Fix bug which, in some cases, does not account for repeated units in VAR_UNITS when
-; determining the VAR_SI_CONVERSION (e.g. VAR_UNITS=W m-2 sr-1 m-1); Put VAR_SI_CONVERSION
-; units in power value order; Correctly scale multiple units by the power
-; e.g. W2 = (m2 kg s-3)^2 (previously assumed only a single unit was being scaled);
-; Generate error if third VAR_SI_CONVERSION value is 'DIMENSIONLESS' or 'NONE' but also
-; includes extra values - Version 4.0b1
-; 20101001: Set up for GEOMS compliance e.g. DIMENSIONLESS changed to 1; MJD2000 changed to MJD2K;
-; NONE removed - Version 4.0b2
-; 20101221: Bug fix - need to correctly account for when units cancel each other out. Was still
-; assigned the value DIMENSIONLESS - Version 4.0b3
-; 20110303: Bug fix - when a dimensionless unit includes a power value (e.g. ppmv2), the base unit
-; in VAR_SI_CONVERSION stays as '1'; Add 'dB' to var_unit_arr - Version 4.0b4
-; 20110719: Add 'pH' - Version 4.0b5
-; 20240908: Bug fix - Using the SORT command to put units in order of power value can give different
-; results depending on the operating system when power values are identical. Change routine
-; so the MAX command is used instead to create an array in descending power order
-; - Version 4.0b6
-; 20240912: Reinstitute sorting power units with the SORT command and save this version as a second
-; var_si_conv option (var_si_conv becomes a string array of size 2). This means the ordering
-; of the units between the two var_si_conv values may differ - Version 4.0b7
-;
-; Inputs: vuvalue - a string containing the Metadata VAR_UNITS value to be checked (everything
-; after the '=')
-; rd - integer flag to indicate if VAR_DATA_TYPE is real/double (-1) or not (1). Used to
-; make VAR_SI_CONVERSION values of the same type
-; tab_type - integer flag differentiating between AVDC (0) and original Envisat (1) styles
-;
-; Output: var_si_conv - a string array of size 2 containing the VAR_SI_CONVERSION value determined
-; by the program (returns '' if an error is encountered). The 2 values may
-; differ if there are multiple units with the same power value, due to using
-; two different sort methods
-; errstate - string describing an error state encountered during testing (o/w set to '')
-;
-; Called by: CHECK_METADATA
-;
-; External Subroutines Called: None
-
-var_unit_arr=['%;0;0.01;1','A;0;1;A','C;0;1;s A','cd;0;1;cd','d;0;86400;s','deg;0;1.74533E-2;rad',$
- 'degC;273.15;1;K','1;0;1;1','h;0;3600;s','Hz;0;1;s-1','J;0;1;m2 kg s-2','K;0;1;K',$
- 'l;0;1E-3;m3','lm;0;1;cd sr','lx;0;1;cd sr m-2','m;0;1;m','min;0;60;s',$
- 'MJD2K;0;86400;s','mol;0;1;mol','molec;0;1.66054E-24;mol','Np;0;1;1',$
- 'N;0;1E3;m kg s-2','Pa;0;1;m-1 kg s-2','pH;0;1E-12;m2 kg s-2 A-2','photons;0;1;photons',$
- 'ppbv;0;1E-9;1','ppmv;0;1E-6;1','pptv;0;1E-12;1','ppv;0;1;1','psu;0;1;psu',$
- 'rad;0;1;rad','s;0;1;s','sr;0;1;sr','V;0;1;m2 kg s-3 A-1','W;0;1;m2 kg s-3','kg;0;1;kg',$
- 'DU;0;4.4614E-4;mol m-2','Gal;0;1E-2;m s-2','dB;0;1;1']
-
-unit_pre_arr=['Y;yotta;1E24','Z;zetta;1E21','E;exa;1E18','P;peta;1E15','T;tera;1E12','G;giga;1E9',$
- 'M;mega;1E6','k;kilo;1E3','h;hecto;1E2','da;deka;1E1','d;deci;1E-1','c;centi;1E-2',$
- 'm;milli;1E-3','u;micro;1E-6','n;nano;1E-9','p;pico;1E-12','f;femto;1E-15',$
- 'a;atto;1E-18','z;zepto;1E-21','y;yocto;1E-24']
-
-;Set up text in case an error is found in the input VAR_UNITS value
-errtxt=STRARR(2)
-IF tab_type EQ 1 THEN errtxt[0]='No match with Table.Dat BASE_UNIT/UNIT_PREFIX values' $
-ELSE errtxt[0]='No match with Table Attribute Values file VAR_UNITS/UNIT_PREFIX values'
-errtxt[1]='Not valid'
-errtxt='VAR_UNITS='+vuvalue+': '+errtxt
-var_si_conv=STRARR(2) & errstate='' ;initialize outputs
-
-nta=N_ELEMENTS(var_unit_arr)
-;extract var_unit_arr/unit_pre_arr sub-values into vu/up arrays
-vuhold=STRSPLIT(var_unit_arr[0],';',/Extract) ;test for number of sub-values
-nvu=N_ELEMENTS(vuhold) & vu=STRARR(nvu,nta)
-FOR j=0,nta-1 DO BEGIN
- vuhold=STRSPLIT(var_unit_arr[j],';',/Extract)
- vu[*,j]=vuhold
-ENDFOR
-nta=N_ELEMENTS(unit_pre_arr)
-vuhold=STRSPLIT(unit_pre_arr[0],';',/Extract) ;test for number of sub-values
-nup=N_ELEMENTS(vuhold) & up=STRARR(nup,nta)
-FOR j=0,nta-1 DO BEGIN
- vuhold=STRSPLIT(unit_pre_arr[j],';',/Extract)
- up[*,j]=vuhold
-ENDFOR
-
-;separate out metadata sub-values into component parts, and set-up holding arrays
-vp=STRSPLIT(STRTRIM(vuvalue,2),' ',/Extract) & vpn=N_ELEMENTS(vp)
-vpx=STRARR(2,vpn) ;0 holds VAR_UNIT value, 1 holds POWER component
-bpx=STRARR(vpn) ;holding string array for base unit
-ex=INTARR(vpn)+1 ;holding integer array for power value (defaults to 1)
-tm=DBLARR(vpn) ;holding array for scale factor
-j=0
-WHILE (j LE vpn-1) AND (errstate EQ '') DO BEGIN
- ;test to see if the sub-value is a base unit in the TAV file
- ti=WHERE(vp[j] EQ vu[0,*],tcnt)
- IF tcnt NE 0 THEN vpx[0,j]=vp[j] $ it is a base unit
- ELSE BEGIN ;separate out into unit and power values as required
- vpx[0,j]=STRMID(vp[j],0,1) ;save first character of vp(j)
- stopchk=0
- IF (STRLEN(vp[j]) GE 2) THEN BEGIN
- FOR k=1,STRLEN(vp[j])-1 DO BEGIN
- ah=STRMID(vp[j],k,1)
- test1=(BYTE(ah) GE 65) AND (BYTE(ah) LE 90) ;A-Z
- test2=(BYTE(ah) GE 97) AND (BYTE(ah) LE 122) ;a-z
- IF (test1[0]) OR (test2[0]) THEN BEGIN
- IF stopchk EQ 0 THEN vpx[0,j]=vpx[0,j]+ah ELSE stopchk=2
- ;if stopchk EQ 2 THEN this means that VAR_UNITS is not legal
- ENDIF ELSE IF stopchk EQ 0 THEN BEGIN
- stopchk=1 ;first non-alpha character so check for numeric or '-' character
- test1=(BYTE(ah) EQ 45) OR ((BYTE(ah) GE 49) AND (BYTE(ah) LE 57)) ;- or 1-9
- IF NOT test1[0] THEN stopchk=2 ELSE vpx[1,j]=ah
- ENDIF ELSE IF stopchk EQ 1 THEN BEGIN
- ;need to check for a numeric character only
- test1=(BYTE(ah) GE 48) AND (BYTE(ah) LE 57) ;0-9
- IF NOT test1[0] THEN stopchk=2 ELSE vpx[1,j]=vpx[1,j]+ah
- ENDIF
- ENDFOR
- IF vpx[1,j] NE '' THEN ex[j]=FIX(vpx[1,j])
- ENDIF
- IF stopchk EQ 2 THEN vpx[0,j]=vp[j] ;in the event that the value is not valid, so will create error
- ;Do TAV check on the VAR_UNIT value
- ti=WHERE(vpx[0,j] EQ vu[0,*],tcnt)
- ENDELSE
-
- IF tcnt NE 0 THEN BEGIN ;VAR_UNIT is a BASE_VALUE
- bemult=(DOUBLE(vu[nvu-2,ti[0]]))^ex[j] & tm[j]=bemult
- bpx[j]=vu[nvu-3,ti[0]]+';'+STRTRIM(STRING(format='(E8.1)',bemult),2)+';'+vu[nvu-1,ti[0]]
- ENDIF ELSE IF vpx[0,j] EQ 'g' THEN BEGIN ;check for VAR_UNIT EQ g (gram) for AVDC style TAV file
- bemult=0.001d^ex[j] & tm[j]=bemult
- bpx[j]='0;'+STRTRIM(STRING(format='(E8.1)',bemult),2)+';kg'
- ENDIF ELSE BEGIN ;separate out vpx value into prefix and base-value and test
- ;check for valid prefix - first check for 'deka' (da)
- pref=STRMID(vpx[0,j],0,2) & bas=STRMID(vpx[0,j],2)
- pi=WHERE(pref EQ up[0,*],pcnt)
- IF pcnt EQ 0 THEN BEGIN ;test for the remaining prefixes
- pref=STRMID(vpx[0,j],0,1) & bas=STRMID(vpx[0,j],1)
- pi=WHERE(pref EQ up[0,*],pcnt)
- ENDIF
- IF pcnt NE 0 THEN BEGIN
- pmult=DOUBLE(up[nup-1,pi[0]])
- ;check for valid base
- ti=WHERE(bas EQ vu[0,*],tcnt)
- IF tcnt NE 0 THEN BEGIN
- bpmult=(DOUBLE(vu[nvu-2,ti[0]])*pmult)^ex[j] & tm[j]=bpmult
- bpx[j]=vu[nvu-3,ti[0]]+';'+STRTRIM(STRING(format='(E8.1)',bpmult),2)+';'+vu[nvu-1,ti[0]]
- ENDIF ELSE IF bas EQ 'g' THEN BEGIN ;check for VAR_UNIT EQ g (gram) for AVDC style TAV file
- bpmult=(pmult*0.001D)^ex[j] & tm[j]=bpmult
- bpx[j]='0;'+STRTRIM(STRING(format='(E8.1)',bpmult),2)+';kg'
- ENDIF ELSE errstate=errtxt[0]
- ENDIF ELSE errstate=errtxt[0]
- ENDELSE
- j=j+1
-ENDWHILE
-
-IF errstate EQ '' THEN BEGIN ;No errors detected so continue
- ;Create VAR_SI_CONVERSION value
- tmult=1.0D
- FOR j=0,vpn-1 DO tmult=tmult*tm[j]
-
- ;reformat the multiplier e.g. 1.000E+003 becomes 1E3
- ;convert to Exponential form if necessary
- IF (tmult EQ 273.15D) OR (tmult MOD 60.D EQ 0.D) OR ((tmult GE 0.01D) AND (tmult LT 1.D2) $
- AND (tmult*1.D4 MOD 1.D2 EQ 0.D)) THEN tmults=STRTRIM(STRUPCASE(tmult),2) $ ;keep the same format
- ELSE tmults=STRTRIM(STRING(format='(E14.6)',tmult),2)
-
- epos=STRPOS(tmults,'E') & ppos=STRPOS(tmults,'.')
- IF epos NE -1 THEN BEGIN ;remove unnecessary characters after 'E'
- ep=FIX(STRMID(tmults,epos+1)) & epx=STRTRIM(ep,2)
- tmults=STRMID(tmults,0,epos+1)+epx
- ENDIF
- IF ppos NE -1 THEN BEGIN ;remove any trailing zeroes after the decimal place
- IF epos NE -1 THEN ep=STRMID(tmults,ppos+1,epos-(ppos+1)) ELSE ep=STRMID(tmults,ppos+1)
- WHILE STRMID(ep,STRLEN(ep)-1,1) EQ '0' DO ep=STRMID(ep,0,STRLEN(ep)-1)
- IF ep NE '' THEN tmults=STRMID(tmults,0,ppos+1)+ep ELSE tmults=STRMID(tmults,0,ppos)
- IF epos NE -1 THEN tmults=tmults+'E'+epx
- ENDIF
-
- ;Scale the units by the power e.g. W2 = (m2 kg s-3)^2
- FOR j=0,vpn-1 DO BEGIN
- vsc=STRSPLIT(bpx[j],';',/EXTRACT)
- vspl=STRSPLIT(vsc[2],' ',/EXTRACT,COUNT=vscnt)
- sichk=STRARR(vscnt) & pwchk=sichk
- IF (vpx[1,j] NE '') AND (vpx[1,j] NE '1') AND (vsc[2] NE '1') THEN BEGIN
- bpx[j]=vsc[0]+';'+vsc[1]+';'
- FOR k=0,vscnt-1 DO BEGIN
- ;separate out SI units and power values
- sires=STRSPLIT(vspl[k],'-0123456789',/Extract)
- sichk[k]=sires[0] ;SI Unit
- IF STRLEN(sichk[k]) NE STRLEN(vspl[k]) THEN $
- pwchk[k]=STRMID(vspl[k],STRLEN(sichk[k])) $
- ELSE pwchk[k]='1' ;Power value
- pwm=FIX(pwchk[k])*FIX(vpx[1,j])
- IF k EQ 0 THEN sp='' ELSE sp=' '
- bpx[j]=bpx[j]+sp+sichk[k]+STRTRIM(pwm,2)
- ENDFOR
- ENDIF
- ENDFOR
-
- ;Put together VAR_SI_CONVERSION
- vsc=STRSPLIT(bpx[0],';',/EXTRACT)
- ;IF vsc[2] EQ '1' THEN vpx[1,0]=''
- var_si_conv[0]=vsc[0]+';'+tmults+';'+vsc[2]
- ;add remaining base units to VAR_SI_CONVERSION
- IF vpn GT 1 THEN BEGIN
- FOR j=1,vpn-1 DO BEGIN
- vsc=STRSPLIT(bpx[j],';',/Extract)
- var_si_conv[0]=var_si_conv[0]+' '+vsc[2]
- ENDFOR
- ENDIF
- var_si_conv[1]=var_si_conv[0] ;Default is that the 2nd entry is the same as the first
-
- ;check VAR_SI_CONVERSION for repeated SI units e.g. m m-3 becomes m-2
- schk=STRSPLIT(var_si_conv[0],' ;',/Extract) & scnt=N_ELEMENTS(schk)
- IF scnt GT 3 THEN BEGIN ;more than one SI unit in VAR_SI_CONVERSION
- sichk=STRARR(scnt-2) & pwchk=sichk
- FOR j=0,scnt-3 DO BEGIN ;separate out SI units and power values
- sires=STRSPLIT(schk[j+2],'-0123456789',/Extract)
- sichk[j]=sires[0] ;SI Unit
- IF STRLEN(sichk[j]) NE STRLEN(schk[j+2]) THEN $
- pwchk[j]=STRMID(schk[j+2],STRLEN(sichk[j])) $
- ELSE pwchk[j]='1' ;Power value
- ENDFOR
- j=0 & pwval=0
- WHILE j LT scnt-3 DO BEGIN
- si=WHERE((sichk[j] NE '') AND (sichk[j] EQ sichk[j+1:scnt-3]),sicnt)
- IF sicnt NE 0 THEN BEGIN
- FOR k=0,sicnt-1 DO BEGIN
- si[k]=si[k]+j+1
- pwval=FIX(pwchk[j])+FIX(pwchk[si[k]])
- IF (pwval[0] EQ 0) AND (k EQ sicnt-1) THEN BEGIN
- sichk[j]='' & pwchk[j]=''
- ENDIF ELSE IF (pwval[0] EQ 1) and (k EQ sicnt-1) THEN pwchk[j]='' $
- ELSE pwchk[j]=STRTRIM(pwval[0],2)
- ;make null all the other SI values
- sichk[si[k]]='' & pwchk[si[k]]=''
- ENDFOR
- ENDIF ELSE IF pwchk[j] EQ '1' THEN pwchk[j]=''
- j=j+1
- ENDWHILE
- ;Also do check on the power value of the last SI Unit
- IF pwchk[scnt-3] EQ '1' THEN pwchk[scnt-3]=''
-
- ;Put units in power value order
- ;Create two possible versions, one using the MAX command (primary) and another using SORT
- ;Note: When using the SORT command, identical elements are sorted arbitrary and may vary between operating systems
- FOR k=0,1 DO BEGIN
- pwhold=pwchk
- oi=WHERE(pwhold EQ '',ocnt)
- IF ocnt NE 0 THEN pwhold[oi]='1'
- IF k EQ 0 THEN BEGIN ;Do MAX order method
- pws=INTARR(scnt-2)
- ;determine minimum power value
- mnp=MIN(FIX(pwhold))
- FOR j=0,scnt-3 DO BEGIN
- mxp=MAX(FIX(pwhold),mxs)
- pws[j]=mxs
- pwhold[mxs]=mnp - 1 ;make it the lowest value
- ENDFOR
- sichk=sichk[pws] & pwchk=pwchk[pws]
- ENDIF ELSE BEGIN ;Do SORT method (results are operating system dependent)
- pws=SORT(FIX(pwhold))
- sichk=sichk[REVERSE(pws)] & pwchk=pwchk[REVERSE(pws)]
- ENDELSE
- var_si_conv[k]=schk[0]+';'+schk[1]+';'+sichk[0]+pwchk[0]
- si=WHERE(sichk NE '',sicnt)
- IF sicnt EQ 0 THEN var_si_conv[k]=var_si_conv[k]+'1' $ ;i.e. values cancelled out
- ELSE BEGIN
- FOR j=1,scnt-3 DO BEGIN
- si=WHERE(sichk[0:j-1] NE '',sicnt)
- IF (sichk[j] EQ '') OR (sicnt EQ 0) THEN var_si_conv[k]=var_si_conv[k]+sichk[j]+pwchk[j] $
- ELSE var_si_conv[k]=var_si_conv[k]+' '+sichk[j]+pwchk[j]
- ENDFOR
- ENDELSE
- ENDFOR
- ENDIF
-
- IF rd LT 0 THEN BEGIN
- ;VAR_DATA_TYPE is Real or Double so make VAR_SI_CONVERSION values floating point
- FOR k=0,1 DO BEGIN
- tchkh=STRSPLIT(var_si_conv[k],';',/Extract) & tup=STRUPCASE(tchkh)
- FOR j=0,1 DO BEGIN
- IF STRPOS(tup[j],'.') EQ -1 THEN BEGIN
- IF STRPOS(tup[j],'E') EQ -1 THEN tchkh[j]=tchkh[j]+'.0' $
- ELSE tchkh[j]=STRMID(tup[j],0,STRPOS(tup[j],'E'))+'.0'+STRMID(tup[j],STRPOS(tup[j],'E'))
- ENDIF
- ENDFOR
- var_si_conv[k]=tchkh[0]+';'+tchkh[1]+';'+tchkh[2]
- ENDFOR
- ENDIF
-
- ;Check for invalid VAR_UNITS - third VAR_SI_CONVERSION is 1 plus extra values
- FOR k=0,1 DO BEGIN
- vsc=STRSPLIT(var_si_conv[k],';',/EXTRACT)
- vsc[2]=STRTRIM(vsc[2],2)
- IF (errstate EQ '') AND (STRMID(vsc[2],0,1) EQ '1') AND (STRLEN(vsc[2]) GT 1) THEN BEGIN
- errstate=errtxt[1] & var_si_conv[k]=''
- ENDIF
- ENDFOR
-ENDIF
-
-END ;procedure Var_Units_Test
-
-
-
-PRO read_tablefile, tablefile
-;Procedure to identify the version number of the TAV file, read the contents of the
-;LABELS/FIELDS (tab_arr), and determine the FILE_META_VERSION in the global attributes (tab_ver).
-;This routine also creates a flag (tab_type) to determine whether the input file is an original
-;table.dat file used by Envisat or the GEOMS TAV file, with the HDF/netCDF file generated
-;accordingly.
-; ----------
-;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
-;
-;History:
-; 20050802: Original IDLCR8HDF Routine - Version 1.0
-; 20061012: Bug-fix to separate semi-colons with a space when one immediately follows the other,
-; so that the number of subvalues is correctly determined by the program. Common
-; variable definition WIDGET_WIN added - Version 2.0
-; 20080302: Added code to differentiate between the AVDC TAV file and original Envisat table.dat.
-; The tab_type flag is used to differentiate between the two formats. Note that some of the
-; table.dat labels are renamed to match the equivalent TAV file labels for compatibility
-; when testing Metadata entries - refer to EnviName/AVDCName arrays; Ensure that the TAV
-; Version value is correctly formatted and is version 03 or greater - Version 3.0
-; 20100205: Add RETURN command after all STOP_WITH_ERROR calls, which allows program to return to the
-; calling program if the reterr argument is included in the idlcr8hdf call - Version 3.09
-; 20110401: Change AVDC references to GEOMS; Test AVDC TAV file version is version 04 or greater;
-; Remove check on format of TAV file version; Conform to new INFOTXT_OUTPUT reporting
-; - Version 4.0b1
-;
-;Input: Tablefile - Name of the file containing the Table Attributes.
-;
-;Outputs: tab_ver - String containing FILE_META_VERSION value.
-; tab_arr - 2-D string array of size nf*mcnt+2, where nf=Number of Fields, and mcnt=maximum
-; number of values detected in any one field. tab_arr(*,0) is the name of each field,
-; and tab_arr(*,1) is the number of values in each field.
-; tab_type - 0/1 where 0 identifies an AVDC format TAV file and 1 identifies an original
-; Envisat format table.dat file.
-;
-;Called by: IDLCR8HDF
-;
-;Subroutines Called: STOP_WITH_ERROR (if error state detected); INFOTXT_OUTPUT
-; Possible Conditions for STOP_WITH_ERROR call (plus [line number] where called):
-; 1. Table Attribute Values file version not identified
-; 2. Envisat table.dat ASC2HDF program version not found
-; 3. First Envisat table.dat field value not found
-; 4. Table Attribute Values file version is not in a valid format
-;
-; Information Conditions (when the program reports issues and continues):
-; 1. [Original EVDC]/[GEOMS] Reporting Guidelines Apply (depending on type of Table Attribute
-; Values file read in)
-; 2. Old version of the TAV file in use. Update from http://avdc.gsfc.nasa.gov/Tools
-
-COMMON TABLEDATA
-COMMON WIDGET_WIN
-
-;Possible error messages for this procedure
-proname='Read_TableFile procedure: '
-errtxt=STRARR(4)
-errtxt[0]='Table Attribute Values file version not identified'
-errtxt[1]='Envisat table.dat ASC2HDF program version not found'
-errtxt[2]='First Envisat table.dat field value not found'
-errtxt[3]='Table Attribute Values file version is invalid (should be ddRddd or ddRdddd): '
-FOR i=0,2 do errtxt[i]=errtxt[i]+' with the search criteria used by this program'
-
-;Array of Envisat Field names to be changed to equivalent AVDC Field names
-enviname=['_NAME','_AFFILIATION','DATA_VARIABLES_00_00','BASE_UNIT']
-avdcname=['ORIGINATOR','AFFILIATION','DATA_VAR_ALL_00_00','VAR_UNITS']
-
-ON_IOERROR,TypeConversionError
-dum='' & tab_ver=''
-min_fmv=4 ;TAV Version must be GE this value e.g. 04R001, 06R002 but not 03R004
-OPENR,lu,tablefile,/GET_LUN
-;determine TAV file version
-REPEAT BEGIN
- READF,lu,dum
- dumup=STRUPCASE(dum)
- envitest=STRPOS(STRCOMPRESS(dumup,/Remove_all),'!TABLE.DATVERSION') NE -1
- avdctest=STRPOS(STRCOMPRESS(dumup,/Remove_all),'!VERSION') NE -1
-ENDREP UNTIL (envitest) OR (avdctest) OR (EOF(lu))
-IF EOF(lu) THEN BEGIN
- STOP_WITH_ERROR,o3[3]+proname,errtxt[0],lu & RETURN
-ENDIF
-res=STRSPLIT(dumup,' ',/Extract) & vi=WHERE(res EQ 'VERSION')
-IF N_ELEMENTS(res) LE vi[0]+1 THEN BEGIN
- STOP_WITH_ERROR,o3[3]+proname,errtxt[0]+': '+dum,lu & RETURN
-ENDIF
-IF envitest THEN BEGIN
- tab_type=1 & infotxt='0 EVDC original style table.dat'
- n_title=4
-ENDIF ELSE BEGIN
- tab_type=0 & infotxt='0 GEOMS compliant Table Attribute Values'
- n_title=5
-ENDELSE
-infotxt=infotxt+' file input. '+STRMID(infotxt,2,n_title)+' Reporting Guidelines apply'
-INFOTXT_OUTPUT,infotxt
-
-;Ensure Meta Version has a valid format
-;i. Check third character is an 'R'
-;ii. Check number of characters is 6 or 7
-;iii. Check dd and ddd(d) are numeric
-;iv. Check that version number is 03 or greater for AVDC file type
-;v. Issue warning and convert to ddRddd if format is ddRdddd
-fmv=res[vi[0]+1] ;File_Meta_Version
-valid=0 ;set to test for valid string to number conversion
-FOR i=0,STRLEN(fmv)-1 DO IF i NE 2 THEN fmvtest=FIX(STRMID(fmv,i,1))
-fmvtest=FIX(STRMID(fmv,0,2)) ;To test for TAV version 'min_fmv' or greater
-valid=1 ;FILE_META_VERSION characters are numeric (except for the 'R') if program gets to here
-TypeConversionError:
-IF (STRMID(fmv,2,1) NE 'R') OR (STRLEN(fmv) lt 6) OR (STRLEN(fmv) gt 7) OR $
- (valid EQ 0) THEN BEGIN
- STOP_WITH_ERROR,o3[3]+proname,errtxt[3]+fmv,lu & RETURN
-ENDIF
-
-IF (tab_type EQ 0) AND (fmvtest LT min_fmv) THEN BEGIN
- infotxt=STRARR(2)
- infotxt[0]='3 Old version of the Table Attribute Values file used as input: '
- infotxt[0]=infotxt[0]+' TAV Version '+fmv
- infotxt[1]=' Please update from http://avdc.gsfc.nasa.gov/Tools'
- INFOTXT_OUTPUT,infotxt
-ENDIF
-
-IF envitest THEN BEGIN ;identify asc2hdf version in table.dat and read past the CHECK_ATTRIBUTE line
- WHILE (STRPOS(STRUPCASE(dum),'ASC2HDF') EQ -1) AND (NOT EOF(lu)) DO READF,lu,dum
- IF EOF(lu) THEN BEGIN
- STOP_WITH_ERROR,o3[3]+proname,errtxt[1],lu & RETURN
- ENDIF
- cr8_hdf_ver=';IDLCR8HDF'
- dum=STRTRIM(dum,2)
- WHILE ((STRMID(dum,0,1) EQ '!') OR (STRMID(dum,0,1) EQ '#') OR (dum EQ '')) AND $
- (NOT EOF(lu)) DO BEGIN
- READF,lu,dum & dum=STRTRIM(dum,2)
- ENDWHILE
- IF EOF(lu) THEN BEGIN
- STOP_WITH_ERROR,o3[3]+proname,errtxt[2],lu & RETURN
- ENDIF
- READF,lu,dum ;to get to the line after 'CHECK_ATTRIBUTE'
-ENDIF ELSE cr8_hdf_ver=';IDLCR8HDF'
-tab_ver=fmv+cr8_hdf_ver ;= the FILE_META_VERSION input value in the global attributes
-
-;determine no. of FIELDS/LABELS as well as the maximum number of elements
-nf=0 & mcnt=0 & firstfield=''
-dum=STRTRIM(dum,2)
-WHILE NOT EOF(lu) DO BEGIN
- ncnt=-1
- IF (STRMID(dum,0,1) NE '!') AND (STRMID(dum,0,1) NE '#') AND $
- (STRMID(dum,0,1) NE '=') AND (dum NE '') THEN BEGIN
-
- FOR i=0,N_ELEMENTS(enviname)-1 DO BEGIN
- epos=STRPOS(STRUPCASE(dum),enviname[i])
- IF (epos NE -1) AND (epos LE 3) THEN ncnt=i
- ENDFOR
- IF (ncnt EQ 0) OR (ncnt EQ 1) THEN BEGIN
- IF STRMID(STRUPCASE(dum),0,3) NE 'PI_' THEN nf=nf-1 ELSE ecnt=0
- ;This puts all PI_,DO_, and DS_NAME or AFFILIATION values into either the ORIGINATOR or AFFILIATION fields
- ENDIF ELSE ecnt=0
-
- nf=nf+1
- IF firstfield EQ '' THEN firstfield=dum
- IF ncnt NE -1 THEN dum=avdcname[ncnt] ;change the name of the Envisat field to AVDC equivalent
- REPEAT BEGIN
- READF,lu,dum & dum=STRTRIM(dum,2)
- IF STRMID(dum,0,1) EQ '=' THEN ecnt=ecnt+1
- ENDREP UNTIL (STRMID(dum,0,1) NE '=') OR (EOF(lu))
- IF ecnt GT mcnt THEN mcnt=ecnt
- ENDIF ELSE IF NOT EOF(lu) THEN BEGIN
- READF,lu,dum & dum=STRTRIM(dum,2)
- ENDIF
-ENDWHILE
-FREE_LUN,lu
-
-;read in the contents of the file
-tab_arr=STRARR(nf,mcnt+2) ;note tab_arr(*,0) EQ FIELD/LABEL name and
-;tab_arr(*,1) EQ N_ELEMENTS under each FIELD/LABEL
-tab_hold=STRARR(mcnt)
-
-OPENR,lu,tablefile,/GET_LUN
-READF,lu,dum & dum=STRTRIM(dum,2)
-WHILE dum NE firstfield DO BEGIN
- READF,lu,dum & dum=STRTRIM(dum,2)
-ENDWHILE
-
-i=0
-WHILE i LE nf-1 DO BEGIN
- ncnt=-1
- FOR j=0,N_ELEMENTS(enviname)-1 DO BEGIN
- epos=STRPOS(STRUPCASE(dum),enviname[j])
- IF (epos NE -1) AND (epos LE 3) THEN ncnt=j
- ENDFOR
- IF (ncnt EQ 0) OR (ncnt EQ 1) THEN BEGIN
- IF STRMID(STRUPCASE(dum),0,3) NE 'PI_' THEN i=i-1 ELSE ecnt=0
- ENDIF ELSE ecnt=0
- IF ncnt NE -1 THEN dum=avdcname[ncnt] ;change the name of the Envisat field to AVDC equivalent
- tab_arr[i,0]=dum
- REPEAT BEGIN
- READF,lu,dum & dum=STRTRIM(dum,2)
- IF STRMID(dum,0,1) EQ '=' THEN BEGIN
- tab_hold[ecnt]=STRMID(dum,1) ;strip the '=' sign
- ;add space between adjacent semi-colons so StrSplit finds correct number of sub-values
- REPEAT BEGIN
- cpos=STRPOS(tab_hold[ecnt],';;')
- IF cpos NE -1 THEN tab_hold[ecnt]=STRMID(tab_hold[ecnt],0,cpos+1)+ $
- ' '+STRMID(tab_hold[ecnt],cpos+1)
- ENDREP UNTIL cpos EQ -1
- ecnt=ecnt+1
- ENDIF
- ENDREP UNTIL (STRMID(dum,0,1) NE '=') OR (EOF(lu))
- tab_arr[i,1]=STRTRIM(ecnt,2)
- tab_arr[i,2:ecnt+1]=tab_hold[0:ecnt-1]
- IF i NE nf-1 THEN BEGIN
- WHILE (STRMID(dum,0,1) EQ '!') OR (STRMID(dum,0,1) EQ '#') OR (dum EQ '') DO BEGIN
- READF,lu,dum & dum=STRTRIM(dum,2)
- ENDWHILE
- ENDIF
- i=i+1
-ENDWHILE
-FREE_LUN,lu
-
-END ;Procedure Read_TableFile
-
-
-
-PRO test_file_input,aname,fentry
-;Procedure to test that a file name given as an entry to a free text attribute is valid, based on Envisat
-;Metadata Guidelines (not currently used by AVDC). Note: No longer permitted under GEOMS guidelines.
-; ----------
-;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
-;
-; History:
-; 20050802: Original IDLCR8HDF Routine - Version 1.0
-; 20061012: Common variable definition WIDGET_WIN added - Version 2.0
-; 20100205: Add RETURN command after all STOP_WITH_ERROR calls, which allows program to return to the
-; calling program if the reterr argument is included in the idlcr8hdf call - Version 3.09
-;
-; Inputs: aname - Global or Variable Attribute Label
-; fentry - Filename entry used as the Global or Variable Attribute value
-;
-; Outputs: None
-;
-; Called by: READ_METADATA
-;
-; Subroutines Called: STOP_WITH_ERROR (if error state detected)
-; Possible Conditions for STOP_WITH_ERROR call (plus [line number] where called):
-; 1. Syntax of Filename entry incorrect
-; 2. Filename entry not found or not usable
-; 3. File size is too large
-
-COMMON WIDGET_WIN
-
-sfile=4096 ;maximum permitted file size
-
-;possible error message for this procedure
-proname='Test_File_Input procedure: '
-errtxt2=STRARR(3) & lu=-1
-errtxt2[0]='Syntax of Filename entry incorrect: '
-errtxt2[1]='Filename entry not found or not usable: '
-errtxt2[2]='File size is too large (maximum permitted: '+STRTRIM(sfile,2)+' bytes)'
-
-si=STRPOS(fentry,'"')+1 & ei=STRPOS(fentry,'"',/Reverse_Search)-si
-IF ei EQ si THEN BEGIN
- STOP_WITH_ERROR,o3[3]+proname+aname+': ',errtxt2[0]+fentry,lu & RETURN
-ENDIF
-faname=STRMID(fentry,si,ei)
-ftest=FILE_TEST(faname,/Read,/Regular)
-IF ftest EQ 0 THEN BEGIN
- STOP_WITH_ERROR,o3[3]+proname+aname+': ',errtxt2[1]+faname,lu & RETURN
-ENDIF
-OPENR,fu,faname,/GET_LUN
-ftest=FSTAT(fu) & FREE_LUN,fu
-IF ftest.size GT sfile THEN BEGIN
- STOP_WITH_ERROR,o3[3]+proname+AName+'='+fentry+': ',errtxt2[2],lu & RETURN
-ENDIF
-
-END ;procedure Test_File_Input
-
-
-
-PRO geoms_rule_changes, code, in1, in2, in3, in4
-;Procedure to check Metadata for old/redundant rules, labels and/or values and update/report
-;as required
-; ----------
-;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
-;
-; History:
-; 20110401: Introduced. Incorporates GEOMS rules changes from v3.0 to v4.0 - Version 4.01
-; 20111220: Add ISO646-US ASCII character set check - Version 4.0b7
-; 20120313: Add UVVIS.DOAS plus additional gases to Code 6 checks - Version 4.0b8
-; 20120703: Bug Fix when checking for non-ASCII characters (rule 10) - Version 4.0b11
-; 20131023: Add GEOMS rule change for MIXING.RATIO[.VOLUME][.MASS], and UNCERTAINTY (rule 5);
-; Fixed bug that caused incorrect rep_list_2 values to be written to file (rule 5);
-; Stopped checks for illegal characters if ORIGINATOR values are present in the
-; TAV file (rule 10) - Version 4.0b16
-; 20131029: Add array of invalid ASCII characters that can be replaced with valid ISO646 US
-; ASCII characters (rule 10) - Version 4.0b17
-; 20131029: Modify UNCERTAINTY rules to allow more exceptions (rule 5); Allow for renaming of
-; LIDAR.DIAL DATA_SOURCE to LIDAR.WATERVAPORDIAL if conditions met (rule 6) -
-; Version 4.0b18
-; 20140226: Add 'DU' to obsolete descriptor value and fix bug causing no new variable name to
-; be generated if more than 3 sub-values in the name (rule 5); Add 'RO.SAC.C' to
-; the obsolete DATA_SOURCE list (rule 6) - Version 4.0b19
-; 20140521: Account for extra parameter in the UVVIS-DOAS data source when trying to determine
-; if file is based on a DATA_TEMPLATE (rule 9) - Version 4.0b21
-; 20140806: Modify UNCERTAINTY rules to allow more exceptions including additional instrument
-; types and check on molec cm-2 to allow for prefix values (rule 5) - Version 4.0b22
-; 20140909: Add RO.F3C.FM[1][2][3] to the obsolete list of DATA_SOURCES (replaced by
-; F3C.FM[1][2][3]) (rule 6) - Version 4.0b23
-; 20141110: Numerous changes while testing harmonization of EVDC files - additional
-; DATA_SOURCEs and units added to identify correct new UNCERTAINTY values, including
-; identification of covariance uncertainties; Bug fix when doing checks on
-; AIR.CONCENTRATION; LIDAR.BACKSCATTER conversion and units definition added; List
-; of non-ASCII characters to be checked expanded; Bug fix when correcting
-; for non-ASCII characters when the replacement value has more than one character
-; e.g. ' deg. ' - Version 4.0b24
-; 20150409: Strip trailing and leading spaces from DATA_TEMPLATE value when doing checks
-; (rule 9) - Version 4.0b28
-; 20160213: Add extra satellite instruments to the obsolete list for DATA_SOURCE (rule 6);
-; Fix issue with DATA_TEMPLATE identification (rule 9) - Version 4.0b37
-; 20171121: Fix bug that selected the incorrect DATA_TEMPLATE in some situations (rule 9)
-; - Version 4.0b43
-; 20190506: Add FILE_META_VERSION information to INFORMATION/ERROR comment when
-; DATA_TEMPLATE value is not as expected; Fix bug that caused a crash if the
-; file does not use a DATA_TEMPLATE (rule 9) - Version 4.0b50
-; 20191205: Add rule 11 which does checks on optional VERSION_NAME sub-field of
-; DATA_SOURCE - Version 4.0b53
-; 20200709: Fix in option 5 when looking for obsolete variable names. Change from doing a
-; STRPOS search to looking for an exact change for DATA_VARIABLES_01 names due to
-; introduction of DRY.AIR.... names to the TAV - Version 4.0b55
-; 20201020: Change infotxt status from 1 to 2 in option 5; Change to LIDAR.H2O data source
-; if needing to update out-of-date H2O LIDAR DATA_SOURCE values in option 6
-; - Version 4.0b56
-; 20220805: Change INFOTXT message in rule 10 (non-ASCII character check) so it includes
-; the entire Metadata entry - Version 4.0b60
-;
-; Inputs: code - Integer value identifying type of check to carry out
-; in1 - First set of inputs required for checks (optional, dependent on code value)
-; in2 - Second set of inputs required for checks (optional, dependent on code value)
-; in3 - Third set of inputs required for checks (optional, dependent on code value)
-; in4 - Fourth set of inputs required for checks (optional, dependent on code value)
-;
-; Outputs: Any or all of the inputs can be changed dependent on the code value
-;
-; Called by: READ_METADATA; CHECK_METADATA; SET_UP_STRUCTURE; FIND_HDF_FILENAME
-;
-; Subroutines Called: INFOTXT_OUTPUT
-; Information Conditions (when the program is able to make changes):
-; 1. Attribute entry is not GEOMS compliant, and removed from the metadata saved to the output file
-; 2. Dataset name renamed according to new DATETIME reporting conventions
-; 3. Double underscore replaced with a single underscore in the variable name
-; 4. VAR_DEPEND value made self-referencing for axis variable
-; 5. Axis variable cannot be dependent on another variable
-; 6. VAR_UNITS=MJD2000 not GEOMS compliant, changed to VAR_UNITS=MJD2K
-; 7. DATA_FILE_VERSION must be in the form 'nnn'
-; 8. Obsolete Dataset name renamed
-; 9. Obsolete DATA_SOURCE value renamed
-; 10. Obsolete FILE_ACCESS values renamed
-; 11. Rename obsolete VAR_UNITS=DIMENSIONLESS/NONE values
-; 12. DATA_TEMPLATE field is not present in the TAV file
-; 13. DATA_TEMPLATE value renamed based on DATA_SOURCE value
-; 14. Entry must only include valid characters from the ISO646-US ASCII character set
-;
-; Code 0: Check for Attributes that have become redundant between v3.0 and v4.0.
-; Inputs: in1=mh, in2=mhgood
-; Code 1: Check for redundant _START/STOP/INTEGRATION.TIME variable names and replace with
-; DATETIME.START/STOP/INTEGRATION.TIME (for single occurences only). Also check
-; for double underscores in the variable names and replace with single underscore
-; Inputs: in1=vncnt, in2=dv
-; Code 2: Check for Axis Variables and, if found, make VAR_DEPEND=INDEPENDENT self-referencing
-; Inputs: in1=vardeptest, in2=vn[vc], in3=resvd[1], in4=holdvd
-; Code 3: Change VAR_UNIT value MJD2000 to MJD2K
-; Inputs: in1=meta_arr[i], in2=res[1], in3=writeonce
-; Code 4: Do DATA_FILE_VERSION checks
-; Inputs: in1=dfvv (DATA_FILE_VERSION value)
-; Code 5: Look for obsolete DATA_VARIABLES (based on list) and, if possible, modify values to
-; GEOMS compliance
-; Inputs: in1=vncnt, in2=dv, in3=vuv, in4=data_source
-; Code 6: Look for obsolete DATA_SOURCE (based on list) and, if possible, modify values to
-; GEOMS compliance
-; Inputs: in1=vncnt, in2=dv, in3=data_source
-; Code 7: Looks for and replaces obsolete CALVAL and NDSC FILE_ACCESS values
-; Inputs: in1=facnt, in2=fav
-; Code 8: Looks for VAR_UNITS=DIMENSIONLESS or NONE and replace with '' or '1'
-; Inputs: in1=vucnt, in2=vuv, in3=vdv
-; Code 9: Do DATA_TEMPLATE/DATA_QUALITY checks
-; Inputs: in1=m_v0, in2=m_v1, in3=ga_chk
-; Code 10: Do checks on ISO646-US ASCII character set
-; Inputs: in1=meta_arr or dtest, in2=vn[vc] (Variable Name for datasets only)
-; Code 11: Do checks on DATA_VERSION_NAME part of DATA_SOURCE value
-; Inputs: in1=DATA_VERSION_NAME, in2=meta_arr
-
-COMMON TABLEDATA
-COMMON METADATA
-COMMON DATA
-COMMON WIDGET_WIN
-
-CASE 1 OF
- code EQ 0: BEGIN
- ;Check for redundant attributes
- redundant=['DATA_TYPE','DATA_LEVEL','VAR_MONOTONE','VAR_DIMENSION',$
- 'VAR_AVG_TYPE','VIS_LABEL','VIS_FORMAT','VIS_PLOT_TYPE',$
- 'VIS_SCALE_TYPE','VIS_SCALE_MIN','VIS_SCALE_MAX']
- nrd=N_ELEMENTS(redundant)
-
- FOR i=0,nrd-1 DO BEGIN
- rdl=STRLEN(redundant[i])
- fai=WHERE(STRMID(STRUPCASE(in1),0,rdl) EQ redundant[i],facnt)
- IF facnt NE 0 THEN BEGIN
- in2[fai]=0 ;Metadata lines not wanted
- test1=(STRMID(redundant[i],0,3) EQ 'VAR') OR (STRMID(redundant[i],0,3) EQ 'VIS')
- IF test1 THEN att_type='Variable' ELSE att_type='Global'
- infotxt='2 '+redundant[i]+' not a GEOMS '+att_type+' attribute|.'
- infotxt=infotxt+' Removed from the metadata saved to the output file'
- INFOTXT_OUTPUT, infotxt
- ENDIF
- ENDFOR
- END ;Case 0
-
- code EQ 1: BEGIN
- ;Check for redundant _START/STOP sub-values and replace with
- ;DATETIME.START/STOP/INTEGRATION.TIME (for single occurences only)
- time_attr=['START','STOP','INTEGRATION','RESOLUTION']
- n_tim=N_ELEMENTS(time_attr)
- iscnt=0 ;Used for RESOLUTION/INTEGRATION.TIME checks
- FOR i=0,n_tim-1 DO BEGIN
- si=WHERE(STRPOS(in2,'_'+time_attr[i]+'.TIME') NE -1,scnt)
- IF (scnt EQ 1) AND (iscnt EQ 0) THEN BEGIN ;change name and also relevant VAR_NAME if present
- IF i EQ 2 THEN iscnt=1 ;used in the event both INTEGRATION and RESOLUTION.TIME are present
- vnchange[si[0]]=in2[si[0]]
- IF i GE 2 THEN in2[si[0]]=time_attr[2]+'.TIME' $
- ELSE in2[si[0]]='DATETIME.'+time_attr[i]
- IF qa_yes THEN itxt=' expected to be ' ELSE itxt=' renamed '
- infotxt='2 Dataset name '+vnchange[si[0]]+itxt+in2[si[0]]
- INFOTXT_OUTPUT,infotxt
- ENDIF
- ENDFOR
- ;Check for double underscore and replace with single underscore
- FOR i=0,in1-1 DO BEGIN
- IF STRPOS(in2[i],'__') NE -1 THEN BEGIN
- vnchange[i]=in2[i]
- infotxt=STRARR(2)
- infotxt[0]='2 Double underscore present in Variable Name '+in2[i]+'|'
- infotxt[1]=' replaced with a single underscore'
- INFOTXT_OUTPUT, infotxt
- vns=STRSPLIT(in2[i],'_',/Extract,COUNT=n_vns)
- FOR j=0,n_vns-1 DO IF j EQ 0 THEN in2[i]=vns[j] ELSE in2[i]=in2[i]+'_'+vns[j]
- ENDIF
- ENDFOR
- END ;Case 1
-
- code EQ 2: BEGIN ;Check for and Apply GEOMS rule changes for INDEPENDENT and axis variables
- ;RULE: Any variable that is mentioned in VAR_DEPEND is by definition an axis variable
- vdtest=STRUPCASE(in1) & vnv=STRUPCASE(in2) & vdv=STRUPCASE(in3) ;vdtest is an array of VAR_DEPEND variables
- avi=WHERE(vnv EQ vdtest,avcnt) ;i.e. is the VAR_NAME an axis variable
- IF avcnt NE 0 THEN BEGIN ;possible axis variable so do axis variable checks
- test1=(vdv EQ 'INDEPENDENT') OR (vdv EQ vnv) ;value is independent or self-referencing
- test2=(vdv NE 'INDEPENDENT') AND (vdv NE vnv) AND (vdv NE 'CONSTANT') ;neither independent, self-referencing, nor constant
- test6=(vdv EQ 'CONSTANT') AND (vnv EQ 'DATETIME') ;i.e. VAR_NAME=DATETIME and VAR_DEPEND=CONSTANT
- IF (test1) OR (test6) THEN BEGIN
- IF vdv NE vnv THEN BEGIN
- infotxt='2 '+in4+' should be self-referencing for axis variable VAR_NAME='+vnv+'|.'
- infotxt=infotxt+' VAR_DEPEND value changed in metadata'
- INFOTXT_OUTPUT,infotxt
- in3=vnv ;self-referencing VAR_DEPEND value
- ENDIF
- ENDIF ELSE IF test2 THEN BEGIN
- in4[0]='E1' ;i.e. axis variable cannot be dependent on another variable so return error
- ENDIF
- ENDIF ELSE BEGIN ;not an axis variable so ensure VAR_DEPEND value is not self-referencing
- IF vdv EQ vnv THEN in4[0]='E3' ;i.e. VAR_DEPEND value cannot be self-referencing
- ENDELSE
- END ;Case 2
-
- code EQ 3: BEGIN ;Change MJD2000 to MJD2K
- in1='VAR_UNITS=MJD2K' & in2='MJD2K'
- IF in3 EQ 0 THEN BEGIN ;First instance of MJD2000 so include comment
- infotxt='2 VAR_UNITS=MJD2000 not GEOMS compliant|. Changed to VAR_UNITS=MJD2K'
- INFOTXT_OUTPUT,infotxt
- ENDIF
- END
-
- code EQ 4: BEGIN ;checks on DATA_FILE_VERSION
- in1=STRTRIM(in1,2)
- vp=0 & errval=0 & in1h=in1
- ON_IOERROR,TypeConversionError
- IF STRMID(STRUPCASE(in1),0,1) EQ 'V' THEN BEGIN
- vp=1 & in1=STRMID(in1,1)
- ENDIF
- dpos=STRPOS(in1,'.')
- valid=0 ;to check for conversion error
- IF dpos NE -1 THEN BEGIN
- IF LONG(STRMID(in1,dpos+1)) EQ 0L THEN atxt='' ELSE atxt=STRMID(in1,dpos+1)
- in1=STRMID(in1,0,dpos)+atxt
- IF FIX(in1) GE 100 THEN in1=STRMID(in1,0,dpos) ;just use version up to the decimal point
- ENDIF
- IF (FIX(in1) GT 999) OR (FIX(in1) LT 1) THEN errval=1
- valid=1 ;OK if got to here
-
- TypeConversionError: ;Check for invalid DATA_FILE_VERSION value
- IF (valid EQ 0) OR (errval EQ 1) THEN BEGIN
- infotxt='3 DATA_FILE_VERSION='+in1h+' must be in the form ''nnn'''
- INFOTXT_OUTPUT, infotxt
- in1=in1h
- ENDIF ELSE BEGIN ;format value correctly (nnn)
- IF FIX(in1) LT 10 THEN in1='00'+STRTRIM(FIX(in1),2) $
- ELSE IF FIX(in1) LT 100 THEN in1='0'+STRTRIM(FIX(in1),2) $
- ELSE in1=STRTRIM(in1,2)
- IF (vp EQ 1) OR (in1h NE in1) THEN BEGIN
- infotxt='2 DATA_FILE_VERSION must be in the form ''nnn''|. Renamed from '
- infotxt=infotxt+in1h+' to '+in1
- INFOTXT_OUTPUT, infotxt
- ENDIF
- ENDELSE
- END
-
- code EQ 5: BEGIN ;Look for obsolete DATA_VARIABLES
- ;Notes = what to do about ALTITUDE.LAYER.INDEX, CHL.A.CONCENTRATION,
- ; PRESSURE.LEVEL.INDEX (still in 04R001), TSM.CONCENTRATION
- ;List of obsolete DATA_VARIABLE_01 values
- obs_list_1=['AIR.TEMPERATURE','TEMPERATURE.AIR','COLUMN.VERTICAL','H2O.RELATIVE.HUMIDITY',$
- 'PRESSURE.SURFACE','TEMPERATURE.SURFACE','TEMPERATURE.INTERNAL.BOX',$
- 'TEMPERATURE.INTERNAL.INSTRUMENT','TEMPERATURE.SEA.SURFACE','AIR.NUMBER.DENSITY',$
- 'SEA.SURFACE.TEMPERATURE','OPACITY.ATMOSPHERIC.VERTICAL']
- ;Corresponding list of replacement DATA_VARIABLE_1 values (direct replacement)
- rep_list_1=['TEMPERATURE','TEMPERATURE','COLUMN','HUMIDITY.RELATIVE',$
- 'SURFACE.PRESSURE','SURFACE.TEMPERATURE','INTERNAL.BOX.TEMPERATURE',$
- 'INTERNAL.INSTRUMENT.TEMPERATURE','SEA.SURFACE.TEMPERATURE','NUMBER.DENSITY',$
- 'SEA.SURFACE.TEMPERATURE.SKIN','OPACITY.ATMOSPHERIC']
- ;Obsolete DATA_VARIABLE_01 which need special handling
- obs_list_spec_1=['.AMF','.AVK','.CONCENTRATION','.MIXING.RATIO']
- ;Obsolete DATA_VARIABLE_02 values
- obs_list_2=['SOLAR','VERTICAL.SOLAR',$
- 'EMISSION.VERTICAL','VERTICAL.EMISSION',$
- 'VERTICAL','SLANT','SOLAR.OCCULTATION','VERTICAL.ZENITH','VERTICAL.NADIR',$
- 'VERTICAL.LUNAR','VERTICAL.SOLAR.FOCUS']
- ;[0,1] for FTIR/UVVIS (except Brewer and Dobson) only, [2,3] for MWR only
- ;Corresponding list of replacement DATA_VARIABLE_2 values (direct replacement)
- rep_list_2=['ABSORPTION.SOLAR|SCATTER.SOLAR','ABSORPTION.SOLAR|SCATTER.SOLAR',$
- 'EMISSION','EMISSION',$
- '','','OCCULTATION.SOLAR','ZENITH','NADIR',$
- 'ABSORPTION.LUNAR','ABSORPTION.SOLAR.FOCUS']
- ;Replacements for .CONCENTRATION - dependent on VAR_UNITS
- rep_list_conc=['.MIXING.RATIO.VOLUME','.MIXING.RATIO.MASS','.NUMBER.DENSITY','.PARTIAL.PRESSURE']
- ;Conditional Miscellaneous changes
- misc_list_1=['TEMPERATURE_SKIN','AIR.MASS.FACTOR']
- ;Corresponding change to miscellaneous variables
- misc_list_2=['SEA.SURFACE.TEMPERATURE.SKIN','O3.COLUMN_AMF']
- ;Obsolete DATA_VARIABLE_03 descriptors (may or may not be able to fix these)
- obs_list_3=['DU','UNCERTAINTY.STDEV','UNCERTAINTY.RMS','UNCERTAINTY.RELATIVE','UNCERTAINTY.TOTAL', $
- 'UNCERTAINTY.RANDOM','UNCERTAINTY.SYSTEMATIC']
- ;Corresponding list of replacement DATA_VARIABLE_03 descriptors
- rep_list_3=['','UNCERTAINTY.COMBINED.STANDARD','UNCERTAINTY.COMBINED.STANDARD', $
- 'UNCERTAINTY.COMBINED.STANDARD.RELATIVE','UNCERTAINTY.COMBINED.STANDARD', $
- 'UNCERTAINTY.RANDOM.STANDARD','UNCERTAINTY.SYSTEMATIC.STANDARD']
- rep_list_4=['','','','','UNCERTAINTY.COMBINED.COVARIANCE','UNCERTAINTY.RANDOM.COVARIANCE', $
- 'UNCERTAINTY.SYSTEMATIC.COVARIANCE']
- vn_v=STRARR(in1,3) & vn_new=STRARR(in1)
- ;Separate out Variable Name/[Mode or Descriptor]/[Descriptor]
- FOR i=0,in1-1 DO BEGIN
- res=STRSPLIT(in2[i],'_',/EXTRACT,COUNT=nres)
- IF nres GT 3 THEN nres=3
- FOR j=0,nres-1 DO vn_v[i,j]=res[j]
- ENDFOR
- ;Test for values in obs_list_1
- FOR i=0,N_ELEMENTS(obs_list_1)-1 DO BEGIN
- ;ri=WHERE(STRPOS(STRUPCASE(vn_v[*,0]),obs_list_1[i]) NE -1,rcnt)
- ;Change to exact match from v4.0b55 (20200709) - due to introduction of DRY.AIR.... variable names
- ri=WHERE(STRUPCASE(vn_v[*,0]) EQ obs_list_1[i],rcnt)
- IF rcnt NE 0 THEN BEGIN ;obsolete values found so replace
- vlen=STRLEN(obs_list_1[i])
- FOR j=0,rcnt-1 DO BEGIN
- change_val=1
- ;Special case for SEA.SURFACE.TEMPERATURE (only change if mode is _SKIN)
- IF obs_list_1[i] EQ 'SEA.SURFACE.TEMPERATURE' THEN BEGIN
- IF STRUPCASE(vn_v[ri[j],1]) EQ 'SKIN' THEN vn_v[ri[j],1]='' ELSE change_val=0
- ENDIF
- IF change_val EQ 1 THEN BEGIN
- vpos=STRPOS(STRUPCASE(vn_v[ri[j],0]),obs_list_1[i])
- vn_v[ri[j],0]=STRMID(vn_v[ri[j],0],0,vpos)+rep_list_1[i]+STRMID(vn_v[ri[j],0],vpos+vlen)
- ENDIF
- ENDFOR
- ENDIF
- ENDFOR
- ;Special cases for DATA_VARIABLE_01 values .AMF, .AVK, .CONCENTRATION, and .MIXING.RATIO
- nols1=N_ELEMENTS(obs_list_spec_1)
- FOR i=0,nols1-1 DO BEGIN
- ri=WHERE(STRPOS(STRUPCASE(vn_v[*,0]),obs_list_spec_1[i]) NE -1,rcnt)
- IF (i EQ nols1-1) AND (rcnt NE 0) THEN BEGIN
- ;do check for .MIXING.RATIO. in which case no correction required
- rxi=WHERE(STRPOS(STRUPCASE(vn_v[ri,0]),'.MIXING.RATIO.') EQ -1,rxcnt)
- IF rxcnt NE 0 THEN BEGIN ;some values are just .MIXING.RATIO so correct these only
- rcnt=rxcnt & ri=ri[rxi]
- ENDIF ELSE rcnt=0
- ENDIF
- IF rcnt NE 0 THEN BEGIN ;obsolete values found
- vlen=STRLEN(obs_list_spec_1[i])
- IF i LE 1 THEN BEGIN ;move .AMF and .AVK to DESCRIPTOR section (2 or 3)
- FOR j=0,rcnt-1 DO BEGIN
- vpos=STRPOS(STRUPCASE(vn_v[ri[j],0]),obs_list_spec_1[i])
- ai=WHERE(vn_v[ri[j],*] EQ '',acnt)
- IF acnt NE 0 THEN BEGIN
- ;the DESCRIPTOR field is empty, so can convert value OK (otherwise no change made)
- vn_v[ri[j],0]=STRMID(vn_v[ri[j],0],0,vpos)
- vn_v[ri[j],ai[0]]=STRMID(obs_list_spec_1[i],1) ;write to first empty field
- ENDIF
- ENDFOR
- ENDIF ELSE BEGIN
- ;.CONCENTRATION or .MIXING.RATIO so attempt to identify actual type of measurement
- ;(mixing ratio etc) by looking at corresponding VAR_UNIT values
- mrvi=WHERE((STRPOS(STRLOWCASE(in3[ri]),'pp') NE -1) OR $
- (STRPOS(STRLOWCASE(in3[ri]),'m-1 sr-1') NE -1),mrvcnt) ;testing for volume mixing ratio
- mrmi=WHERE(STRPOS(STRLOWCASE(in3[ri]),' g ') NE -1,mrmcnt) ;test for mass mixing ratio
- ndi=WHERE(STRPOS(STRLOWCASE(in3[ri]),'mol') NE -1,ndcnt) ;test for number density
- ppi=WHERE(STRPOS(STRLOWCASE(in3[ri]),'pa') NE -1,ppcnt) ;test for partial pressure
- IF i EQ nols1-1 THEN cnti=[mrvcnt,mrmcnt,0,0] ELSE cnti=[mrvcnt,mrmcnt,ndcnt,ppcnt]
- ci=WHERE(cnti GT 0,ccnt)
- IF ccnt EQ 1 THEN BEGIN ;type of measurement successfully found
- ;Note: not set up to handle if more than one type of measurement found in the file
- vlen=STRLEN(obs_list_spec_1[i])
- FOR j=0,rcnt-1 DO BEGIN
- IF (STRUPCASE(vn_v[ri[j],0]) EQ 'AIR.CONCENTRATION') AND (ci[0] EQ 2) THEN $
- ;Do special case check for AIR.CONCENTRATION - can only change to NUMBER.DENSITY
- vn_v[ri[j],0]='NUMBER.DENSITY' $
- ELSE BEGIN
- vpos=STRPOS(STRUPCASE(vn_v[ri[j],0]),obs_list_spec_1[i])
- vn_v[ri[j],0]=STRMID(vn_v[ri[j],0],0,vpos)+rep_list_conc[ci[0]]+STRMID(vn_v[ri[j],0],vpos+vlen)
- ENDELSE
- ENDFOR
- ENDIF
- ENDELSE
- ENDIF
- ENDFOR
- FOR i=0,N_ELEMENTS(obs_list_2)-1 DO BEGIN ;Check for obsolete mode values
- ri=WHERE(STRUPCASE(vn_v[*,1]) EQ obs_list_2[i],rcnt)
- IF rcnt NE 0 THEN BEGIN
- test1=(i LE 1) AND (STRPOS(STRUPCASE(in4),'FTIR') NE -1)
- test2=(i LE 1) AND (STRPOS(STRUPCASE(in4),'UVVIS') NE -1) AND $
- (STRPOS(STRUPCASE(in4),'DOBSON') EQ -1) AND (STRPOS(STRUPCASE(in4),'BREWER') EQ -1)
- test3=((i EQ 2) OR (i EQ 3)) AND ((STRPOS(STRUPCASE(in4),'MWR') NE -1) OR $
- (STRPOS(STRUPCASE(in4),'MICROWAVE') NE -1))
- IF i LE 1 THEN mi=STRSPLIT(rep_list_2[i],'|',/EXTRACT)
- ;test1, test2, and test3 are special case obsolete values
- IF test1 THEN vn_v[ri,1]=mi[0] $
- ELSE IF test2 THEN vn_v[ri,1]=mi[1] $
- ELSE IF test3 THEN vn_v[ri,1]=rep_list_2[i] $ ;direct replacement
- ELSE BEGIN ;all other obsolete mode values
- IF i LE 1 THEN vn_v[ri,1]=mi[0] ELSE vn_v[ri,1]=rep_list_2[i] ;direct replacement
- IF i EQ 5 THEN BEGIN ;SLANT (if column measurement then add to main variable name)
- FOR j=0,rcnt-1 DO BEGIN
- IF STRLEN(vn_v[ri[j],0])-STRPOS(vn_v[ri[j],0],'COLUMN') EQ 6 THEN $
- vn_v[ri[j],0]=vn_v[ri[j],0]+'.'+obs_list_2[i]
- ENDFOR
- ENDIF
- ENDELSE
- ENDIF
- ENDFOR
- ;Check conditional miscellaneous changes
- FOR i=0,N_ELEMENTS(misc_list_1)-1 DO BEGIN
- IF i EQ 0 THEN ri=WHERE(STRPOS(STRUPCASE(in2),misc_list_1[i]) NE -1,rcnt) $
- ELSE ri=WHERE(STRUPCASE(in2) EQ misc_list_1[i],rcnt)
- IF rcnt NE 0 THEN BEGIN
- IF i EQ 0 THEN BEGIN ;ensure DATA_SOURCE is BUOY before changing
- IF STRPOS(STRUPCASE(in4),'BUOY') NE -1 THEN BEGIN
- FOR j=0,rcnt-1 DO BEGIN
- vn_v[ri[j],0]=misc_list_2[i] & vn_v[ri[j],1]=''
- ENDFOR
- ENDIF
- ENDIF ELSE BEGIN ;ensure O3.COLUMN is also present as a DATA_VARIABLE before changing
- mi=WHERE(STRUPCASE(vn_v[*,0]) EQ 'O3.COLUMN',mcnt)
- IF mcnt NE 0 THEN BEGIN
- vn_v[ri[0],0]='O3.COLUMN' & vn_v[ri[0],1]='AMF'
- ENDIF
- ENDELSE
- ENDIF
- ENDFOR
-
- FOR i=0,N_ELEMENTS(obs_list_3)-1 DO BEGIN ;Check for obsolete descriptor values (UNCERTAINTY)
- ;acceptable units for identifying correct replacement name (test1 instruments only)
- vuok=['ppv','ppmv','ppbv','pptv','k','du','molec cm-2','molec cm-3','degc','m-1 sr-1','dimensionless']
- vuok2=['ppv2','ppmv2','ppbv2','pptv2'] ;covariance
- test1=(STRPOS(STRUPCASE(in4),'MWR') NE -1) OR (STRPOS(STRUPCASE(in4),'MICROWAVE') NE -1) OR $
- (STRPOS(STRUPCASE(in4),'HAGAR') NE -1) OR (STRPOS(STRUPCASE(in4),'MIPAS.STR') NE -1) OR $
- (STRPOS(STRUPCASE(in4),'SIOUX') NE -1) OR (STRPOS(STRUPCASE(in4),'UVVIS.AMAXDOAS') NE -1) OR $
- (STRPOS(STRUPCASE(in4),'UVVIS.SAOZ') NE -1) OR (STRPOS(STRUPCASE(in4),'ATMOINSPECTOR') NE -1) OR $
- (STRPOS(STRUPCASE(in4),'FISH') NE -1) OR (STRPOS(STRUPCASE(in4),'AMON') NE -1) OR $
- (STRPOS(STRUPCASE(in4),'LPMA') NE -1) OR (STRPOS(STRUPCASE(in4),'SALOMON') NE -1) OR $
- (STRPOS(STRUPCASE(in4),'RADIOMETER.IR.CIMEL') NE -1) OR (STRPOS(STRUPCASE(in4),'UVVIS') NE -1) OR $
- (STRPOS(STRUPCASE(in4),'SPECTROMETER') NE -1) OR (STRPOS(STRUPCASE(in4),'LIDAR.WATERVAPORRAMAN_UNIVAQ') NE -1)
- ;check for DATA_SOURCE where CoVariance as well as Standard uncertainties are reported
- test2=(STRPOS(STRUPCASE(in4),'FTIR.CO_KIT') NE -1) OR (STRPOS(STRUPCASE(in4),'FTIR.CO_ULG') NE -1) OR $
- (STRPOS(STRUPCASE(in4),'FTIR.HNO3_KIT') NE -1) OR (STRPOS(STRUPCASE(in4),'FTIR.HNO3_ULG') NE -1) OR $
- (STRPOS(STRUPCASE(in4),'FTIR.N2O_KIT') NE -1) OR (STRPOS(STRUPCASE(in4),'FTIR.N2O_ULG') NE -1) OR $
- (STRPOS(STRUPCASE(in4),'FTIR.NO2_KIT') NE -1) OR (STRPOS(STRUPCASE(in4),'FTIR.NO2_ULG') NE -1)
- vnvi=1 ;determines the location index of the descriptor variable
- ri=WHERE(STRUPCASE(vn_v[*,1]) EQ obs_list_3[i],rcnt)
- IF rcnt EQ 0 THEN BEGIN
- vnvi=2 & ri=WHERE(STRUPCASE(vn_v[*,2]) EQ obs_list_3[i],rcnt)
- ENDIF
- IF rcnt NE 0 THEN BEGIN
- IF i LE 3 THEN vn_v[ri,vnvi]=rep_list_3[i] $ ;direct replacement
- ELSE BEGIN ;need to identify whether the UNITS are % or fit vuok criteria
- FOR j=0,rcnt-1 DO BEGIN
- IF STRTRIM(in3[ri[j]],2) EQ '%' THEN vn_v[ri[j],vnvi]=rep_list_3[i]+'.RELATIVE' $
- ELSE BEGIN
- ;test for Standard Uncertainties
- vi=WHERE(STRTRIM(STRLOWCASE(in3[ri[j]]),2) EQ vuok,vcnt)
- IF vcnt EQ 0 THEN $ ;do test for [Prefix]molec cm-2/molec cm-3
- IF (STRPOS(STRLOWCASE(in3[ri[j]]),'molec cm-2') NE -1) OR $
- (STRPOS(STRLOWCASE(in3[ri[j]]),'molec cm-3') NE -1) THEN vcnt=1
- IF ((test1) OR (test2)) AND (vcnt NE 0) THEN $ can assume that the given error values are Standard Deviation
- vn_v[ri[j],vnvi]=rep_list_3[i]
- ;test for Covariance Uncertainties
- vi=WHERE(STRTRIM(STRLOWCASE(in3[ri[j]]),2) EQ vuok2,vcnt)
- IF (test2) AND (vcnt NE 0) THEN vn_v[ri[j],vnvi]=rep_list_4[i]
- ENDELSE
- ENDFOR
- ENDELSE
- ENDIF
- ENDFOR
- ;create any comments for INFOTXT_OUTPUT
- FOR i=0,in1-1 DO BEGIN
- ;put full DATA_VARIABLE names back together
- vn_new[i]=vn_v[i,0]
- FOR j=1,2 DO IF vn_v[i,j] NE '' THEN vn_new[i]=vn_new[i]+'_'+vn_v[i,j]
- ;Check to see if it is different to the original DATA_VARIABLE
- IF vn_new[i] NE in2[i] THEN BEGIN
- vnchange[i]=in2[i] & in2[i]=vn_new[i]
- IF qa_yes THEN itxt=' expected to be ' ELSE itxt=' renamed '
- infotxt='2 Dataset name '+vnchange[i]+itxt+in2[i]
- INFOTXT_OUTPUT,infotxt
- ENDIF
- ENDFOR
- END
-
- code EQ 6: BEGIN ;Look for obsolete DATA_SOURCE
- ;List of obsolete DATA_SOURCE_01 values
- obs_list=['RO.CHAMP','RO.SAC.C','RO.F3C.FM','RO.CNOFS','RO.GRACE.A','FTIR_', $
- 'UVVIS.DOAS_','MICROWAVE.RADIOMETER','LIDAR.BACKSCATTER',$
- 'LIDAR.DIAL','LIDAR.OLEX','LIDAR.RIEGL','LIDAR.RMR']
- ;Corresponding replacement list
- rep_list=['CHAMP','SAC.C','F3C.FM','CNOFS','GRACE.A','FTIR.','UVVIS.DOAS.','MWR.','LIDAR.']
- ;Possible list of species for the above instruments (note: gases will take precedence
- ;over Aerosol and Temperature, o/w the first species found will be considered the
- ;primary species measured)
- spc_list=['BrO.','C2H2.','C2H6.','CCl2F2.','CCl3F.','CH4.','CHF2Cl.','CHOCHO.','ClO.','ClONO2.','CO.',$
- 'CO2.','COF2.','H2CO.','H2O.','HCFC22.','HCl.','HCN.','HCOOH.','HF.','HNO3.','HONO.','IO.',$
- 'N2O.','NO.','NO2.','O3.','OClO.','OCS.','SF6.','SO2.','AEROSOL','TEMPERATURE']
-
- oli=-1 ;will change to obs_list index value if obsolete DATA_SOURCE_01 found
- n_obsl=N_ELEMENTS(obs_list)
- lid_i=n_obsl-5 ;needs to be start index of the lidar measurements in obs_list
- spc_i=n_obsl-8 ;needs to be start index of the instruments requiring species information in obs_list
- uv_i=n_obsl-7 ;needs to be UVVIS.DOAS index in obs_list
- FOR i=0,N_ELEMENTS(obs_list)-1 DO IF STRPOS(STRUPCASE(in3),obs_list[i]) NE -1 THEN oli=i
- IF oli NE -1 THEN BEGIN ;obsolete DATA_SOURCE value found
- IF oli GE lid_i THEN oli=lid_i ;to correspond with rep_list index for lidar
- pri=-1 ;will change if species found
- IF oli GE spc_i THEN BEGIN ;if not an RO DATA_SOURCE then need to determine species
- n_sl=N_ELEMENTS(spc_list)
- ;Look through VAR_NAME values to try and identify new DATA_SOURCE from species list
- spc_found=INTARR(in1)-1 ;identifies index of found species names in the Datasets
- FOR i=0,n_sl-1 DO BEGIN
- fi=WHERE(STRPOS(STRMID(in2,0,STRLEN(spc_list[i])),spc_list[i]) NE -1,fcnt)
- IF fcnt NE 0 THEN spc_found[fi]=i
- ENDFOR
- ;Check list of found species - don't include Aerosol and Temperature
- fi=WHERE((spc_found NE -1) AND (spc_found LT n_sl-2),fcnt)
- IF fcnt NE 0 THEN pri=spc_found[fi[0]] $ ;primary species index determined
- ELSE IF (oli EQ uv_i) OR (oli EQ lid_i) THEN BEGIN ;Test for Temperature (Lidar Only) and Aerosol
- fi=WHERE(spc_found NE -1,fcnt)
- IF fcnt NE 0 THEN BEGIN
- pri=MIN(spc_found[fi]) ;i.e. Aerosol takes precedence over Temperature
- IF (oli EQ uv_i) AND (pri EQ n_sl-1) THEN pri=-1 ;UVVIS.DOAS can't be Temperature
- ENDIF
- ENDIF
- ENDIF ELSE BEGIN ;cases with straight replacement
- old_ds=STRTRIM(in3,2) & in3=rep_list[oli]+STRMID(old_ds,STRLEN(obs_list[oli])) & pri=-2
- ENDELSE
- IF pri NE -1 THEN BEGIN ;update DATA_SOURCE
- IF pri GE 0 THEN BEGIN
- old_ds=in3
- ;check special cases for LIDAR water vapor measurements
- IF (spc_list[pri] EQ 'H2O.') AND (STRPOS(STRUPCASE(in3),'LIDAR.DIAL') NE -1) THEN $
- in3='H2O' $
- ELSE IF (spc_list[pri] EQ 'H2O.') AND (STRPOS(STRUPCASE(in3),'LIDAR.BACKSCATTER') NE -1) THEN $
- in3='H2O' $
- ELSE in3=spc_list[pri]
- IF STRPOS(in3,'.') NE -1 THEN in3=STRMID(in3,0,STRLEN(in3)-1)
- in3=rep_list[oli]+in3+STRMID(old_ds,STRPOS(old_ds,'_'))
- ENDIF
- IF qa_yes THEN itxt=' expected to be ' ELSE itxt=' renamed '
- infotxt='2 DATA_SOURCE value '+old_ds+itxt+in3
- INFOTXT_OUTPUT,infotxt
- ENDIF
- ENDIF
- END
-
- code EQ 7: BEGIN ;Look for obsolete CALVAL and NDSC FILE_ACCESS values
- obs_list=['CALVAL','NDSC']
- rep_list=['EVDC','NDACC']
-
- FOR i=0,in1-1 DO BEGIN
- oi=WHERE(in2[i] EQ obs_list,ocnt)
- IF ocnt NE 0 THEN BEGIN
- IF qa_yes THEN itxt=' expected to be ' ELSE itxt=' renamed '
- infotxt='2 FILE_ACCESS value '+in2[i]+itxt+rep_list[oi[0]]
- INFOTXT_OUTPUT,infotxt
- in2[i]=rep_list[oi[0]]
- ENDIF
- ENDFOR
- END
-
- code EQ 8: BEGIN ;Look for VAR_UNITS=DIMENSIONLESS or NONE and replace with '' or '1'
- vuv=STRUPCASE(in2)
- FOR i=0,in1-1 DO BEGIN
- IF (vuv[i] EQ 'DIMENSIONLESS') OR (vuv[i] EQ 'NONE') THEN BEGIN
- IF in3[i] EQ 'STRING' THEN newv='' ELSE newv='1'
- IF qa_yes THEN itxt=' expected to be ' ELSE itxt=' renamed '
- infotxt='2 VAR_UNITS='+vuv[i]+itxt+'VAR_UNITS='+newv+' based on VAR_DATA_TYPE='+in3[i]
- INFOTXT_OUTPUT,infotxt
- in2[i]=newv
- ENDIF
- ENDFOR
- END
-
- code EQ 9: BEGIN ;DATA_QUALITY/DATA_TEMPLATE checks
- attr_arr_glob_prov=['DATA_TEMPLATE','DATA_QUALITY']
- geoms_te='' & meta_te='' & std_txt=' Required Metadata Template: '
- ;Check if DATA_TEMPLATE field is present in the TAV file
- dti=WHERE(tab_arr[*,0] EQ attr_arr_glob_prov[0],dtcnt)
- ;Check if DATA_TEMPLATE label is present in the Metadata
- gi=WHERE(in1 EQ attr_arr_glob_prov[0],gcnt)
- IF gcnt EQ 1 THEN IF in2[gi[0]] NE '' THEN meta_te=STRUPCASE(STRTRIM(in2[gi[0]],2))
- IF dtcnt NE 0 THEN tab_arr_sub=tab_arr[dti[0],2:FIX(tab_arr[dti[0],1]+1)] ;Valid Template values from TAV file
- IF dtcnt EQ 0 THEN BEGIN
- infotxt='3'+std_txt+'DATA_TEMPLATE field is not present in the TAV file. Checks cannot be performed'
- INFOTXT_OUTPUT,infotxt
- std_txt=''
- ENDIF ELSE IF meta_te NE '' THEN BEGIN
- ;check if value is present in the TAV file
- ci=WHERE(STRUPCASE(tab_arr_sub) EQ meta_te,ccnt)
- IF ccnt NE 0 THEN BEGIN
- geoms_te=tab_arr_sub[ci[0]] & in2[gi[0]]=geoms_te
- infotxt='0'+std_txt+geoms_te
- INFOTXT_OUTPUT,infotxt
- std_txt=''
- ENDIF
- ENDIF
- IF std_txt NE '' THEN BEGIN ;template value not present in the metadata or value not in the TAV file
- ;Determine DATA_TEMPLATE value based on DATA_SOURCE value (if applicable)
- di=WHERE(in1 EQ 'DATA_SOURCE',dcnt)
- IF dcnt EQ 1 THEN BEGIN ;DATA_SOURCE found
- ;Extract DATA_SOURCE value
- IF in2[di[0]] NE '' THEN BEGIN
- res=STRSPLIT(STRTRIM(in2[di[0]],2),'_',/EXTRACT)
- ;First check - First part of DATA_SOURCE matches text in TAV entry
- res1=STRUPCASE(STRSPLIT(res[0],'.',/EXTRACT,COUNT=res1cnt))
- ci=WHERE(STRPOS(STRUPCASE(tab_arr_sub),res1[0]) NE -1,ccnt)
- IF ccnt EQ 1 THEN geoms_te=tab_arr_sub[ci[0]] $ ;Template found
- ELSE IF (ccnt GT 1) AND (N_ELEMENTS(res1) GT 1) THEN BEGIN
- ;2nd Check - More than one option so try and match 2nd part of DATA_SOURCE with 4th part in TAV entry
- tab_arr_sub=tab_arr_sub[ci]
- tab_arr_part_sub=STRARR(ccnt)
- numi=-1
- FOR i=0,ccnt-1 DO BEGIN
- rest=STRSPLIT(tab_arr_sub[i],'-',/EXTRACT)
- rest=[rest,'','',''] ;ensure there are at least 4 sub-values
- tab_arr_part_sub[i]=rest[3]
- IF IS_A_NUMBER_HDF(rest[3]) THEN numi=i
- ENDFOR
- ci=WHERE(STRUPCASE(tab_arr_part_sub) EQ res1[1],ccnt)
- IF (ccnt EQ 0) AND (numi NE -1) THEN BEGIN
- ;2nd part is likely a gas e.g. FTIR.CO, so use template where 4th part of TAV entry is a number
- geoms_te=tab_arr_sub[numi]
- ENDIF ELSE IF ccnt EQ 1 THEN geoms_te=tab_arr_sub[ci[0]] $ ;unique template found
- ELSE IF ccnt GT 1 THEN BEGIN ;ccnt GT 1, Try alternatives for H2O and O3, or test for fourth part
- tasx=tab_arr_sub[ci]
- CASE 1 OF
- (res1[0] EQ 'UVVIS') AND (res1[1] EQ 'DOAS'): BEGIN
- IF res1cnt GE 4 THEN BEGIN
- IF res1[3] EQ 'AEROSOL' THEN res1x=STRUPCASE(res1[2])+'-AEROSOL' $
- ELSE res1x=STRUPCASE(res1[2])+'-GAS'
- ENDIF ELSE res1x=res1[1]
- END
- res1[1] EQ 'H2O': res1x='WATERVAPOR'
- res1[1] EQ 'O3': res1x='OZONE'
- ELSE: res1x=res1[1]
- ENDCASE
- ci=WHERE(STRPOS(STRUPCASE(tasx),res1x) NE -1,ccnt)
- IF ccnt EQ 1 THEN geoms_te=tasx[ci[0]]
- ENDIF
- ENDIF
- IF geoms_te NE '' THEN BEGIN ;DATA_TEMPLATE value found based on DATA_SOURCE
- IF gcnt EQ 1 THEN BEGIN ;DATA_TEMPLATE label is present
- fmvi=WHERE(in1 EQ 'FILE_META_VERSION',fmvcnt)
- fmvtxt=''
- IF fmvcnt EQ 1 THEN BEGIN
- resfmv=STRSPLIT(in2[fmvi[0]],' ;',/EXTRACT,COUNT=fmvcnt)
- IF fmvcnt GE 1 THEN BEGIN
- fmv_ver=resfmv[0]
- fmvtxt=' and '+fmv_ver+' metadata definitions'
- ENDIF
- ENDIF
- in2[gi[0]]=geoms_te ;Add value to DATA_TEMPLATE label
- IF qa_yes THEN itxt=' expected to be ' ELSE itxt=' renamed '
- infotxt='2 DATA_TEMPLATE value'+itxt+geoms_te+' based on DATA_SOURCE value'+fmvtxt
- INFOTXT_OUTPUT,infotxt
- std_txt=''
- ENDIF ELSE IF gcnt EQ 0 THEN in3[0]=geoms_te ;Need to add new Global Attribute Information
- ENDIF
- ENDIF
- ENDIF
- ENDIF
- IF geoms_te NE '' THEN BEGIN ;Check if DATA_QUALITY label is present
- gi=WHERE(in1 EQ attr_arr_glob_prov[1],gcnt)
- IF gcnt EQ 0 THEN in3[1]='DATA_QUALITY' ;Need to add new Global Attribute Information
- ENDIF
- IF std_txt NE '' THEN BEGIN ;No template file required based on selection criteria
- infotxt='0'+std_txt+'DATA_TEMPLATE value not required based on selection criteria'
- INFOTXT_OUTPUT,infotxt
- ENDIF
- END
-
- code EQ 10: BEGIN ;check printable characters in Metadata and Datasets (of type String)
- ;Note: Also allowed are Control Characters Null Character (0B), Horizontal Tab (9B),
- ;Line Feed (10B), and Carriage Return (13B) - Note: 10B and 13B for Metadata only
- exempt=['PI_NAME','DO_NAME','DS_NAME','PI_AFFILIATION','DO_AFFILIATION','DS_AFFILIATION',$
- 'PI_ADDRESS','DO_ADDRESS','DS_ADDRESS','PI_EMAIL','DO_EMAIL','DS_EMAIL']
- non_char=[27B, 128B, 147B, 176B, 186B, 197B, 216B, 226B, 232B, 233B, 248B, 252B] ;128B = Euro
- rep_char=['',' ','"',' deg. ',' deg. ','A','O','a','e','e','o','u'] ;replace above non-US ASCII chars with these values
-
- ;Do check for the Full TAV which includes ORIGINATOR attributes
- ochk=STRPOS(tab_arr(*,0),'ORIGINATOR') NE -1
- oi=WHERE(ochk NE 0,ocnt)
- mentry=0 ;Default assumes String Dataset Entry
- n_nchar=N_ELEMENTS(non_char)
- IF N_PARAMS() EQ 3 THEN in2='Dataset '+STRTRIM(in2,2) $ ;Dataset Entry
- ELSE mentry=1 ;Metadata entry
- FOR i=0L,N_ELEMENTS(in1)-1L DO BEGIN
- exempt_att=0 ;default assumes attribute is not exempt from checks
- IF mentry EQ 1 THEN BEGIN
- res=STRSPLIT(in1[i],'=',/EXTRACT)
- in2='Metadata entry '+in1[i] ;res[0]
- IF ocnt NE 0 THEN BEGIN
- ;Do not need to check 'exempt' attributes if using the full version of the TAV file
- ei=WHERE(STRUPCASE(res[0]) EQ exempt,ecnt)
- IF ecnt NE 0 THEN exempt_att=1 ;invalid entries can be corrected by the code
- ENDIF
- ENDIF
-
- IF (~qa_yes) AND (exempt_att EQ 0) THEN BEGIN
- ;If not doing QA then check for characters that can be replaced
- non_char_i=[-1]
- FOR j=0,n_nchar-1 DO BEGIN
- testb=BYTE(in1[i])
- ni=WHERE(testb EQ non_char[j],ncnt)
- IF ncnt NE 0 THEN BEGIN
- IF non_char_i[0] EQ -1 THEN non_char_i=[j] $
- ELSE non_char_i=[non_char_i,j]
- inval=in1[i]
- FOR k=0,ncnt-1 DO BEGIN
- IF k NE 0 THEN ni[k]=ni[k]+((STRLEN(rep_char[j])-1)*k)
- inval=STRMID(inval,0,ni[k])+rep_char[j]+STRMID(inval,ni[k]+1)
- ENDFOR
- in1[i]=inval
- ENDIF
- ENDFOR
- IF non_char_i[0] NE -1 THEN BEGIN
- infotxt='2 Illegal character(s) '
- FOR j=0,N_ELEMENTS(non_char_i)-1 DO BEGIN
- IF j EQ 0 THEN infotxt=infotxt+STRTRIM(non_char[non_char_i[j]],2) $
- ELSE infotxt=infotxt+', '+STRTRIM(non_char[non_char_i[j]],2)
- ENDFOR
- infotxt=infotxt+' in '+in2+' replaced with valid ISO646-US ASCII character(s)'
- INFOTXT_OUTPUT,infotxt
- ENDIF
- ENDIF
- testb=BYTE(in1[i])
-
- IF exempt_att EQ 0 THEN BEGIN
- IF mentry EQ 1 THEN $
- test=(testb NE 0B) AND (testb NE 9B) AND (testb NE 10B) AND (testb NE 13B) $
- ELSE test=(testb NE 0B) AND (testb NE 9B) ;Check for Dataset Control Characters
- ;check for non-char value and replace
- bi=WHERE(((testb LT 32B) OR (testb GT 126B)) AND (test),bcnt)
- IF bcnt NE 0 THEN BEGIN ;Illegal character present
- IF bcnt GT 1L THEN BEGIN ;check for repeat illegal characters and only display once
- bix=[bi[0]]
- FOR j=1L,bcnt-1L DO BEGIN
- ri=WHERE(testb[bi[j]] EQ testb[bi[0L:j-1L]],rcnt)
- IF rcnt EQ 0 THEN bix=[bix,bi[j]]
- ENDFOR
- bi=bix & bcnt=N_ELEMENTS(bi)
- ENDIF
- infotxt=STRARR(2)
- infotxt[0]='3 Entry must only include valid characters from the ISO646-US ASCII character set.'
- infotxt[1]=' Illegal character(s) '
- FOR j=0L,bcnt-1L DO BEGIN
- cval=STRTRIM(testb[bi[j]],2)
- ;print,testb[bi[j]] ;to output actual byte value
- IF (cval EQ STRTRIM(8B,2)) OR (cval EQ '') THEN BEGIN
- ;Illegal non-printable character so print in Decimal Byte form
- cval=STRING(format='(I4)',testb[bi[j]]) & cval=STRTRIM(cval,2)+'B'
- ENDIF
- IF j NE bcnt-1L THEN itxt=', ' ELSE itxt=''
- infotxt[1]=infotxt[1]+cval+itxt
- ENDFOR
- infotxt[1]=infotxt[1]+' in '+in2+' (may not display correctly)'
- INFOTXT_OUTPUT,infotxt
- ENDIF
- ENDIF
- ENDFOR
- END
- code EQ 11: BEGIN ;Check DATA_VERSION_NAME part of the DATA_SOURCE
- ;Check that entry is made up of dot-separated alpha_numeric words
- n_char=STRLEN(in1)
- strok=1B ;default is that entry is OK
- test2=0B & test3=0B ;will change to 1 if test conditions are true
- charchkm1=0B ;to check test character against the previous character
- FOR i=0,n_char-1 DO BEGIN
- charchk=BYTE(STRMID(in1,i,1))
- test1=(charchk EQ 46B) OR ((charchk GE 65B) AND (charchk LE 90B)) OR $ ;'.' or A-Z
- ((charchk GE 97B) AND (charchk LE 122B)) OR $ ;a-z
- ((charchk GE 48B) AND (charchk LE 57B)) ;0-9
- IF (i EQ 0) OR (i EQ n_char-1) THEN test2=charchk EQ 46B ;i.e. first or last character is a '.'
- test3=(charchk EQ 46B) AND (charchkm1 EQ 46B) ;i.e. two consecutive dots
- IF (~test1) OR (test2) OR (test3) THEN strok=0B
- charchkm1=charchk
- ENDFOR
- IF ~strok THEN BEGIN
- ;write error message
- infotxt='3 VERSION_NAME sub-value of the DATA_SOURCE must consist of dot-separated alpha-numeric words: '
- infotxt=infotxt+in1
- INFOTXT_OUTPUT,infotxt
- ENDIF
- END
-ENDCASE
-
-END ;Procedure GEOMS_Rule_Changes
-
-
-
-PRO pre_defined_att_checks, ct, nav, vav, vnx, vdtv,verror
-;Procedure to perform checks on the HDF4 pre-defined attributes, as well as check that
-;the numeric variable attribute values are of the same data type as that given by the
-;corresponding VAR_DATA_TYPE value (if input is via session memory)
-; ----------
-;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
-;
-; History:
-; 20101120: Introduced - Version 4.0
-;
-; Inputs: ct - Identifies the type of check to be performed on the inputs
-; nav - The HDF pre-defined attribute to check
-; vav - The Variable Attribute corresponding to the pre-defined attribute
-; vnx - The Dataset from which the attributes are derived
-; vdtv - The Data Type of the Dataset
-; verror - Boolean to indicate whether the Variable Attribute data type error has
-; already been written
-;
-; Outputs: None - Any Pre-defined attributes written to an HDF file are recreated by the program
-;
-; Called by: IDLCR8HDF, READ_METADATA
-;
-; Subroutines Called: INFOTXT_OUTPUT
-; Information Conditions (when the program is able to make changes):
-; 1. VAR_VALID_MIN/MAX or VAR_FILL_VALUE data types do not match VAR_DATA_TYPE
-; 2. HDF4 pre-defined attribute data type does not match VAR_DATA_TYPE
-; 3. HDF4 pre-defined attribute units value does not match VAR_UNITS
-; 4. HDF4 pre-defined attribute value does not equal equivalent variable
-; attribute value (valid_range, _FillValue)
-; 5. HDF4 pre-defined attribute [long_name/format/coordsys] present in the HDF file
-; 6. Datasets that have been scaled or had an offset applied are not permitted
-
-COMMON DATA
-COMMON WIDGET_WIN
-
-IF ct NE -1 THEN BEGIN ;check units, _FillValue, and valid_range attributes only
- ncsa=['units','_Fillvalue','valid_range','valid_range']
- var_atts=['VAR_UNITS','VAR_FILL_VALUE','VAR_VALID_MIN','VAR_VALID_MAX']
-
- ;Change VAR_DATA_TYPE value to IDL Type code
- CASE 1 OF
- vdtv EQ 'BYTE': idltc=1
- vdtv EQ 'SHORT': idltc=2
- (vdtv EQ 'INTEGER') OR (vdtv EQ 'LONG'): idltc=3
- (vdtv EQ 'REAL') OR (vdtv EQ 'FLOAT'): idltc=4
- vdtv EQ 'DOUBLE': idltc=5
- vdtv EQ 'STRING': idltc=7
- ELSE: idltc=0
- ENDCASE
- test1=(SIZE(vav,/TYPE) EQ 7) AND (STRTRIM(vav,2) NE '')
- test2=SIZE(vav,/TYPE) NE 7
- test3=(SIZE(nav,/TYPE) EQ 7) AND (STRTRIM(nav,2) NE '')
- test4=SIZE(nav,/TYPE) NE 7
- IF ((test1) OR (test2)) AND (ct NE 0) THEN BEGIN
- IF (SIZE(vav,/TYPE) NE idltc) AND (idltc NE 0) AND (verror[0] EQ 0) THEN BEGIN
- ;Check data type of Variable Attribute
- infotxt=STRARR(2)
- itxt='VAR_VALID_MIN, VAR_VALID_MAX or VAR_FILL_VALUE'
- infotxt[0]='2 '+itxt+' data types do not match VAR_DATA_TYPE='+vdtv
- IF vnx NE '' THEN infotxt[1]=' for Dataset '+vnx+'|.' $
- ELSE BEGIN
- infotxt[0]=infotxt[0]+'|.' & infotxt[1]=' '
- ENDELSE
- infotxt[1]=infotxt[1]+' Data type will be changed to match VAR_DATA_TYPE value in HDF file'
- IF ~qa_yes THEN INFOTXT_OUTPUT,infotxt ;error for QA output separately
- verror[0]=1
- ENDIF
- ENDIF
- IF ((test3) OR (test4)) AND (ct NE 0) THEN BEGIN
- IF (SIZE(nav,/TYPE) NE idltc) AND (idltc NE 0) AND (o3[0] EQ 'H4') AND (verror[1] EQ 0) THEN BEGIN
- ;Check data type of pre-defined attribute
- infotxt=STRARR(2)
- infotxt[0]='2 HDF4 pre-defined attribute '+ncsa[ct]+' data type does not match VAR_DATA_TYPE='+vdtv
- IF vnx NE '' THEN infotxt[0]=infotxt[0]+' for Dataset '+vnx+'|.' $
- ELSE infotxt[0]=infotxt[0]+'|.'
- infotxt[1]=' Data type will be changed to match VAR_DATA_TYPE value in HDF file'
- INFOTXT_OUTPUT,infotxt
- IF ncsa[ct] EQ 'valid_range' THEN verror[1]=1 ;to only report 1 valid_range error
- ENDIF
- ENDIF
- IF ((test1) OR (test2)) AND ((test3) OR (test4)) THEN BEGIN
- ;Compare Variable Attributes against pre-defined attributes
- IF ct EQ 0 THEN BEGIN ;units check
- IF (STRTRIM(nav,2) NE STRTRIM(vav,2)) AND (o3[0] EQ 'H4') THEN BEGIN
- infotxt=STRARR(2)
- infotxt[0]='2 HDF4 pre-defined attribute units='+nav+' does not match VAR_UNITS='+vav
- IF vnx NE '' THEN infotxt[0]=infotxt[0]+' for Dataset '+vnx+'|.' $
- ELSE infotxt[0]=infotxt[0]+'|.'
- infotxt[1]=' HDF4 units will be changed to match VAR_UNITS value in HDF file'
- INFOTXT_OUTPUT,infotxt
- ENDIF
- ENDIF ELSE BEGIN
- IF (nav NE vav) AND (idltc NE 7) AND (idltc NE 0) AND (o3[0] EQ 'H4') THEN BEGIN
- infotxt=STRARR(2)
- infotxt[0]='2 HDF4 pre-defined attribute '+ncsa[ct]+' value does not equal '+var_atts[ct]+' value'
- IF vnx NE '' THEN infotxt[0]=infotxt[0]+' for Dataset '+vnx+'|.' $
- ELSE infotxt[0]=infotxt[0]+'|.'
- infotxt[1]=' HDF4 '+ncsa[ct]+' will be changed to match '+var_atts[ct]+' value in HDF file'
- INFOTXT_OUTPUT,infotxt
- ENDIF
- ENDELSE
- ENDIF
-ENDIF ELSE BEGIN ;Do checks on remaining pre-defined attributes
- ncsa_str=['long_name','format','coordsys']
- ncsa_cal=['scale_factor','scale_factor_err','add_offset','add_offset_err','calibrated_nt']
- FOR i=0,N_ELEMENTS(ncsa_str)-1 DO BEGIN
- ni=WHERE(STRLOWCASE(nav) EQ ncsa_str[i],ncnt)
- IF ncnt NE 0 THEN BEGIN
- IF qa_yes THEN itxt=' present in' ELSE itxt=' will not be written to'
- infotxt='0 HDF4 pre-defined attribute '+ncsa_str[i]+itxt+' the HDF file'
- INFOTXT_OUTPUT,infotxt
- ENDIF
- ENDFOR
- calfound=0
- FOR i=0,N_ELEMENTS(ncsa_cal)-1 DO BEGIN
- ni=WHERE((STRLOWCASE(nav) EQ ncsa_cal[i]) AND (calfound EQ 0),ncnt)
- IF ncnt NE 0 THEN BEGIN
- calfound=1
- infotxt='3 Datasets that have been scaled or had an offset applied are not permitted'
- INFOTXT_OUTPUT,infotxt
- ENDIF
- ENDFOR
-ENDELSE
-
-END ;Procedure Pre_Defined_Att_Checks
-
-
-
-PRO read_metadata, metafile, inf
-;Procedure to check Metadata contents and return as meta_arr - also:
-;1. Ensures uniform input (making ATTRIBUTE_NAME uppercase, deleting unwanted spaces etc).
-;2. Checks that attribute length falls within allowable limits
-;3. Checks for repeated, missing, new, obsolete attributes
-;4. Update FILE_META_VERSION with TAV and software version (tab_ver)
-
-;Please also note the following:
-;1. Any extra valid Global Attributes will be included (and added to the attr_arr_glob array).
-;2. If redundant attributes are detected they will be omitted from the Metadata written
-; to the HDF file.
-; ----------
-;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
-;
-; History:
-; 20050802: Original IDLCR8HDF Routine - Version 1.0
-; 20061012: Error messages modified; Allow VAR_MONOTONE as a Variable Attribute if it is present in the
-; Metadata; If DATA_TYPE is present in the Metadata then change it to DATA_LEVEL; Allow
-; additional Global Attributes; Puts Variable Attribute names in the same order as
-; attr_arr_data; Common variable definition WIDGET_WIN added - Version 2.0
-; 20080302: VAR_MONOTONE not included in the output Metadata if it is present in the input Metadata
-; and an AVDC style TAV file is read in; if DATA_TYPE is present in the Metadata it is changed
-; to DATA_LEVEL if an AVDC style TAV file is read in, and vice versa if an original style
-; Envisat table.dat file read in; Warnings added if VAR_MONOTONE attribute is omitted, when
-; changing DATA_TYPE to DATA_LEVEL (or vice versa), and if extra Global Attributes are
-; included in the Metadata; Checks for missing or repeated Global Attributes; Global
-; Attributes can now be listed in any order in the Metadata (will be written to the HDF file
-; in a standard order) - Version 3.0
-; 20091203: Stop using DATA_TYPE for EVDC style input (if found change to DATA_LEVEL); If VAR_MONOTONE
-; is present in the Metadata it is removed - Version 3.08
-; 20100205: Add RETURN command after all STOP_WITH_ERROR calls, which allows program to return to the
-; calling program if the reterr argument is included in the idlcr8hdf call - Version 3.09
-; 20101120: Add DATA_STOP_DATE and remove DATA_LEVEL from Global Attributes; Remove VAR_AVG_TYPE,
-; VAR_DIMENSION, and VIS attributes from Variable Attributes; Add call to GEOMS_RULE_CHANGES
-; to account for changes in Metadata setup between v3.0 and v4.0; Remove option to have a
-; filename as a value for free text attributes; Increase checks between DATA_VARIABLES and
-; VAR_NAME values - Version 4.0b0
-; 20111220: Call additional GEOMS_RULE_CHANGES; Perform checks on HDF4 pre-defined attributes
-; - Version 4.0b7
-; 20150127: Stop removing leading and trailing spaces from variable attribute values (when there is
-; only 1 value) to allow for possibility of VAR_FILL_VALUEs being an empty string
-; - Version 4.0b25
-; 20150811: Add checks for invalid FILE_META_VERSION entries - Version 4.0b29
-; 20150924: Change to v4.0b25 so that leading and trailing spaces are not removed from VAR_FILL_VALUE
-; entries only (see attr_arr_str) - Version 4.0b30
-; 20151012: Improve/fix checks for invalid whitespace in Metadata labels or values and add log output when
-; changes are made. Checks previously done in CHECK_METADATA moved to this routine
-; - Version 4.0b31
-; 20151021: Do not do checks on invalid whitespace for attributes that have numeric values - Version 4.0b32
-; 20151109: Add log message when adding/checking for missing mandatory variable attributes - Version 4.0b34
-; 20151116: Fix bug when doing whitespace checks and there is no metadata entry - Version 4.0b35
-; 20180528: Fix bug when adding non-standard global attributes to meta_arr - Version 4.0b47
-; 20191205: Do checks for sub-value separators (. or ; or _) being at the start or end of attribute
-; values, and for consecutive sub-value separators (e.g. ;;). This can occur as the STRSPLIT
-; command can remove these while testing sub-values, so they are not detected as part of
-; any tests - Version 4.0b53
-; 20200311: Do not include '.' separator when doing checks for sub-value separators at the start or end of
-; the attribute values e.g. allow DO_NAME=Boyd;Ian S. - Version 4.0b54
-; 20201020: Generate message for invalid metadata attributes, previously removed them 'quietly'
-; - Version 4.0b56
-; 20201026: Fix bug from 4.0b56 which caused an error message if _FillValue was present in the GEOMS file
-; - Version 4.0b57
-;
-; Inputs: metafile - Either, the filename of the input file containing the Metadata or, if program
-; input is via string array and Structure, a string array containing the Global
-; and Variable Attributes.
-; inf - flag, where 0=Metafile is an array; 1=Metafile is a file.
-;
-; Output: meta_arr - a string array containing the Global and Variable Attributes, which have been
-; checked and uniformally formatted.
-;
-; Called by: IDLCR8HDF
-;
-; Subroutines Called: STOP_WITH_ERROR (if error state detected);
-; INFOTXT_OUTPUT, GEOMS_RULE_CHANGES
-; Possible Conditions for STOP_WITH_ERROR call (plus [line number] where called):
-; 1. Metadata value contains too many characters
-; 2. No or invalid DATA_VARIABLES and VAR_NAME attributes present in the metadata
-; 3. DATA_VARIABLES or VAR_NAME value appears more than once
-; 4. No valid Metadata in command line parameter file or arrays
-; 5. VAR_NAME does not match corresponding DATA_VARIABLES value
-; 6. Number of DATA_VARIABLES values does not match the number of VAR_NAME occurrences
-; 7. A required Global Attribute Label is not present in the Metadata
-; 8. A required Global Attribute Label is repeated in the Metadata
-; 9. Number of Attribute occurrences does not match the number of datasets
-; 10. DATA_QUALITY is not present but is a mandatory attribute when DATA_TEMPLATE is reported
-;
-; Information Conditions (when the program is able to make changes):
-; 1. Invalid Attribute Label ignored
-; 2. Missing Global Attribute added
-; 3. DATA_VARIABLES attribute added to metadata based on VAR_NAME values
-; 4. Values for DATA_VARIABLES appended to the metadata based on VAR_NAME values
-; 5. VAR_NAME value appended to label based on DATA_VARIABLES value
-; 6. New Global Attribute added
-
-COMMON TABLEDATA
-COMMON METADATA
-COMMON DATA
-COMMON WIDGET_WIN
-
-attr_arr_glob=['PI_NAME','PI_AFFILIATION','PI_ADDRESS','PI_EMAIL',$
- 'DO_NAME','DO_AFFILIATION','DO_ADDRESS','DO_EMAIL',$
- 'DS_NAME','DS_AFFILIATION','DS_ADDRESS','DS_EMAIL',$
- 'DATA_DESCRIPTION','DATA_DISCIPLINE','DATA_GROUP','DATA_LOCATION',$
- 'DATA_SOURCE','DATA_VARIABLES','DATA_START_DATE','DATA_STOP_DATE',$
- 'DATA_FILE_VERSION','DATA_MODIFICATIONS','DATA_CAVEATS','DATA_RULES_OF_USE',$
- 'DATA_ACKNOWLEDGEMENT','DATA_QUALITY','DATA_TEMPLATE','DATA_PROCESSOR',$
- 'FILE_NAME','FILE_GENERATION_DATE','FILE_ACCESS','FILE_PROJECT_ID','FILE_DOI', $
- 'FILE_ASSOCIATION','FILE_META_VERSION']
-
-attr_arr_glob_opt=['DATA_DESCRIPTION','DATA_MODIFICATIONS','DATA_CAVEATS','DATA_RULES_OF_USE',$
- 'DATA_ACKNOWLEDGEMENT','DATA_QUALITY','DATA_TEMPLATE','DATA_PROCESSOR',$
- 'FILE_PROJECT_ID','FILE_ASSOCIATION']
-
-attr_arr_glob_ins=['DATA_START_DATE','DATA_STOP_DATE','FILE_GENERATION_DATE', $
- 'FILE_DOI','FILE_META_VERSION']
-
-attr_arr_glob_prov=['DATA_TEMPLATE','DATA_QUALITY']
-
-attr_arr_data=['VAR_NAME','VAR_DESCRIPTION','VAR_NOTES','VAR_SIZE','VAR_DEPEND',$
- 'VAR_DATA_TYPE','VAR_UNITS','VAR_SI_CONVERSION','VAR_VALID_MIN',$
- 'VAR_VALID_MAX','VAR_FILL_VALUE']
-
-attr_arr_data_opt=['VAR_NOTES']
-
-;attr_arr_str=['VAR_FILL_VALUE','_fillvalue'] ;do not take STRTRIM of these values
-
-;Attributes that have user defined values
-attr_free=['DATA_DESCRIPTION','DATA_MODIFICATIONS','DATA_CAVEATS',$
- 'DATA_RULES_OF_USE','DATA_ACKNOWLEDGEMENT','DATA_QUALITY', $
- 'DATA_PROCESSOR','FILE_PROJECT_ID','FILE_ASSOCIATION', $
- 'VAR_DESCRIPTION','VAR_NOTES']
-
-;Atrributes that have numeric values (except for string datatype)
-num_atts=['VAR_VALID_MIN','VAR_VALID_MAX','VAR_FILL_VALUE','_fillvalue','valid_range']
-
-;To check for pre-defined attributes
-ncsa=['long_name','units','format','coordsys','valid_range','_fillvalue','scale_factor', $
- 'scale_factor_err','add_offset','add_offset_err','calibrated_nt']
-
-;Possible sub-value separators
-sv_sep=['.','_',';']
-
-nchar=65535L ;limit on number of characters
-
-;possible error messages for this procedure
-errtxt=STRARR(9)
-proname='Read_Metadata procedure: '
-errtxt[0]=' value contains too many characters (maximum permitted: '+STRTRIM(nchar,2)+')'
-errtxt[1]='No or invalid DATA_VARIABLES and VAR_NAME attributes present in the metadata'
-errtxt[2]=' value appears more than once: '
-errtxt[3]='No valid Metadata in input'
-errtxt[4]='VAR_NAME does not match corresponding DATA_VARIABLES value: '
-errtxt[5]='Number of DATA_VARIABLES values does not match the number of VAR_NAME occurrences '
-errtxt[6]='Global Attribute Label not present in the Metadata: ' ;now INFOTXT
-errtxt[7]='Global Attribute Label repeated in the Metadata: '
-errtxt[8]=' occurrences does not match the number of datasets (should be '
-
-dum='' & nrline=0L & dvfound=-1L
-nd=N_ELEMENTS(attr_arr_data)
-ng=N_ELEMENTS(attr_arr_glob)
-nna=N_ELEMENTS(num_atts)
-dsei=WHERE(attr_arr_glob EQ 'DS_EMAIL') ;any new global attributes should be after this attribute
-vni=WHERE(attr_arr_data EQ 'VAR_NAME') ;used in VAR_NAME checks
-ndv=0 ;will reset to the number of datasets
-
-IF inf EQ 1 THEN BEGIN ;count number of lines in the file
- OPENR,lu,metafile,/GET_LUN
- WHILE NOT EOF(lu) DO BEGIN
- READF,lu,dum & nrline=nrline+1L
- ENDWHILE
- FREE_LUN,lu
- mh=STRARR(nrline) & mv_lng=LON64ARR(nrline) & mv_dbl=DBLARR(nrline)
- OPENR,lu,metafile,/GET_LUN
- READF,lu,mh
- FREE_LUN,lu
-ENDIF ELSE mh=metafile ;metadata is already in an array
-mhhold=STRTRIM(mh,1) ;create temporary array and remove leading blanks from all the entries
-mhgood=INTARR(N_ELEMENTS(mh))+1 ;will be assigned zero for unwanted metadata lines
-
-;Check for Redundant Attribute Labels
-GEOMS_RULE_CHANGES,0,mhhold,mhgood
-
-;test1 - check first character is A-Z or a-z; test2 - check for '=' and not a redundant label
-test1=(((BYTE(STRMID(mhhold,0,1)) GE 65) AND (BYTE(STRMID(mhhold,0,1)) LE 90)) OR $
- ((BYTE(STRMID(mhhold,0,1)) GE 97) AND (BYTE(STRMID(mhhold,0,1)) LE 122)) OR $
- (STRMID(STRUPCASE(mhhold),0,10) EQ '_FILLVALUE'))
-test2=(STRPOS(mhhold,'=') GE 1) AND (mhgood EQ 1)
-
-nri=WHERE((test1) AND (test2),nrline)
-IF nrline EQ 0 THEN BEGIN
- STOP_WITH_ERROR,o3[3]+proname,errtxt[3],lu & RETURN
-ENDIF
-
-;Generate an error for invalid metadata attributes
-bnri=WHERE(~test1,bnrline)
-IF bnrline NE 0 THEN BEGIN
- FOR i=0L,bnrline-1L DO BEGIN
- infotxt='2 Invalid Metadata Attribute: '+mhhold[bnri[i]]+'|'
- infotxt=infotxt+' removed from the metadata saved to the output file'
- INFOTXT_OUTPUT,infotxt
- ENDFOR
-ENDIF
-
-;Set holding arrays for metadata
-mh=mh[nri] & mv_lng=mv_lng[nri] & mv_dbl=mv_dbl[nri]
-lu=-1
-m_v0=STRARR(nrline) & m_v1=m_v0 ;to hold metadata labels and values
-m_v2=INTARR(nrline) ;global/variable attribute switch
-
-;Separate out Metadata entries
-FOR i=0L,nrline-1L DO BEGIN
- ;separate out the Meta entry into two components
- eqpos=STRPOS(mh[i],'=')
- m_v0[i]=STRMID(mh[i],0,eqpos)
- IF eqpos EQ STRLEN(mh[i])-1 THEN m_v1[i]='' $ ;ie. the last character is the '='
- ELSE m_v1[i]=STRMID(mh[i],eqpos+1)
-ENDFOR
-
-;Check for any attribute label that is not uppercase (and not an HDF pre-defined label) or has leading or trailing spaces
-ui=WHERE(m_v0 NE STRUPCASE(m_v0),ucnt)
-lti=WHERE(m_v0 NE STRTRIM(m_v0,2),ltcnt)
-
-IF ltcnt NE 0L THEN BEGIN ;some attribute labels have leading or trailing spaces
- m_v0=STRTRIM(m_v0,2)
- IF qa_yes THEN BEGIN ;only report if doing QA, otherwise fix quietly
- infotxt='2 Leading or trailing spaces in Metadata Attribute Labels not permitted'
- INFOTXT_OUTPUT,infotxt
- ENDIF
-ENDIF
-
-IF ucnt NE 0L THEN BEGIN ;some attribute labels are not fully uppercase
- writeonce=0 ;write out information text once only
- FOR i=0L,ucnt-1L DO BEGIN
- ;check for HDF predefined labels which should be lowercase
- pi=WHERE(STRLOWCASE(m_v0[ui[i]]) EQ ncsa,pcnt)
- IF pcnt EQ 0 THEN BEGIN ;label expected to be in uppercase
- m_v0[ui[i]]=STRUPCASE(m_v0[ui[i]])
- IF writeonce EQ 0 THEN BEGIN ;write INFORMATION text to log
- writeonce=1
- IF qa_yes THEN itxt='must be' ELSE itxt='made'
- infotxt='2 Metadata Attribute Labels '+itxt+' UPPERCASE'
- INFOTXT_OUTPUT,infotxt
- ENDIF
- ENDIF
- ENDFOR
-ENDIF
-
-;Check for and remove leading and trailing spaces in Metadata (sub-)values (except for free-text attributes
-;and VAR_FILL_VALUEs). Also for non-free-text attributes check if the first and last characters are
-;sub-value separators, or if there are consecutive sub-value separators in the attribute value.
-;Also check for values with excessive length.
-IF qa_yes THEN wstxt='are not permitted' ELSE wstxt='removed' ;used when reporting white space in sub-values
-
-FOR i=0L,nrline-1L DO BEGIN
- ;si=WHERE(STRUPCASE(m_v0[i]) EQ STRUPCASE(num_atts),scnt)
- ;IF scnt EQ 0 THEN si=WHERE(m_v0[i] EQ attr_free,scnt)
- si=WHERE(m_v0[i] EQ attr_free,scnt)
- IF scnt EQ 0 THEN BEGIN ;not a free text attribute
- nmi=WHERE(STRUPCASE(m_v0[i]) EQ STRUPCASE(num_atts),nmcnt) ;is it an attribute that should have a numeric value
- res=STRSPLIT(m_v1[i],';',/Extract,COUNT=svcnt)
- spi=WHERE(STRTRIM(res,2) NE res,spcnt)
- IF spcnt NE 0 THEN BEGIN
- IF (svcnt EQ 1) AND (qa_yes) THEN BEGIN
- IF (nmcnt NE 0) AND (STRTRIM(res[0],2) NE '') THEN infotxt='' $ value should be a number not a string so do not check for spaces
- ELSE IF res[0] EQ ' ' THEN infotxt='' $ there could be a single space in the HDF file if there is no entry
- ELSE infotxt='2 Leading or trailing spaces in '+m_v0[i]+'='+m_v1[i]+' value '+wstxt
- ENDIF ELSE IF svcnt GT 1 THEN BEGIN
- infotxt='2 Leading or trailing spaces between '+m_v0[i]+' sub-values '+wstxt
- ENDIF ELSE infotxt='' ;Quietly fix if only single value and not doing QA
- IF infotxt NE '' THEN INFOTXT_OUTPUT,infotxt
- res=STRTRIM(res,2)
- m_v1[i]=res[0]
- IF svcnt GT 1 THEN FOR j=1,svcnt-1 DO m_v1[i]=m_v1[i]+';'+res[j]
- ENDIF
-
- IF nmcnt EQ 0 THEN BEGIN
- ;Check that first and last characters of the metadata value aren't sub-value separators
- ;and that there are no consecutive sub-value separators
- mv1len=STRLEN(m_v1[i]) ;string length of attribute value
- mv1start=STRMID(m_v1[i],0,1) ;first character
- mv1end=STRMID(m_v1[i],mv1len-1L,1) ;last character
- FOR k=0,1 DO BEGIN
- writeonce=1B ;so that error message is only written once for each attribute
- IF k EQ 0 THEN $
- infotxt='3 Sub-value separators at the start or end of the attribute value are not permitted' $
- ELSE infotxt='3 Consecutive sub-value separators are not permitted'
- FOR j=0,N_ELEMENTS(sv_sep)-1 DO BEGIN
- spchk=sv_sep[j]+sv_sep[j]
- ;Note: in some cases a '.' may be present at the end of a sub-value e.g. DO_NAME=Boyd;Ian S.
- IF (k EQ 0) AND (sv_sep[j] NE '.') THEN test=(mv1start EQ sv_sep[j]) OR (mv1end EQ sv_sep[j]) $
- ELSE IF k EQ 1 THEN test=STRPOS(m_v1[i],spchk) NE -1 $
- ELSE test=0B
- IF (test) AND (writeonce) THEN BEGIN
- IF mv1len GT 50 THEN infotxt=infotxt+' in '+m_v0[i]+' attribute' $
- ELSE infotxt=infotxt+': '+m_v0[i]+'='+m_v1[i]
- INFOTXT_OUTPUT,infotxt
- writeonce=0B
- ENDIF
- ENDFOR
- ENDFOR
- ENDIF
- ENDIF
- ;check for string length of attribute value
- IF STRLEN(m_v1[i]) GT nchar THEN BEGIN
- infotxt='3 '+m_v0[i]+errtxt[0]
- INFOTXT_OUTPUT,infotxt
- ENDIF
-ENDFOR
-
-FOR i=0L,nrline-1L DO BEGIN
- ;Do check for DATA_VARIABLES and separate out the values
- IF m_v0[i] EQ 'DATA_VARIABLES' THEN BEGIN
- dv=STRSPLIT(m_v1[i],' ;',/Extract,COUNT=ndv) & dvfound=i
- ENDIF
-
- ;Check for FILE_META_VERSION and add value derived from READ_TABLEFILE if not doing QA
- IF m_v0[i] EQ 'FILE_META_VERSION' THEN BEGIN
- IF (~qa_yes) AND (m_v1[i] NE tab_ver) THEN BEGIN
- m_v1[i]=tab_ver
- infotxt='0 FILE_META_VERSION updated with corrected TAV file version and tool name'
- INFOTXT_OUTPUT,infotxt
- ENDIF ELSE IF qa_yes THEN BEGIN
- ;Check Revision number is less than or equal to the TAV revision number, must be '04''R''xxx'
- revno=STRSPLIT(m_v1[i],';',/EXTRACT,count=fmvcnt) & revno=STRTRIM(revno,2)
- tavno=STRSPLIT(tab_ver,';',/EXTRACT) & tavno=STRTRIM(tavno,2)
- IF fmvcnt NE 2 THEN BEGIN
- infotxt='2 FILE_META_VERSION values must include the TAV file version and the HDF file '
- infotxt=infotxt+'creation tool name e.g. FILE_META_VERSION='+tavno[0]+';CUSTOM'
- INFOTXT_OUTPUT,infotxt
- ENDIF
- revok=0 ;default is to report a TAV file version error unless revok eq 3
- IF STRLEN(revno[0]) EQ STRLEN(tavno[0]) THEN BEGIN
- vrev=STRMID(revno[0],0,2) & vtav=STRMID(tavno[0],0,2)
- rrev=STRMID(revno[0],2,1) & rtav=STRMID(tavno[0],2,1)
- srev=STRMID(revno[0],3) & stav=STRMID(tavno[0],3)
- IF IS_A_NUMBER_HDF(vrev) THEN BEGIN
- IF (FIX(vrev) GE 4) AND (FIX(vrev) LE FIX(vtav)) THEN revok++ ;TAV version OK
- ENDIF
- IF rrev EQ rtav THEN revok++ ;'R' is present as the third character
- IF (IS_A_NUMBER_HDF(srev)) AND (revok eq 2) THEN BEGIN
- IF (FIX(srev) LE FIX(stav)) OR ((FIX(srev) GT FIX(stav)) AND (FIX(vrev) LT FIX(vtav))) THEN revok++ ;TAV sub-version OK
- ENDIF
- ENDIF
- IF revok NE 3 THEN BEGIN
- infotxt='2 FILE_META_VERSION sub-value '+revno[0]+' is not valid. It must be the TAV file version e.g. '+tavno[0]
- INFOTXT_OUTPUT,infotxt
- ENDIF
- ENDIF
- ENDIF
-
- ;Check whether a Global Attribute (1), GEOMS Variable Attribute (2), or pre-defined Variable Attribute (3)
- gi=WHERE(m_v0[i] EQ attr_arr_glob,gcnt)
- vi=WHERE(m_v0[i] EQ attr_arr_data,vcnt)
- pi=WHERE(STRLOWCASE(m_v0[i]) EQ ncsa,pcnt)
- IF gcnt NE 0 THEN m_v2[i]=1 $
- ELSE IF vcnt NE 0 THEN m_v2[i]=2 $
- ELSE IF pcnt NE 0 THEN m_v2[i]=3 $
- ELSE BEGIN ;check for new Global Attribute
- IF i GT 0 THEN test1=m_v2[i-1] EQ 1 ELSE test1=1 ;i.e. the previous attribute was also a global attribute
- test2=(STRMID(m_v0[i],0,3) NE 'VAR') AND (STRMID(m_v0[i],0,3) NE 'VIS') ;does not start with 'VAR' or 'VIS'
- test3=STRPOS(m_v0[i],' ') EQ -1 ;No spaces in the label
- ;Check for possible mispellings of an actual attribute by removing '_' and comparing
- gv_chk=STRUPCASE(STRJOIN(STRSPLIT(m_v0[i],' _',/EXTRACT),/SINGLE))
- test4=1
- FOR j=0,ng-1 DO BEGIN
- ga_j=STRUPCASE(STRJOIN(STRSPLIT(attr_arr_glob[j],' _',/EXTRACT),/SINGLE))
- IF gv_chk EQ ga_j THEN test4=0 ;Match with actual Global Attribute so do not keep
- ENDFOR
- IF (test1) AND (test2) AND (test3) AND (test4) THEN m_v2[i]=1 ;it is considered a new global attribute
- ENDELSE
-ENDFOR
-
-;Check for any invalid attribute labels and remove
-bi=WHERE(m_v2 EQ 0,bcnt)
-IF bcnt NE 0 THEN BEGIN
- FOR i=0,bcnt-1 DO BEGIN
- infotxt='2 Unknown Attribute Label '+m_v0[bi[i]]+'| ignored'
- INFOTXT_OUTPUT, infotxt
- ENDFOR
-ENDIF
-;resize arrays, or stop the program if no valid data
-gi=WHERE(m_v2 NE 0,nrline)
-IF nrline NE 0 THEN BEGIN
- m_v0=m_v0[gi] & m_v1=m_v1[gi] & m_v2=m_v2[gi]
- mv_lng=mv_lng[gi] & mv_dbl=mv_dbl[gi]
-ENDIF ELSE BEGIN
- STOP_WITH_ERROR,o3[3]+proname,errtxt[3],lu & RETURN
-ENDELSE
-
-;Check to see that Global and Variable Attributes are present
-gi=WHERE(m_v2 EQ 1,gcnt)
-vi=WHERE(m_v2 EQ 2,vcnt)
-pi=WHERE(m_v2 EQ 3,pcnt) ;to do pre-defined attribute checks if necessary
-IF (gcnt EQ 0) OR (vcnt EQ 0) THEN BEGIN
- STOP_WITH_ERROR,o3[3]+proname,errtxt[3],lu & RETURN
-ENDIF
-
-;Do QA on pre-defined attributes
-IF pcnt NE 0 THEN PRE_DEFINED_ATT_CHECKS,-1,m_v0[pi]
-
-;Do DATA_VARIABLES checks
-IF ndv EQ 0 THEN BEGIN
- ;If no DATA_VARIABLES values found then try to determine values from VAR_NAME values
- dvi=WHERE((m_v0 EQ 'VAR_NAME') AND (m_v1 NE ''),ndv)
- IF ndv NE 0 THEN BEGIN
- dv=m_v1[dvi]
- ;Create or append to DATA_VARIABLES attribute
- IF dvfound EQ -1L THEN BEGIN ;increase array sizes by 1 to accommodate DATA_VARIABLES
- dvfound=nrline & nrline=nrline+1L
- m_v0=[m_v0,'DATA_VARIABLES'] & m_v1=[m_v1,''] & m_v2=[m_v2,1]
- mv_lng=[mv_lng,0LL] & mv_dbl=[mv_dbl,0.D]
- infotxt='2 Missing DATA_VARIABLES Global Attribute| added to metadata based on VAR_NAME values'
- ENDIF ELSE BEGIN
- infotxt='2 Missing DATA_VARIABLES attribute values| appended to the metadata based on VAR_NAME values'
- ENDELSE
- INFOTXT_OUTPUT, infotxt
- ENDIF ELSE BEGIN
- STOP_WITH_ERROR,o3[3]+proname,errtxt[1],lu & RETURN
- ;No or invalid DATA_VARIABLES and VAR_NAME attributes present in the metadata so cannot proceed
- ENDELSE
-ENDIF
-;Check for repeated DATA_VARIABLES
-FOR i=0,ndv-1 DO BEGIN
- ri=WHERE(STRUPCASE(dv[i]) EQ STRUPCASE(dv),rcnt)
- IF (rcnt GT 1) AND (dv[i] NE '') THEN BEGIN
- STOP_WITH_ERROR,o3[3]+proname,'DATA_VARIABLES'+errtxt[2]+dv[i],lu & RETURN
- ENDIF
-ENDFOR
-
-;Do VAR_NAME checks
-vi=WHERE(m_v0 EQ attr_arr_data[vni[0]],vcnt) ;identify VAR_NAME indices
-IF vcnt NE 0 THEN BEGIN
- ;Check if VAR_NAME values are present, if not add value from DATA_VARIABLES (if number of DATA_VARIABLES values match vcnt)
- IF ndv EQ vcnt THEN BEGIN
- FOR i=0,vcnt-1 DO BEGIN
- IF (m_v1[vi[i]] NE '') AND (m_v1[vi[i]] NE dv[i]) THEN BEGIN
- STOP_WITH_ERROR,o3[3]+proname,errtxt[4]+'('+m_v1[vi[i]]+'/'+dv[i]+')',lu & RETURN
- ENDIF ELSE IF m_v1[vi[i]] EQ '' THEN BEGIN
- m_v1[vi[i]]=dv[i]
- infotxt='2 Missing VAR_NAME value for '+m_v0[vi[i]]+'|. '+dv[i]+'| appended to label based on DATA_VARIABLES value
- INFOTXT_OUTPUT,infotxt
- ENDIF
- ENDFOR
- ENDIF ELSE BEGIN ;number of DATA_VARIABLES values does not match the number of VAR_NAME occurrences
- STOP_WITH_ERROR,o3[3]+proname,errtxt[5]+'('+STRTRIM(ndv,2)+'/'+STRTRIM(vcnt,2)+')',lu & RETURN
- ENDELSE
- ;Check for repeated VAR_NAMES
- FOR i=0,vcnt-1 DO BEGIN
- ri=WHERE(STRUPCASE(m_v1[vi[i]]) EQ STRUPCASE(m_v1[vi]),rcnt)
- IF (rcnt GT 1) AND (m_v1[vi[i]] NE '') THEN BEGIN
- STOP_WITH_ERROR,o3[3]+proname,'VAR_NAME'+errtxt[2]+m_v1[vi[i]],lu & RETURN
- ENDIF
- ENDFOR
-ENDIF ELSE BEGIN ;No VAR_NAME labels so create from DATA_VARIABLES
- FOR i=0,ndv-1 DO BEGIN
- m_v0=[m_v0,'VAR_NAME'] & m_v1=[m_v1,dv[i]] & m_v2=[m_v2,2]
- mv_lng=[mv_lng,0LL] & mv_dbl=[mv_dbl,0.D]
- IF i EQ 0 THEN vi=[nrline] ELSE vi=[vi,nrline+1L]
- ENDFOR
- infotxt='1 Missing VAR_NAME attribute labels and values| added to metadata based on DATA_VARIABLES values'
- INFOTXT_OUTPUT,infotxt
-ENDELSE
-
-;Check for obsolete FTIR, UVVIS.DOAS, LIDAR, and MWR DATA_SOURCE_01 values and modify of possible
-dsi=WHERE(m_v0 EQ 'DATA_SOURCE',dsicnt)
-vni=WHERE(m_v0 EQ 'VAR_NAME',vncnt)
-;extract VAR_NAME values
-dv=STRARR(vncnt) & data_source=''
-FOR i=0,vncnt-1 DO dv[i]=m_v1[vni[i]]
-IF dsicnt EQ 1 THEN BEGIN
- IF m_v1[dsi[0]] NE '' THEN BEGIN
- data_source=m_v1[dsi[0]] & dschange=data_source
- GEOMS_RULE_CHANGES,6,vncnt,dv,data_source
- IF data_source NE dschange THEN m_v1[dsi[0]]=data_source
- ENDIF
-ENDIF
-
-;Check for missing Global Attributes which have values which can be calculated by the program
-FOR i=0,N_ELEMENTS(attr_arr_glob_ins)-1 DO BEGIN
- gaii=WHERE(m_v0 EQ attr_arr_glob_ins[i],gaicnt)
- IF gaicnt EQ 0 THEN BEGIN ;missing mandatory global attribute so add to the array
- IF attr_arr_glob_ins[i] EQ 'FILE_META_VERSION' THEN itxt=tab_ver ELSE itxt=''
- nrline=nrline+1L
- m_v0=[m_v0,attr_arr_glob_ins[i]] & m_v1=[m_v1,itxt] & m_v2=[m_v2,1]
- mv_lng=[mv_lng,0LL] & mv_dbl=[mv_dbl,0.D]
- infotxt='2 Missing mandatory Global Attribute '+attr_arr_glob_ins[i]+'| added'
- INFOTXT_OUTPUT, infotxt
- ENDIF
-ENDFOR
-;Do DATA_TEMPLATE/DATA_QUALITY checks
-ngp=N_ELEMENTS(attr_arr_glob_prov)
-ga_chk=STRARR(ngp) ;initialize DATA_TEMPLATE/QUALITY values
-GEOMS_RULE_CHANGES,9,m_v0,m_v1,ga_chk
-;Check to see if Global Attributes have to be added
-FOR i=0,ngp-1 DO BEGIN
- IF ga_chk[i] NE '' THEN BEGIN
- m_v0=[m_v0,attr_arr_glob_prov[i]]
- IF i EQ 0 THEN m_v1=[m_v1,ga_chk[i]] ELSE m_v1=[m_v1,'']
- m_v2=[m_v2,1] & nrline=nrline+1L
- mv_lng=[mv_lng,0LL] & mv_dbl=[mv_dbl,0.D]
- IF i EQ 0 THEN itxt='='+ga_chk[i] ELSE itxt=''
- infotxt='2 Missing Global Attribute '+attr_arr_glob_prov[i]+itxt+'| added'
- INFOTXT_OUTPUT, infotxt
- ENDIF
-ENDFOR
-
-;Calculate expected size of meta_arr array
-gi=WHERE(m_v2 EQ 1,gcnt)
-IF gcnt GT ng THEN ngv=gcnt ELSE ngv=ng
-n_arr=ngv+(nd*ndv)
-meta_arr=STRARR(n_arr) & mlh=LON64ARR(n_arr) & mdh=DBLARR(n_arr)
-;mlh and mdh are holding arrays to hold numeric values in the metadata (will be renamed mv_lng and mv_dbl)
-
-;Check for missing or repeated standard Global Attributes
-i=0
-WHILE i LE ng-1 DO BEGIN
- gai=WHERE(m_v0[gi] EQ attr_arr_glob[i],gacnt)
- IF gacnt EQ 0 THEN BEGIN ;Check to see if Global Attribute Optional or Mandatory
- gaoi=WHERE(attr_arr_glob[i] EQ attr_arr_glob_opt, gaocnt)
- IF gaocnt EQ 0 THEN BEGIN ;it is mandatory, therefore missing
- infotxt='3 Global Attribute Label not present in the Metadata: '+attr_arr_glob[i]
- INFOTXT_OUTPUT,infotxt
- ENDIF
- ENDIF ELSE IF gacnt GT 1 THEN BEGIN ;Global Attribute Repeated
- STOP_WITH_ERROR,o3[3]+proname,errtxt[7]+attr_arr_glob[i],lu & RETURN
- ENDIF
- IF gacnt EQ 0 THEN BEGIN ;remove attribute from the attr_arr_glob array
- attr_arr_glob=[attr_arr_glob[0:i-1],attr_arr_glob[i+1:ng-1]]
- ng=ng-1
- ENDIF ELSE i=i+1
-ENDWHILE
-
-;Check for all Standard Global Attributes, add any new ones, and put in expected order
-xg=0 & mi=0
-FOR i=0,gcnt-1 DO BEGIN
- gai=WHERE(m_v0[gi[i]] EQ attr_arr_glob,gacnt)
- CASE 1 OF
- gacnt EQ 0: BEGIN ;New Global Attribute
- IF i EQ 0 THEN BEGIN
- attr_arr_glob=[m_v0[gi[i]],attr_arr_glob]
- ENDIF ELSE IF i EQ ng THEN BEGIN
- attr_arr_glob=[attr_arr_glob,m_v0[gi[i]]]
- ENDIF ELSE BEGIN
- attr_arr_glob=[attr_arr_glob[0:mi],m_v0[gi[i]],attr_arr_glob[mi+1:N_ELEMENTS(attr_arr_glob)-1]]
- mi=mi+1
- ENDELSE
- ;IF (i GE ng-(1+xg)) OR (i LE dsei) THEN BEGIN
- ; attr_arr_glob=[attr_arr_glob,m_v0[gi[i]]] & mi=ng
- ; IF i LE dsei THEN xg=xg+1
- ;ENDIF ELSE BEGIN
- ; attr_arr_glob=[attr_arr_glob[0:mi],m_v0[gi[i]],attr_arr_glob[mi+1:ng-1]]
- ; mi=mi+1
- ;ENDELSE
- ng=ng+1
- infotxt='1 Non-standard Global Attribute '+m_v0[gi[i]]+'| added'
- INFOTXT_OUTPUT, infotxt
- END
- ELSE: mi=i ;gai[0]
- ENDCASE
-ENDFOR
-FOR i=0,gcnt-1 DO BEGIN
- gai=WHERE(m_v0[gi[i]] EQ attr_arr_glob,gacnt)
- meta_arr[gai[0]]=m_v0[gi[i]]+'='+m_v1[gi[i]]
-ENDFOR
-
-;Check for repeated new Global Attributes
-FOR i=0,ng-1 DO BEGIN
- gai=WHERE(m_v0[gi] EQ attr_arr_glob[i],gacnt)
- IF gacnt GT 1 THEN BEGIN ;Global Attribute Repeated
- STOP_WITH_ERROR,o3[3]+proname,errtxt[7]+attr_arr_glob[i],lu & RETURN
- ENDIF
-ENDFOR
-
-;Do checks for all the variable attributes and put in expected order
-vni=-1
-FOR i=0,nd-1 DO BEGIN
- si=ng+i ;start index of meta_arr for variable attributes
- di=WHERE(m_v0 EQ attr_arr_data[i],dcnt)
- IF attr_arr_data[i] EQ 'VAR_NAME' THEN vni=di ;to enable checks on optional variable attributes
- IF dcnt EQ ndv THEN BEGIN ;i.e. correct number of attribute labels found so add to meta_arr in correct order
- FOR j=0,ndv-1 DO BEGIN
- meta_arr[si+(nd*j)]=m_v0[di[j]]+'='+m_v1[di[j]]
- mlh[si+(nd*j)]=mv_lng[di[j]] & mdh[si+(nd*j)]=mv_dbl[di[j]]
- ENDFOR
- ENDIF ELSE IF dcnt EQ 0 THEN BEGIN ;attribute not present - so add with no label
- oi=WHERE(attr_arr_data[i] EQ attr_arr_data_opt,ocnt) ;check whether missing attribute is optional
- IF ocnt EQ 0 THEN BEGIN
- FOR j=0,ndv-1 DO meta_arr[si+(nd*j)]=attr_arr_data[i]+'='
- IF qa_yes THEN itxt=' must be present in the metadata' ELSE itxt=' added to the metadata with no value'
- infotxt='2 Mandatory variable attribute '+attr_arr_data[i]+itxt
- INFOTXT_OUTPUT,infotxt
- ENDIF
- ENDIF ELSE BEGIN ;incorrect number of attributes - report with error unless attribute is optional
- oi=WHERE(attr_arr_data[i] EQ attr_arr_data_opt,ocnt)
- IF (ocnt NE 0) AND (vni[0] GE 0) AND (di[0]-vni[0] GT 0) THEN BEGIN ;match VAR_NOTES attributes to the corresponding VAR_NAME index value
- FOR j=0,dcnt-1 DO BEGIN
- vnih=di[j]-vni[0] & jh=0
- FOR k=0,N_ELEMENTS(vni)-1 DO BEGIN
- dvn=di[j]-vni[k]
- IF (dvn LT vnih) AND (dvn GT 0) THEN BEGIN
- vnih=di[j]-vni[k] & jh=k
- ENDIF
- ENDFOR
- meta_arr[si+(nd*jh)]=m_v0[di[j]]+'='+m_v1[di[j]]
- mlh[si+(nd*jh)]=mv_lng[di[j]] & mdh[si+(nd*jh)]=mv_dbl[di[j]]
- ENDFOR
- ENDIF ELSE BEGIN ;Not optional or too much out of order, therefore report as an error
- STOP_WITH_ERROR,o3[3]+proname,'Number of '+attr_arr_data[i]+errtxt[8]+STRTRIM(ndv,2)+')',lu & RETURN
- ENDELSE
- ENDELSE
-ENDFOR
-
-;remove any empty meta_arr cells (can happen due to optional attributes)
-;and save numeric metadata values in correct order
-gi=WHERE(STRTRIM(meta_arr,2) NE '')
-meta_arr=meta_arr[gi] & mv_lng=mlh[gi] & mv_dbl=mdh[gi]
-
-;Call GEOMS_RULE_CHANGES to check for obsolete Dataset names and update if possible
-;Identify VAR_NAME and VAR_UNITS indices
-vni=WHERE(STRMID(meta_arr,0,8) EQ 'VAR_NAME',vncnt)
-vui=WHERE(STRMID(meta_arr,0,9) EQ 'VAR_UNITS',vucnt)
-vnchange=STRARR(vncnt) ;array to hold original VAR_NAME values if they have been changed
-;extract VAR_NAME values
-dv=STRARR(vncnt)
-FOR i=0,vncnt-1 DO BEGIN
- res=STRSPLIT(meta_arr[vni[i]],' =',/EXTRACT,COUNT=nres)
- IF nres EQ 2 THEN dv[i]=res[1]
-ENDFOR
-;Check for obsolete _START/STOP/INTEGRATION sub-values and replace with DATETIME.START/STOP
-;or INTEGRATION.TIME (for single occurences only). Also check for double underscores and
-;replace with single underscores in the Variable Names
-GEOMS_RULE_CHANGES,1,vncnt,dv
-
-;Check for other obsolete variable names
-IF vncnt EQ vucnt THEN BEGIN
- vuv=STRARR(vucnt)
- FOR i=0,vucnt-1 DO BEGIN
- res=STRSPLIT(meta_arr[vui[i]],'=',/EXTRACT,COUNT=nres)
- IF nres EQ 2 THEN vuv[i]=STRTRIM(res[1],2)
- ENDFOR
- GEOMS_RULE_CHANGES,5,vncnt,dv,vuv,data_source
-ENDIF
-;Update meta_arr in the case of modified Dataset name values
-vci=WHERE(vnchange NE '',vcres)
-IF vcres NE 0 THEN BEGIN ;need to update VAR_NAME and DATA_VARIABLES attribute values
- dvi=WHERE(STRMID(meta_arr,0,14) EQ 'DATA_VARIABLES')
- meta_arr[dvi[0]]='DATA_VARIABLES='
- FOR i=0,vncnt-1 DO BEGIN
- ;Check for new VAR_NAME value and update
- IF vnchange[i] NE '' THEN meta_arr[vni[i]]='VAR_NAME='+dv[i]
- ;Append to new DATA_VARIABLES list
- IF i EQ 0 THEN itxt='' ELSE itxt=';'
- meta_arr[dvi[0]]=meta_arr[dvi[0]]+itxt+dv[i]
- ENDFOR
-ENDIF
-
-;Check for obsolete CALVAL and EVDC FILE_ACCESS values and replace
-fai=WHERE(STRMID(meta_arr,0,11) EQ 'FILE_ACCESS')
-res=STRSPLIT(meta_arr[fai[0]],' =;',/EXTRACT,COUNT=facnt)
-IF facnt GE 2 THEN BEGIN
- fav=STRUPCASE(res[1:facnt-1]) & fachange=fav
- GEOMS_RULE_CHANGES,7,facnt-1,fav
- IF ARRAY_EQUAL(fav,fachange) EQ 0 THEN BEGIN ;FILE_ACCESS values changed so update meta_arr
- meta_arr[fai[0]]='FILE_ACCESS='
- FOR i=0,facnt-2 DO BEGIN
- IF i EQ 0 THEN itxt='' ELSE itxt=';'
- meta_arr[fai[0]]=meta_arr[fai[0]]+itxt+fav[i]
- ENDFOR
- ENDIF
-ENDIF
-
-;Check for VAR_UNITS=DIMENSIONLESS or NONE and fix if possible.
-;Need to check VAR_DATA_TYPE. If string then VAR_UNITS value should be blank, o/w replace with '1'
-vdi=WHERE(STRMID(meta_arr,0,13) EQ 'VAR_DATA_TYPE',vdcnt)
-IF (vdcnt NE 0) AND (vdcnt EQ vucnt) THEN BEGIN
- vdv=STRARR(vdcnt)
- FOR i=0,vdcnt-1 DO BEGIN
- res=STRSPLIT(meta_arr[vdi[i]],'=',/EXTRACT,COUNT=nres)
- IF nres EQ 2 THEN vdv[i]=STRUPCASE(STRTRIM(res[1],2))
- ENDFOR
- vuchange=vuv
- GEOMS_RULE_CHANGES,8,vucnt,vuv,vdv
- FOR i=0,vucnt-1 DO BEGIN ;Check for changes and update meta_arr
- IF STRUPCASE(vuchange[i]) NE vuv[i] THEN meta_arr[vui[i]]='VAR_UNITS='+vuv[i]
- ENDFOR
-ENDIF
-
-;Check for non-permitted characters (must be ISO646-US character set)
-GEOMS_RULE_CHANGES,10,meta_arr
-
-END ;Procedure Read_Metadata
-
-
-
-PRO extract_and_test, dentry, cpos, nel, ta1, ta2, taname, aname, ti
-;Procedure called by Check_Metadata to extract the relevant parts of tab_arr and test against
-;a corresponding Metadata value.
-; ----------
-;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
-;
-; History:
-; 20050802: Original IDLCR8HDF Routine - Version 1.0
-; 20050909: Make VAR_SI_CONVERSION values match the VAR_DATA_TYPE, and add fix for VAR_SI_CONVERSION
-; calculation rules e.g. if VAR_UNITS=cm m-3 then VAR_SI_CONVERSION=0;0.01;m-2 - Version 1.1
-; 20061012: Procedure simplified, with all portions of the code related to the VAR_UNITS and
-; VAR_SI_CONVERSION calculations moved to the Var_Units_Test routine; Common variable
-; definition WIDGET_WIN added - Version 2.0
-; 20100205: Add RETURN command after all STOP_WITH_ERROR calls, which allows program to return to the
-; calling program if the reterr argument is included in the idlcr8hdf call - Version 3.09
-; 20120313: Add option to check for Metadata Entries that are only different to the TAV entry
-; because of case sensitivity and/or use of incorrect apostrophe ("`" instead of "'")
-; - Version 4.0b8
-; 20140226: For AFFILIATION, if there is no match, do additional check of the acronym only and,
-; if present, correct the value - Version 4.0b19
-; 20150811: Add infoerr array. For these Global Attributes the program will no longer stop, but
-; will generate an ERROR message and continue if there is a problem with any of the
-; entries - Version 4.0b29
-; 20161230: Add COUNTRY and AFFILIATION to infoerr array and allow for this when creating infotxt
-; message - Version 4.0b41
-; 20171121: Fix error caused by incorrectly referring to the variable infoerr as inforerr
-; - Version 4.0b43
-; 20191205: Account for DATA_SOURCE being able to have 2 or 3 sub-values, where the third sub-
-; value is a VERSION_NAME - Version 4.0b53
-; 20220805: Ensure the 3-digit instrument ID is numeric (each character is now checked in
-; IS_A_NUMBER_HDF) - Version 4.0b60
-;
-; Inputs: dentry - the Metadata entry to be checked (everything after the '=')
-; cpos - a pattern used by StrSplit on DEntry to split it into its component parts e.g. ';' or '_'
-; nel - the number of sub-values in the set of tab_arr entries being checked (ta1 only)
-; ta1 - the set of tab_arr values used to check against the Metadata contents
-; ta2 - a second set of tab_arr values used for DATA_SOURCE checks, o/w set to ['']
-; taname - a combination of the Metadata Attribute name and Table Field/Label used for the
-; error messages
-; aname - the Metadata attribute name to be used for the error message
-; ti - the index value of the ta2 values to be checked. Used for DATA_SOURCE checks only
-;
-; Output: ti - the index value(s) of tab_arr which match the Metadata value (used in Check_Metadata
-; to do further checks on the ORIGINATOR values)
-;
-; Called by: CHECK_METADATA
-;
-; Subroutines Called: STOP_WITH_ERROR (if error state detected)
-; Possible Conditions for STOP_WITH_ERROR call (plus [line number] where called):
-; 1. Incorrect number of (sub-)values for a particular Metadata attribute
-; 2. Metadata value does not match any values in the Table Attribute Values file
-
-COMMON TABLEDATA
-COMMON WIDGET_WIN
-
-proname='Check_Metadata/Extract_And_Test procedures: ' & lu=-1L
-errtxt2=STRARR(2)
-errtxt2[0]='Number of (sub-)values for this attribute should be '
-errtxt2[1]='Doesn''t match any Table Attribute Values under FIELD: '
-
-infoerr=['COUNTRY','AFFILIATION','FILE_ACCESS','FILE_META_VERSION','DATA_LOCATION', $
- 'DATA_DISCIPLINE','DATA_GROUP'] ;do not stop the program for errors in these attributes
-
-;Check number of Metadata entry components is as expected
-achk=STRSPLIT(dentry,cpos,/Extract,COUNT=n_achk)
-oki=WHERE(n_achk EQ nel,okcnt) ;Note: DATA_SOURCE can have 2 or 3 sub-values (nel=[2,3] from 4.0b53)
-IF okcnt EQ 0 THEN BEGIN
- STOP_WITH_ERROR,o3[3]+proname+taname+'='+dentry+': ',errtxt2[0]+STRTRIM(nel,2)+'.',lu
- RETURN
-ENDIF
-
-;allow fix to be made to the affiliation if the acronym is correct but institute name is wrong
-n_ta1=N_ELEMENTS(ta1)
-IF (STRPOS(taname,'AFFILIATION') NE -1) AND (cpos NE '_') THEN BEGIN
- affchk=1 & acroval=STRTRIM(achk[1],2) & t_acrovals=STRARR(n_ta1)
-ENDIF ELSE affchk=0
-
-;check Metadata entry with relevant TAV entry or entries
-IF cpos EQ '_' THEN BEGIN ;DATA_SOURCE entry so need to treat components separately
- achk=STRCOMPRESS(achk,/Remove_All) & nel=1
- achkh=' '+achk[1]
- ;strip the last three characters (instrument ID) off achk(1)
- achk[1]=STRMID(achk[1],0,STRLEN(achk[1])-3)
- ;Check that the instrument ID is valid
- instidchk=STRMID(achkh,STRLEN(achkh)-3,3)
- IF ~IS_A_NUMBER_HDF(instidchk,2) THEN BEGIN
- infotxt='3 DATA_SOURCE 3-digit instrument identifier missing or invalid'
- INFOTXT_OUTPUT,infotxt
- ENDIF
-ENDIF ELSE achk=STRCOMPRESS(dentry,/Remove_All)
-FOR j=0,n_ta1-1 DO BEGIN
- ;extract valid portion of the entries (depending on ATTRIBUTE name)
- tchk=STRSPLIT(ta1[j],';',/Extract)
- ta1[j]=STRCOMPRESS(tchk[0],/Remove_All)
- IF nel GT 1 THEN $
- FOR k=1,nel-1 DO ta1[j]=ta1[j]+';'+STRCOMPRESS(tchk[k],/Remove_All)
- IF affchk EQ 1 THEN t_acrovals[j]=tchk[1]
-ENDFOR
-IF ta2[0] NE '' THEN BEGIN
- FOR j=0,N_ELEMENTS(ta2)-1 DO BEGIN
- ;extract valid portion of the entries (depending on ATTRIBUTE name)
- tchk=STRSPLIT(ta2[j],';',/Extract)
- ta2[j]=STRCOMPRESS(tchk[ti],/Remove_All)
- ENDFOR
-ENDIF
-IF cpos EQ '_' THEN BEGIN
- gi=WHERE(ta1 EQ achk[0],gcnt)
- IF gcnt NE 0 THEN gi=WHERE(ta2 EQ achk[1],gcnt)
-ENDIF ELSE BEGIN ;Check for Case Sensitive or incorrect apostrophe problem
- gi=WHERE(ta1 EQ achk,gcnt)
- IF gcnt EQ 0 THEN BEGIN
- ;Check for Case Sensitivity and/or incorrect apostrophe usage or correct affiliation acronym only
- ;Will be fixed in CHECK_METADATA
- gi=WHERE(STRLOWCASE(ta1) EQ STRLOWCASE(achk),gcnt)
- IF gcnt NE 0 THEN dentry=ta1[gi[0]] ELSE BEGIN
- ;Check for "`" instead of "'" in metadata entry
- IF STRPOS(achk,'`') NE -1 THEN BEGIN
- achkh=achk
- WHILE STRPOS(achkh,'`') NE -1 DO STRPUT,achkh,'''',STRPOS(achkh,'`')
- gi=WHERE(STRLOWCASE(ta1) EQ STRLOWCASE(achkh),gcnt)
- IF gcnt NE 0 THEN dentry=ta1[gi[0]]
- ENDIF
- ;If AFFILIATION check to see if the acronym only is valid
- IF affchk EQ 1 THEN BEGIN
- gi=WHERE(STRLOWCASE(acroval) EQ STRLOWCASE(t_acrovals),gcnt)
- IF gcnt NE 0 THEN dentry=ta1[gi[0]]
- ENDIF
- ENDELSE
- ENDIF
-ENDELSE
-
-IF gcnt EQ 0 THEN BEGIN
- ei=-1
- FOR i=0,N_ELEMENTS(infoerr)-1 DO IF STRPOS(taname,infoerr[i]) NE -1 THEN ei=i
-
- IF ei NE -1 THEN BEGIN ;Log message instead of STOP_WITH_ERROR
- IF ei LE 1 THEN metalabel=STRMID(taname,STRPOS(taname,'/')+1) ELSE metalabel=infoerr[ei]
- infotxt='3 '+metalabel+'='+aname+' doesn''t match any '
- infotxt=infotxt+'Table Attribute Values under '+infoerr[ei]+' field'
- INFOTXT_OUTPUT,infotxt
- ENDIF ELSE BEGIN
- STOP_WITH_ERROR,o3[3]+proname+aname+': ',errtxt2[1]+taname,lu & RETURN
- ENDELSE
-ENDIF
-ti=gi ;ti returned to the Check_MetaData procedure to check AFFILIATION, ADDRESS and EMAIL values
-
-END ;procedure Extract_and_Test
-
-
-
-PRO check_metadata
-;Procedure to check the validity of Metadata values by checking them against the corresponding
-;entries in the Table Attribute Values File
-; ----------
-;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
-;
-; History:
-; 20050802: Original IDLCR8HDF Routine - Version 1.0
-; 20050909: 'rd' flag included to convert VAR_SI_CONVERSION values to floating point if the
-; corresponding VAR_DATA_TYPE is real or double; Section included to check for
-; repeated SI units in the calculated VAR_SI_CONVERSION - Version 1.1
-; 20061012: Section added to do simple checks on Metadata ORIGINATOR values if the information
-; is not otherwise available in the TAV file; Warning messages written to output
-; windows and/or log file depending on the options chosen by the user; Section relating
-; to checks on VAR_UNITS and VAR_SI_CONVERSION moved to the VAR_UNITS_TEST routine;
-; Common variable definition WIDGET_WIN added - Version 2.0
-; 20080302: Sections included to handle checks when an Envisat table.dat file is used, including;
-; ensuring metadata labels DATA_TYPE and VAR_DATA_TYPE are not both found when using the
-; Strpos procedure; Handling different table.dat format of ORIGINATOR attributes; using
-; DATA_SOURCE_02 instead of AFFILIATION for the DATA_SOURCE checks. Section which puts
-; TAV file VAR_UNITS and UNIT_PREFIX values into arrays for the VAR_UNITS_TEST call is
-; no longer needed and is removed; Change made so that Free Attribute Metadata values
-; which include semi-colons do not have white space removed (i.e. the semi-colons are
-; not defined as separating sub-values) - Version 3.0
-; 20091203: Account for the situation when a VAR_NAME does not include a Variable Mode, but still
-; includes a Variable Descriptor ('__' will be present in the name) - Version 3.08
-; 20100205: Add RETURN command after all STOP_WITH_ERROR calls, which allows program to return to the
-; calling program if the reterr argument is included in the idlcr8hdf call - Version 3.09
-; 20101120: Do check for MJD2000 VAR_UNIT, and change to MJD2K; Check for LONG datatype (now refers to
-; LONG64) and change to INTEGER (now means LONG32) for TAV checks - Version 4.0b0
-; 20121015: Change VAR_SI_CONVERSION information/error message so will also report if the
-; VAR_SI_CONVERSION label does not have an accompanying value - Version 4.0b13
-; 20141110: Add check for DATETIME where VAR_SI_CONVERSION=0.0;86400.0;s, but VAR_UNITS is not MJD2K.
-; Will change VAR_UNITS to MJD2K - Version 4.0b24
-; 20150127: Stop removing leading and trailing spaces from variable attribute values (when there is
-; only 1 value) to allow for possibility of VAR_FILL_VALUEs being an empty string
-; - Version 4.0b25
-; 20150811: Remove STOP_WITH_ERROR message if FILE_META_VERSION doesn't have 2 values. This check is
-; now done in the READ_METADATA routine - Version 4.0b29
-; 20151012: Remove checks on white space in Metadata values. This check is now done in the
-; READ_METADATA routine - Version 4.0b31
-; 20161130: Allow the program to continue when an error is generated after doing VAR_UNITS and
-; VAR_SI_CONVERSION checks (previously called STOP_WITH_ERROR) - Version 4.0b40
-; 20161230: Modify checks on ORIGINATOR values as this section removed from the TAV file from
-; 04R025. Add tests for COUNTRY in XX_ADDRESS, and check for '@' in XX_EMAIL -
-; Version 4.0b41
-; 20170331: Allow for COUNTRY value in XX_ADDRESS to be all uppercase or have initial capitalization
-; (allow for special cases e.g. Cote d'Ivoire; words such as the, and, of also not
-; capitalized) - Version 4.0b42
-; 20191205: Add check for optional VERSION_NAME sub-value of DATA_SOURCE. If present then call
-; GEOMS_RULE_CHANGES to do checks on the value - Version 4.0b53
-; 20240912: Allow for VAR_SI_CONVERSION to have two possible values depending on the ordering of units
-; that have the same power value e.g. m-1 sr-1 or sr-1 m-1 - Version 4.0b64
-;
-; Inputs: meta_arr - a string array containing the Global and Variable Attributes
-; tab_arr - a string array containing the TAV or table.dat file attributes
-
-; Output: meta_arr - some meta_arr values may be corrected or calculated in this routine. White space
-; between metadata sub-values is also removed.
-;
-; Called by: IDLCR8HDF
-;
-; Subroutines Called: EXTRACT_AND_TEST
-; VAR_UNITS_TEST
-; GEOMS_RULE_CHANGES
-; STOP_WITH_ERROR (if error state detected); INFOTXT_OUTPUT
-; Possible Conditions for STOP_WITH_ERROR call (plus [line number] where called):
-; 1. An attribute value is expected but is absent or invalid
-; 2. Incorrect number of (sub-)values for a particular Metadata attribute
-; 3. NAME and AFFILIATION values don't match (only applies if reading in the full AVDC
-; TAV file)
-; 4. AFFILIATION (AVDC) or DATA_SOURCE_02 (Envisat) field not found in the TAV file
-; 5. There is no match between a given VAR_UNIT, and the list of valid units and prefixes
-; present in the TAV file
-; 6. VAR_UNITS value is not valid
-;
-; Information Conditions (when the program is able to make changes):
-; 1. ORIGINATOR values for ADDRESS or EMAIL replaced with those from the TAV file
-; 2. VAR_SI_CONVERSION values replaced with those calculated by the program
-
-COMMON TABLEDATA
-COMMON METADATA
-COMMON DATA
-COMMON WIDGET_WIN
-
-;Possible error messages for this procedure
-proname='Check_Metadata procedure: ' & lu=-1L
-errtxt=STRARR(4)
-errtxt[0]='No or Invalid value for this Attribute: '
-errtxt[1]='Number of (sub-)values for this Attribute should be ' ;changed to information message
-errtxt[2]='AFFILIATION value doesn''t match NAME value: '
-errtxt[3]=' field not found in the input Table Attribute Values file'
-
-;originator values to be checked
-orig_attr=['PI_','DO_','DS_','NAME','AFFILIATION','ADDRESS','EMAIL']
-countrysp1=['Congo, The Democratic Republic of the','Cote d''Ivoire'] ;special cases for initial capitalization of the COUNTRY value
-countrysp2=['the','and','of'] ;special cases for initial capitalization of the COUNTRY value (individual words)
-
-;initialize variables for VAR_UNITS/VAR_SI_CONVERSION checks
-firstvu=0 & tchk=[''] & rd=1 & writeonce=0
-xval=[2,2,3,1] ;expected number of sub-values for the four originator attributes
-new_name=[''] ;Entry which will contain any ORIGINATOR name values not present in the TAV file
-
-FOR i=0,N_ELEMENTS(meta_arr)-1 DO BEGIN
- res=STRSPLIT(meta_arr[i],'=',/Extract,COUNT=nres)
- IF res[0] EQ 'VAR_NAME' THEN vnval=meta_arr[i]
- FOR j=0,2 DO BEGIN
- FOR k=3,6 DO BEGIN
- ;need to change ORIGINATOR NAME ATTRIBUTE names to match TAV File
- IF res[0] EQ orig_attr[j]+orig_attr[k] THEN BEGIN
- res[0]='ORIGINATOR' & jh=j & kh=k
- ENDIF
- ENDFOR
- ENDFOR
- ;Test whether metadata label is also a TAV field
- ;First test checks for exact match between metadata label and TAV field
- ;(This avoids possible problem using STRPOS (e.g. DATA_TYPE won't also pick up VAR_DATA_TYPE))
- mi=WHERE(tab_arr[*,0] EQ res[0],mcnt)
- ;Back-up test in the event that label has more than one TAV field (e.g. DATA_VARIABLES)
- IF mcnt EQ 0 THEN mi=WHERE(STRPOS(tab_arr[*,0],res[0]) EQ 0,mcnt) ;was NE -1
-
- IF res[0] EQ 'ORIGINATOR' THEN BEGIN
- ;Do checks on _NAME, _AFFILIATION, _ADDRESS, and _EMAIL sub-values
- resx=STRSPLIT(meta_arr[i],'=;',/Extract)
- nel=N_ELEMENTS(resx)-1
-
- IF (kh EQ 4) AND (nel EQ xval[1]) THEN BEGIN ;check on AFFILIATION value
- resy=STRSPLIT(meta_arr[i],'=',/Extract)
- mci=WHERE(tab_arr[*,0] EQ 'AFFILIATION',mccnt)
- IF mccnt NE 0 THEN BEGIN
- ta1=tab_arr[mci[0],2:tab_arr[mci[0],1]+1] ;subset of relevant FIELD ENTRIES
- taname=tab_arr[mci[0],0]+'/'+resy[0] ;name combination for error message
- dentry=resy[1] & ti=-1
- EXTRACT_AND_TEST,dentry,';',xval[1],ta1,[''],taname,resy[1],ti
- IF STRLEN(rerr[0]) GT 2 THEN RETURN
- IF dentry NE resy[1] THEN BEGIN
- ;Case and/or apostrophe differences between Metadata entry and TAV entry
- achk2=STRSPLIT(tab_arr[mci[0],ti[0]+2],';',/Extract)
- achk=STRTRIM(achk2[0],2)
- FOR m=1,xval[1]-1 DO achk=achk+';'+STRTRIM(achk2[m],2)
- IF qa_yes THEN itxt=' should be ' ELSE itxt=' replaced with '
- infotxt='2 '+meta_arr[i]+itxt+resy[0]+'='+achk+' based on the TAV entry'
- INFOTXT_OUTPUT,infotxt
- meta_arr[i]=resy[0]+'='+achk
- ENDIF
- ENDIF
- ENDIF
- IF (kh EQ 5) AND (nel EQ xval[2]) THEN BEGIN ;check on COUNTRY value
- mci=WHERE(tab_arr[*,0] EQ 'COUNTRY',mccnt)
- IF mccnt NE 0 THEN BEGIN
- ;check the COUNTRY value against the TAV values (note: can be all uppercase or initial capitalization)
- aname=res[1] & ti=-1
- ta1=tab_arr[mci[0],2:tab_arr[mci[0],1]+1] ;subset of relevant FIELD ENTRIES
- taname=tab_arr[mci[0],0]+'/'+orig_attr[jh]+'ADDRESS' ;name combination for error message
- EXTRACT_AND_TEST,STRUPCASE(resx[nel]),';',1,ta1,[''],taname,aname,ti
- IF STRLEN(rerr[0]) GT 2 THEN RETURN
- ;Test for upper case or initial capitalization
- IF resx[nel] NE STRUPCASE(resx[nel]) THEN BEGIN
- counterr=0 ;error code to detect problem with initial capitalization
- sci=WHERE(STRUPCASE(resx[nel]) EQ STRUPCASE(countrysp1),spcnt)
- IF spcnt NE 0 THEN BEGIN ;Test special case countries e.g. Cote d'Ivoire
- IF resx[nel] NE countrysp1[sci[0]] THEN counterr=1
- ENDIF ELSE BEGIN ;Test individual words in the country value
- sepcount=STRSPLIT(resx[nel],' ,',COUNT=spcnt,/EXTRACT)
- FOR j=0,spcnt-1 DO BEGIN
- slen=STRLEN(sepcount[j])
- sci=WHERE(STRUPCASE(sepcount[j]) EQ STRUPCASE(countrysp2),sp2cnt)
- IF sp2cnt NE 0 THEN test=countrysp2[sci[0]] $ ;special case lowercase word e.g. the, and, of
- ELSE IF slen EQ 1 THEN test=STRUPCASE(sepcount[j]) $ ;only one letter in the word so make uppercase
- ELSE test=STRUPCASE(STRMID(sepcount[j],0,1))+STRLOWCASE(STRMID(sepcount[j],1)) ;capitalize the first letter
- IF sepcount[j] NE test THEN counterr=1
- ENDFOR
- ENDELSE
- IF counterr EQ 1 THEN BEGIN
- uccountry=STRUPCASE(resx[nel])
- IF qa_yes THEN itxt=' should be ' ELSE itxt=' replaced with '
- infotxt='2 COUNTRY value for '+meta_arr[i]+itxt+uccountry
- INFOTXT_OUTPUT,infotxt
- meta_arr[i]=STRMID(meta_arr[i],0,STRPOS(meta_arr[i],';',/REVERSE_SEARCH)+1)+uccountry
- ENDIF
- ENDIF
- ENDIF
- ENDIF
- IF (kh EQ 6) AND (nel EQ xval[3]) THEN BEGIN ;check @ present in the e-mail value
- amppos=STRPOS(res[1],'@')
- IF (amppos LE 0) OR (amppos EQ STRLEN(res[1])-1) THEN BEGIN
- infotxt='3 Invalid format for '+orig_attr[jh]+'EMAIL value'
- INFOTXT_OUTPUT,infotxt
- ENDIF
- ENDIF
- IF (nel NE xval[kh-3]) OR (STRMID(meta_arr[i],STRLEN(meta_arr[i])-1,1) EQ ';') THEN BEGIN
- IF xval[kh-3] GT 1 THEN itxt='sub-' ELSE itxt=''
- infotxt='3 Number of '+itxt+'values for attribute '+meta_arr[i]+' should be '+STRTRIM(xval[kh-3],2)
- INFOTXT_OUTPUT,infotxt
- ENDIF
- mcnt=0
- ENDIF
-
- IF mcnt NE 0 THEN BEGIN ;check Metadata value(s) against TAV file entries
- IF (nres NE 2) AND (rd NE 0) THEN BEGIN
- STOP_WITH_ERROR,o3[3]+proname,errtxt[0]+meta_arr[i],lu & RETURN
- ENDIF ELSE IF res[0] EQ 'DATA_SOURCE' THEN BEGIN
- ;need to test against 'DATA_SOURCE' and 'AFFILIATION' (AVDC) or 'DATA_SOURCE_02'
- ;(original Envisat) entries
- ta1=tab_arr[mi[0],2:tab_arr[mi[0],1]+1] ;DATA_SOURCE ENTRIES
- IF tab_type EQ 0 THEN BEGIN
- atxt='AFFILIATION' & atx='' & ti=1
- ENDIF ELSE BEGIN
- atxt='DATA_SOURCE_02' & atx='_01' & ti=0
- ENDELSE
- ai=WHERE(STRPOS(tab_arr[*,0],atxt) NE -1,acnt)
- IF acnt EQ 0 THEN BEGIN
- STOP_WITH_ERROR,o3[3]+proname,atxt+errtxt[3],lu & RETURN
- ENDIF
- ta2=tab_arr[ai[0],2:tab_arr[ai[0],1]+1] ;AFFILIATION/DATA_SOURCE_02 ENTRIES
- taname=res[0]+atx+'/'+tab_arr[ai[0],0] ;name combination for error message
- n_sv=[2,3] ;possible number of DATA_SOURCE sub-values
- EXTRACT_AND_TEST,res[1],'_',n_sv,ta1,ta2,taname,res[1],ti
- IF STRLEN(rerr[0]) GT 2 THEN RETURN
-
- achk=STRSPLIT(res[1],'_',/EXTRACT,COUNT=n_achk)
- IF n_achk EQ 3 THEN BEGIN
- ;Check DATA_VERSION_NAME if there are 3 sub-values in DATA_SOURCE (introduced v4.0b53)
- GEOMS_RULE_CHANGES,11,achk[2],meta_arr
- ENDIF
-
- ENDIF ELSE IF res[0] EQ 'DATA_VARIABLES' THEN BEGIN
- ;make into 2-D array and separate out the components of the VAR_NAMES
- achk=STRSPLIT(res[1],';',/Extract,COUNT=nel)
- achk2=STRARR(mcnt,nel)
- FOR j=0,nel-1 DO BEGIN
- dv=STRSPLIT(achk[j],'_',/Extract,Count=n_dv)
- IF n_dv GT mcnt THEN BEGIN ;invalid VAR_NAME composition
- STOP_WITH_ERROR,o3[3]+proname,errtxt[0]+res[0]+'='+achk[j],lu & RETURN
- ENDIF
- achk2[0:N_ELEMENTS(dv)-1,j]=dv
- ENDFOR
- FOR j=0,nel-1 DO BEGIN
- dv=WHERE(achk2[*,j] NE '',dvcnt)
- FOR k=0,mcnt-1 DO BEGIN
- IF achk2[k,j] NE '' THEN BEGIN
- taname=tab_arr[mi[k],0]
- ta1=REFORM(tab_arr[mi[k],2:tab_arr[mi[k],1]+1]) ;subset of TAV DATA_VARIABLES entries
- IF (k NE 0) AND (dvcnt EQ 2) THEN BEGIN
- ;need to check both variable mode and descriptor options
- FOR m=k+1,mcnt-1 DO BEGIN
- taname=taname+'/'+tab_arr[mi[m],0]
- ta1=[ta1,REFORM(tab_arr[mi[m],2:tab_arr[mi[m],1]+1])]
- ENDFOR
- ENDIF
- EXTRACT_AND_TEST,achk2[k,j],';',1,ta1,[''],taname,achk[j],0
- IF STRLEN(rerr[0]) GT 2 THEN RETURN
- ENDIF
- ENDFOR
- ENDFOR
- ENDIF ELSE IF res[0] EQ 'FILE_ACCESS' THEN BEGIN
- ;need to check each Metadata entry individually
- achk=STRSPLIT(res[1],';',/Extract)
- nel=N_ELEMENTS(achk)
- ta1=tab_arr[mi[0],2:tab_arr[mi[0],1]+1] ;FILE_ACCESS entries
- FOR j=0,nel-1 DO BEGIN
- EXTRACT_AND_TEST,achk[j],';',1,ta1,[''],res[0],achk[j],0
- IF STRLEN(rerr[0]) GT 2 THEN RETURN
- ENDFOR
- ENDIF ELSE IF res[0] EQ 'FILE_META_VERSION' THEN BEGIN
- ;only need to look at the second value
- achk=STRSPLIT(res[1],';',/Extract)
- IF N_ELEMENTS(achk) NE 2 THEN achk=[achk,'x','x'] ;ensure there are at least 2 values
- achk=[STRTRIM(achk[1],2)]
- ta1=tab_arr[mi[0],2:tab_arr[mi[0],1]+1] ;FILE_META_VERSION entries
- EXTRACT_AND_TEST,achk[0],';',1,ta1,[''],res[0],achk[0],0
- IF STRLEN(rerr[0]) GT 2 THEN RETURN
- ENDIF ELSE IF res[0] EQ 'VAR_UNITS' THEN BEGIN
- ;VAR_UNITS and VAR_SI_CONVERSION entries (not needed for STRING data type)
- IF rd NE 0 THEN BEGIN
- IF STRUPCASE(res[1]) EQ 'MJD2000' THEN BEGIN ;change to MJD2K
- in1=meta_arr[i] & in2=res[1]
- GEOMS_RULE_CHANGES,3,in1,in2,writeonce
- meta_arr[i]=in1 & res[1]=in2
- writeonce=1
- ENDIF
- errstate='' & datetimechk=STRPOS(STRUPCASE(vnval),'DATETIME') NE -1
- VAR_UNITS_TEST,res[1],rd,tab_type,tchk,errstate
- IF errstate NE '' THEN BEGIN ;VAR_UNIT value not valid
- ;STOP_WITH_ERROR,o3[3]+proname+STRMID(errstate,0,STRPOS(errstate,':')+2), $
- ; STRMID(errstate,STRPOS(errstate,':')+2),lu
- ;RETURN
- IF ~datetimechk THEN itxt='. VAR_SI_CONVERSION value not checked' ELSE itxt=''
- infotxt='3 '+errstate+itxt
- INFOTXT_OUTPUT,infotxt
- ENDIF
- ;Find VAR_SI_CONVERSION attribute
- k=1
- WHILE STRMID(meta_arr[i+k],0,17) NE 'VAR_SI_CONVERSION' DO k++
- achk=STRSPLIT(meta_arr[i+k],'=',/Extract) ;VAR_SI_CONVERSION values
- IF N_ELEMENTS(achk) EQ 1 THEN achk=[achk,' ']
- IF datetimechk THEN BEGIN
- ;Check if VAR_SI_CONVERSION values are 0.0;86400.0;s regardless of VAR_UNITS. If so, then don't change
- achk1=STRSPLIT(achk[1],' ;',/EXTRACT)
- achk1=[achk1,'','',''] ;Ensure there are at least 3 values extracted
- IF (is_a_number_hdf(achk1[0])) AND (is_a_number_hdf(achk1[1])) THEN BEGIN
- IF (FIX(achk1[0]) EQ 0) AND (LONG(achk1[1]) EQ 86400L) AND $
- (STRLOWCASE(achk1[2]) EQ 's') THEN tchk[*]='0.0;86400.0;s'
- ENDIF ELSE IF errstate NE '' THEN tchk[*]='0.0;86400.0;s'
- ENDIF
- ;check calculated VAR_SI_CONVERSION value(s) against existing value in the metadata
- IF (errstate EQ '') OR (datetimechk) THEN BEGIN
- vscgi=WHERE(achk[1] EQ tchk,vscgcnt)
- IF vscgcnt EQ 0 THEN BEGIN
- IF qa_yes THEN itxt=' should be ' ELSE itxt=' replaced with '
- infotxt='2 '+meta_arr[i+k]+itxt+'VAR_SI_CONVERSION='+tchk[0]+' for '+vnval
- INFOTXT_OUTPUT, infotxt
- meta_arr[i+k]=achk[0]+'='+tchk[0]
- ENDIF ELSE meta_arr[i+k]=achk[0]+'='+tchk[vscgi[0]]
- ENDIF
- ENDIF
- ENDIF ELSE BEGIN ;perform checks on the remaining ATTRIBUTE_NAMES
- achk=STRSPLIT(res[1],';',/Extract)
- achk=STRTRIM(achk,2)
- nel=N_ELEMENTS(achk) & ta2=['']
- IF res[0] EQ 'VAR_DATA_TYPE' THEN BEGIN
- ;to convert VAR_SI_CONVERSION values to floating point if necessary
- CASE 1 OF
- (STRUPCASE(res[1]) EQ 'REAL') OR (STRUPCASE(res[1]) EQ 'DOUBLE'): rd=-1
- STRUPCASE(res[1]) EQ 'STRING': rd=0
- ELSE: rd=1
- ENDCASE
- IF STRUPCASE(achk[0]) EQ 'LONG' THEN achk[0]='INTEGER'
- ;Changed to INTEGER for EXTRACT_AND_TEST only - actual change occurs in SET_UP_STRUCTURE
- ENDIF
- IF (nel NE mcnt) OR (STRMID(meta_arr[i],STRLEN(meta_arr[i])-1,1) EQ ';') THEN BEGIN
- IF mcnt GT 1 THEN itxt='sub-' ELSE itxt=''
- infotxt='3 Number of '+itxt+'values for attribute '+meta_arr[i]+' should be '+STRTRIM(mcnt,2)
- INFOTXT_OUTPUT,infotxt
- ;STOP_WITH_ERROR,o3[3]+proname+meta_arr[i]+': ',errtxt[1]+STRTRIM(mcnt,2)+'.',lu
- ;RETURN
- ENDIF
- nel=1
- FOR j=0,mcnt-1 DO BEGIN
- ta1=tab_arr[mi[j],2:tab_arr[mi[j],1]+1] ;subset of relevant ENTRIES
- dentry=achk[j] & ti=mi[j]
- EXTRACT_AND_TEST,dentry,';',nel,ta1,ta2,tab_arr[mi[j],0],achk[j],ti
- IF STRLEN(rerr[0]) GT 2 THEN RETURN
- IF (mcnt EQ 1) AND (dentry NE achk[j]) THEN BEGIN
- ;Case and/or apostrophe differences between Metadata entry and TAV entry
- achk2=STRSPLIT(tab_arr[mi[j],ti[0]+2],';',/Extract)
- achk=STRTRIM(achk2[0],2)
- IF qa_yes THEN itxt=' should be ' ELSE itxt=' replaced with '
- infotxt='2 '+meta_arr[i]+itxt+res[0]+'='+achk+' based on the TAV entry'
- INFOTXT_OUTPUT,infotxt
- meta_arr[i]=res[0]+'='+achk
- ENDIF
- IF j EQ 0 THEN ta2[0]=''
- ENDFOR
- ENDELSE
- ENDIF
-ENDFOR
-
-END ;Procedure Check_Metadata
-
-
-
-PRO extract_data, sds, inf, vc, dtest, ndl, infowrite
-;Procedure to extract the set of data corresponding to a particular Variable Attribute
-; ----------
-;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
-;
-; History:
-; 20050802: Introduced to IDLCR8HDF, originally part of the READ_DATA procedure - Version 1.0
-; 20050909: After reading in a data segment according to the VAR_SIZE values, check the next line to
-; ensure it is not a further data point (i.e. there are more data points than specified by
-; VAR_SIZE); If sds is a structure, and there is only one data point then return dtest as
-; an array rather than a scalar - Version 1.1
-; 20061012: The portion of READ_DATA dealing with extracting a specific data segment renamed
-; EXTRACT_DATA - Version 2.0
-; 20080302: The sets of data values in a file can now be in any order, rather than the same order as
-; that given by the DATA_VARIABLES attribute; dtest used as an input flag to indicate
-; whether the procedure call is from SET_UP_STRUCTURE [-1] or READ_DATA [0]. If from
-; SET_UP_STRUCTURE then only the number of data lines (ndl) is determined by the routine;
-; an error is generated in the case where the same VAR_NAME is repeated in the data file;
-; Error messages clarified when an incorrect number of data values are found in the
-; input data file - Version 3.0
-; 20100205: Add RETURN command after all STOP_WITH_ERROR calls, which allows program to return to the
-; calling program if the reterr argument is included in the idlcr8hdf call - Version 3.09
-; 20130116: Account for empty string data values in a dataset - previously '' values ignored in an
-; input data file - Version 4.0b15
-; 20150227: Do not automatically run STRTRIM on the dataset when it is read in to allow checks on
-; string datasets - Version 4.0b26
-; 20150811: Add infowrite parameter so that if DATETIME type dataset is in ISO8601 format then the
-; information/error message will only be written once instead of for each value
-; - Version 4.0b29
-; 20171130: Remove ds from STOP_WITH_ERROR parameters - Version 4.0b44
-;
-; Inputs: sds - Either a structure containing the Variable Attributes and Data, or the input
-; data file
-; inf - flag identifying sds type where, 0=structure; 1=data file
-; vc - the index value of the vn array holding the VAR_NAME being searched for in the data
-; file, or the index value of the sds structure which holds the data
-; dtest - a single string array to show which procedure called this routine, either
-; SET_UP_STRUCTURE [-1], or READ_DATA [0]
-; infowrite - optional input to stop ISO8601 to MJD2K information message being written multiple
-; times (when called by READ_DATA only)
-;
-; Outputs: dtest - a string array of size ndl that holds the data values
-; ndl - The total number of data lines i.e. the product of the VAR_SIZE values
-;
-; Called by: SET_UP_STRUCTURE; READ_DATA
-;
-; Subroutines Called: STOP_WITH_ERROR (if error state detected)
-; Possible Conditions for STOP_WITH_ERROR call (plus [line number] where called):
-; 1. EOF encountered when trying to find the VAR_NAME or while attempting to read in data
-; [2146,2220,2221]
-; 2. Invalid data point found in the dataset [2193]
-; 3. Two instances of the same VAR_NAME found in the input data file [2204]
-; 4. Too many data values present in the dataset or the next line in the file is invalid [2211]
-; 5. Number of data values does not match product of VAR_SIZE [2237]
-; 6. ISO8601 datetime format not valid [2253]
-
-COMMON DATA
-COMMON WIDGET_WIN
-
-;possible error messages for this procedure
-errtxt=STRARR(10)
-errtxt[0]='EOF encountered but not expected'
-errtxt[1]=' when trying to find VAR_NAME in '
-errtxt[2]=' when trying to determine VAR_SIZE for VAR_NAME='
-errtxt[3]=' - Number of data lines should be '
-errtxt[4]='Invalid data point found in the dataset at line '
-errtxt[5]='Two instances of this VAR_NAME present in the Data File'
-errtxt[6]='Too many data values present in the dataset or invalid data line entry following the dataset'
-errtxt[7]='Number of data values does not match product of VAR_SIZE: '
-errtxt[8]='ISO8601 format not valid: '
-errtxt[9]='Unable to determine correct number of data values due to VAR_SIZE calculation error'
-proname='Extract_Data procedure: '
-
-ndl=1L ;initialize data line count
-vsi=WHERE(vserror EQ vn[vc],vcnt) ;check whether there is a VAR_SIZE calculation error for this dataset
-sdata=ndm(vc,8) EQ 6 ;Check for String data type
-
-IF inf EQ 1 THEN BEGIN ;Find the first data value for the requested VAR_NAME
- ON_IOERROR,TypeConversionError
- dum='' & vntest=vn[vc]
- ;Add check in case VAR_NAME has been changed from original input
- IF vnchange[vc] NE '' THEN vntest=vnchange[vc]
- OPENR,lu,sds,/GET_LUN
- IF NOT EOF(lu) THEN BEGIN ;read through data file until the correct Variable Name is found
- REPEAT READF,lu,dum UNTIL (EOF(lu)) OR (STRTRIM(STRUPCASE(dum),2) EQ STRUPCASE(vntest)) OR $
- (STRTRIM(STRUPCASE(dum),2) EQ STRUPCASE(vn[vc]))
- ENDIF
- IF STRTRIM(STRUPCASE(dum),2) EQ STRUPCASE(vn[vc]) THEN vntest=vn[vc]
- IF EOF(lu) THEN BEGIN
- STOP_WITH_ERROR,o3[3]+proname+vn[vc],errtxt[0]+errtxt[1]+sds,lu & RETURN
- ENDIF
-
- ;Find first data point
- dum='!'
- valid=-1 ;set to test for successful line read
- WHILE (STRMID(dum,0,1) EQ '!') OR (STRMID(dum,0,1) EQ '#') OR ((~sdata) AND (strtrim(dum,2) EQ '')) DO BEGIN
- READF,lu,dum ;& dum=STRTRIM(dum,2)
- ENDWHILE
- valid=1 ;i.e. got to here without prematurely reaching the eof so first data point found OK
-
- IF dtest[0] EQ '-1' THEN BEGIN ;need to determine the total number of data lines only (ndl)
- IF NOT EOF(lu) THEN BEGIN
- ;read through until eof or comment line or another VAR_NAME is reached
- REPEAT BEGIN
- READF,lu,dum
- dum=STRTRIM(STRUPCASE(dum),2)
- bi=WHERE(dum EQ STRUPCASE(vn))
- IF (STRMID(dum,0,1) NE '!') AND (STRMID(dum,0,1) NE '#') AND $
- ((dum NE '') OR ((sdata) AND (dum EQ ''))) AND (bi[0] EQ -1) THEN incnt=1L $
- ELSE incnt=0L
- ndl=ndl+incnt ;presume it is a valid data line so add to total
- ENDREP UNTIL (incnt EQ 0L) OR (EOF(lu))
- ENDIF
- ENDIF ELSE BEGIN ;return dtest and ndl
- di=WHERE(ndm[vc,0:7] NE 0L,ndim)
- ;multiply together the array dimensions to determine expected total number of datalines
- FOR i=0,ndim-1 DO ndl=ndl*ndm[vc,di[i]]
- ;attempt to read any remaining data values
- IF ndl GT 1L THEN BEGIN
- valid=0 ;set to test for successful block read
- dtest=STRARR(ndl-1) & READF,lu,dtest
- valid=1 ;i.e. got to here without prematurely reaching the EOF so block read was OK
- dtest=[dum,dtest] ;add first value to dtest
- ENDIF ELSE dtest=[dum]
- dtestup=STRUPCASE(STRTRIM(dtest,2)) ;& dtest=STRTRIM(dtest,2)
- ;Test to see if any of the 'data' read in is a VAR_NAME - if so bcnt will NE 0 (and will generate error message)
- bcnt=0 & i=0
- WHILE (bcnt EQ 0) AND (i LE nvn-1) DO BEGIN
- bi=WHERE(dtestup EQ STRUPCASE(vn[i]),bcnt)
- IF bcnt EQ 0 THEN i=i+1
- ENDWHILE
- ;Test to see if any of the 'data' read in is not a valid data point - if so bcnt will NE 0
- IF bcnt EQ 0 THEN $
- bi=WHERE((STRMID(dtestup,0,1) EQ '!') OR (STRMID(dtestup,0,1) EQ '#') OR $
- ((dtestup EQ '') AND (~sdata)) OR (dtestup EQ STRUPCASE(vntest)),bcnt)
-
- IF bcnt NE 0 THEN BEGIN ;Non-valid data point found in the dataset
- STOP_WITH_ERROR,o3[3]+proname+vn[vc]+': ',errtxt[4]+STRTRIM(bi[0]+1,2)+errtxt[3]+ $
- STRTRIM(ndl,2),lu
- RETURN
- ENDIF
-
- ;Read next line if not EOF to test for the correct number of data lines
- IF NOT EOF(lu) THEN READF,lu,dum ELSE dum='!'
- dum=STRTRIM(STRUPCASE(dum),2)
- ;Test to see if the next line read in is another VAR_NAME - if not then bi[0] will equal -1
- bi=WHERE(dum EQ STRUPCASE(vn))
- IF bi[0] EQ vc THEN BEGIN ;Same VAR_NAME found twice in the data file
- STOP_WITH_ERROR,o3[3]+proname+vn[vc]+': ',errtxt[5],lu & RETURN
- ENDIF
- IF (bi[0] EQ -1) AND (dum NE '!') THEN bi=WHERE(dum EQ STRUPCASE(vnchange))
- ;Test to check if the next line read in is another data point - if so bcnt will be set to 1
- IF (STRMID(dum,0,1) NE '!') AND (STRMID(dum,0,1) NE '#') AND $
- ((dum NE '') OR ((sdata) AND (dum EQ ''))) AND (bi[0] EQ -1) THEN $
- bcnt=1
- IF bcnt NE 0 THEN BEGIN ;not enough data lines read in or next line in the file is invalid
- IF vcnt NE 0 THEN STOP_WITH_ERROR,o3[3]+proname+'VAR_NAME='+vn[vc]+': ',errtxt[9],lu $
- ELSE $
- STOP_WITH_ERROR,o3[3]+proname+'VAR_NAME='+vn[vc]+': ',errtxt[6]+errtxt[3]+STRTRIM(ndl,2),lu
- RETURN
- ENDIF
- ENDELSE
- FREE_LUN,lu
-
- TypeConversionError:
- IF valid LE 0 THEN BEGIN ;EOF encountered but not expected
- IF (dtest[0] EQ '-1') AND (valid EQ -1) THEN $
- STOP_WITH_ERROR,o3[3]+proname,errtxt[0]+errtxt[2]+vn[vc],lu $
- ELSE BEGIN
- ;Check to see if it is due to a VAR_SIZE calculation error or not
- IF vcnt NE 0 THEN STOP_WITH_ERROR,o3[3]+proname+'VAR_NAME='+vn[vc]+': ',errtxt[9],lu $
- ELSE $
- STOP_WITH_ERROR,o3[3]+proname+'VAR_NAME='+vn[vc]+': ',errtxt[0]+errtxt[3]+STRTRIM(ndl,2),lu
- ENDELSE
- RETURN
- ENDIF
-ENDIF ELSE BEGIN ;sds is a structure
- IF dtest[0] EQ '-1' THEN BEGIN ;determine the total number of data lines only
- dtest=*sds[vc].data
- ndl=LONG(N_ELEMENTS(dtest))
- dtest=['-1']
- ENDIF ELSE BEGIN ;return dtest and ndl
- lu=-1
- di=WHERE(ndm[vc,0:7] NE 0L,ndim)
- ;multiply together the array dimensions to determine expected total number of datalines
- FOR i=0,ndim-1 DO ndl=ndl*ndm[vc,di[i]]
- dtest=*sds[vc].data
- ;check structure contains the correct number of elements
- IF N_ELEMENTS(dtest) NE ndl THEN BEGIN
- STOP_WITH_ERROR,o3[3]+proname+vn[vc]+': ',errtxt[7]+STRTRIM(N_ELEMENTS(dtest),2)+'/'+STRTRIM(ndl,2),lu
- RETURN
- ENDIF
- stest=SIZE(dtest) ;test to see if dtest is an array or scalar
- IF stest[0] EQ 0 THEN dtest=[dtest] ELSE dtest=REFORM(dtest,ndl,/Overwrite) ;convert to a 1-D array if necessary
- ENDELSE
-ENDELSE
-
-;Note: this conditional will only be satisfied if the routine is called by READ_DATA
-;(vu still not assigned VAR_UNITS value if called by SET_UP_STRUCTURE)
-IF STRUPCASE(vu[vc]) EQ 'MJD2K' THEN BEGIN
- ;Convert time from ISO8601 to MJD2000 format if necessary
- FOR i=0L,ndl-1L DO BEGIN
- IF STRPOS(STRUPCASE(dtest[i]),'Z') NE -1 THEN BEGIN
- mjd2000=JULIAN_DATE(dtest[i],/I,/M) ;return time in MJD2000 format
- IF mjd2000 EQ -99999.d THEN BEGIN ;conversion error
- STOP_WITH_ERROR,o3[3]+proname+vn[vc]+': ',errtxt[8]+dtest[i],lu & RETURN
- ENDIF
- dtest[i]=STRING(format='(d18.9)',mjd2000)
- IF infowrite EQ 1 THEN BEGIN
- IF qa_yes THEN itxt='cannot be in ISO8601 format' ELSE itxt='changed from ISO8601 to MJD2K format'
- infotxt='2 Dataset values '+itxt+' for VAR_NAME='+vn[vc]
- INFOTXT_OUTPUT,infotxt
- infowrite=0
- ENDIF
- ENDIF
- ENDFOR
-ENDIF
-
-END ;Procedure Extract_Data
-
-
-
-PRO set_up_structure, sds, inf
-;Procedure to do additional checks on VAR_NAME, VAR_DEPEND, VAR_DATA_TYPE, VAR_UNITS, and
-;VAR_FILL_VALUE metadata attributes and set up a structure which will be used to hold the
-;data values.
-; ----------
-;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
-;
-; History:
-; 20050802: Introduced to IDLCR8HDF, as Determine_Arr_Sizes. Data originally stored in 4
-; arrays according to the data type of the variable attributes - Version 1.0
-; 20050909: Additional checks performed on the metadata attributes as follows: that the
-; VAR_DEPEND variable name(s), themselves, have a VAR_DEPEND value of CONSTANT or
-; INDEPENDENT, and matching VAR_SIZE values; that VAR_DATA_TYPE is DOUBLE for
-; attributes with VAR_UNITS=MJD2000; that VAR_UNITS=MJD2000 when VAR_NAME=DATETIME
-; - Version 1.1
-; 20061012: Procedure renamed Set_Up_Structure. A structure using pointers used to store the
-; data instead of arrays. VAR_FILL_VALUE value saved in common array and, if
-; VAR_UNITS=MJD2000, VAR_FILL_VALUE converted to its MJD2000 form if it is in
-; ISO8601 format. Common variable definition WIDGET_WIN added - Version 2.0
-; 20080302: Can calculate the VAR_SIZE value(s) from the input data or from the VAR_SIZE
-; value(s) of the VAR_DEPEND variable name(s), if they are not present in the
-; metadata. Section added to account for the new variables, ALTITUDE/ALTITUDE.GPH/
-; PRESSURE.BOUNDARIES; Additional checks added for VIS_PLOT_TYPE, VIS_SCALE_TYPE,
-; VIS_SCALE_MIN, and VIS_SCALE_MAX values - Version 3.0
-; 20100205: Add RETURN command after all STOP_WITH_ERROR calls, which allows program to return to the
-; calling program if the reterr argument is included in the idlcr8hdf call - Version 3.09
-; 20101120: Account for GEOMS changes between v3.0 and v4.0; Remove checks on VAR_DIMENSION, and the
-; VIS attributes; Add new VAR_DATA_TYPEs (Byte and Short); Check for self-referencing
-; VAR_DEPEND values (which replace INDEPENDENT in that context); Remove requirement for
-; DATETIME to be of datatype DOUBLE - Version 4.0
-; 20140521: Account for new Dimension Ordering rules for *.BOUNDARIES variables - should be
-; INDEPENDENT;*[;DATETIME]. In the event that the input metadata specifies a data template,
-; then the template VAR_DEPEND ordering will apply (and be checked by the template checker),
-; otherwise full checks will be made - Version 4.0b21
-; 20140806: Allow VAR_UNITS=d/D to be changed to MJD2K for DATETIME checks - Version 4.0b22
-; 20150811: Bug fix - initialize the infotxt array if reporting an ISO8601 to MJD2K conversion
-; issue - Version 4.0b29
-; 20151109: Check that VAR_SIZE values are in INTEGER format, even though they are
-; written to the HDF file as a string - Version 4.0b34
-; 20161116: Bug fix - resvd array value set correctly when checking whether a dataset with
-; VAR_DEPEND=CONSTANT only has a single value (was 0 should have been 1) - Version 4.0b39
-; 20161130: Allow program to continue when VAR_UNITS is not set to MJD2K for DATETIME
-; and related datasets (previously called STOP_WITH_ERROR in some conditions -
-; Version 4.0b40
-; 20180218: Change Unexpected number of attribute values extracted error message so that it calls
-; INFOTXT_OUTPUT rather than STOP_WITH_ERROR - Version 4.0b45
-; 20210617: Add section to do VAR_DATA_TYPE checks on the datasets, VAR_VALID_MIN, VAR_VALID_MAX and
-; VAR_FILL_VALUE values, to ensure they are all matching - Version 4.0b59
-; 20231218: Change check that an axis variable must be listed in DATA_VARIABLES before any variable
-; that depends on it. The new check is that the axis variable is present anywhere in the file.
-; Text for errtxt[3] changed, code added to identify axis variables and determine their
-; VAR_SIZE - Version 4.0b61
-; 20240112: Remove test that INDEPENDENT can only be a second dimension variable if the variable includes
-; .BOUNDARIES in the name e.g. ALTITUDE.BOUNDARIES. This is too limiting e.g. for Aerosol Lidar
-; RANGE_INDEPENDENT_NORMALIZATION has VAR_DEPEND=INDEPENDENT;DATETIME - Version 4.0b62
-;
-; Inputs: sds - Either a structure containing the Variable Attributes and Data, or the input
-; data file
-; inf - flag identifying SDS type where, 0=structure; 1=data file
-; meta_arr - a string array containing the Global and Variable Attributes
-;
-; Outputs: meta_arr - some meta_arr values may be corrected or calculated in this routine, including:
-; VAR_DIMENSION, VAR_SIZE, and VAR_FILL_VALUE
-; nvn - an integer giving the number of Variable Attributes present in the input
-; vn - a string array of size nvn containing the VAR_NAME values
-; vu - a string array of size nvn containing the VAR_UNIT values
-; ndm - a (nvn,9) long array, where ndm(*,0:7) describes the dimensions of each dataset
-; (maximum of 8 dimensions allowed), and ndm(*,8) describes the VAR_DATA_TYPE
-; ds - the structure that will hold the data
-;
-; Called by: IDLCR8HDF
-;
-; Subroutines Called: EXTRACT_DATA (if it is necessary to determine VAR_SIZE from the input data)
-; STOP_WITH_ERROR (if error state detected); INFOTXT_OUTOUT
-; Possible Conditions for STOP_WITH_ERROR call (plus [line number] where called):
-; 1. Program expects to find the VAR_NAME in the metadata in the same order as that listed under
-; DATA_VARIABLES
-; 2. The VAR_DIMENSION value is greater than 8
-; 3. Number of VAR_SIZE and VAR_DEPEND attribute values is different
-; 4. The VAR_DEPEND value is not 'CONSTANT', 'INDEPENDENT' or self-referencing, and doesn't
-; match a previous VAR_NAME
-; 5. The VAR_DEPEND variable name must, itself, have a VAR_DEPEND value of 'CONSTANT' or
-; 'INDEPENDENT' or be self-referencing
-; 6. The VAR_DEPEND variable name must have a VAR_SIZE value matching the current VAR_NAME
-; 7. VAR_DATA_TYPE not BYTE, SHORT, INTEGER, LONG, REAL, DOUBLE, or STRING
-; 8. Unexpected number of ATTRIBUTE values extracted
-; 9. VAR_DATA_TYPE should be DOUBLE for VAR_UNITS=MJD2K - No longer a requirement
-; 10. VAR_UNITS should be MJD2K for VAR_NAME=DATETIME
-; 11. VAR_FILL_VALUE not a valid value - occurs on an unsuccessful call to JULIAN_DATE
-; 12. VAR_DEPEND values not valid - occurs when INDEPENDENT or CONSTANT is a VAR_DEPEND value
-; when it should not be
-; 13. First VAR_SIZE value is not '2' when VAR_NAME=ALTITUDE/ALTITUDE.GPH/PRESSURE.BOUNDARIES
-; 14. Axis variable cannot be dependent on another variable
-; 15. Axis variable in the vertical dimension already found
-; 16. Attribute which is not an axis variable has a self-referencing VAR_DEPEND value
-; 17. Type conversion error. Called when trying to convert a string, which is expected to be
-; numeric, to a number
-;
-; Information Conditions (when the program is able to make changes):
-; 1. 64-bit LONG Data Type is not currently supported in GEOMS. Will attempt to write data in 32-bit
-; Integer type instead
-; 2. Dataset/VAR_VALID_MIN/VAR_VALID_MAX/VAR_FILL_VALUE value(s) have data type that does not match
-; VAR_DATA_TYPE
-
-COMMON METADATA
-COMMON DATA
-COMMON WIDGET_WIN
-
-;possible error messages for this procedure
-errtxt=STRARR(17) & lu=-1L
-errtxt[0]='Does not match the equivalent VAR_NAME in DATA_VARIABLES: '
-errtxt[1]='Number of dimensions too high to make HDF file (max. allowed 8)'
-errtxt[2]='VAR_DEPEND attribute value missing or does not match the number of VAR_SIZE values'
-errtxt[3]='VAR_DEPEND value not CONSTANT,INDEPENDENT nor self-referencing, and doesn''t match any VAR_NAME: '
-errtxt[4]='VAR_DEPEND variable name(s) must have VAR_DEPEND value of CONSTANT or be self_referencing: '
-errtxt[5]='VAR_DEPEND variable name must have matching VAR_SIZE value (= '
-errtxt[6]='Data type not BYTE, SHORT, INTEGER, REAL, DOUBLE, or STRING: '
-errtxt[7]='Unexpected number of ATTRIBUTE values extracted: '
-errtxt[8]='VAR_DATA_TYPE must be DOUBLE for VAR_UNITS=MJD2K: '
-errtxt[9]='VAR_UNITS must be MJD2K for ' ;no longer used (infotxt message instead)
-errtxt[10]='VAR_FILL_VALUE not a valid value: '
-errtxt[11]=' not valid'
-errtxt[12]='Second VAR_SIZE value should be ''2'' for this VAR_NAME'
-errtxt[13]='Axis variable cannot be dependent on another variable: '
-errtxt[14]=' is already the axis variable in the vertical dimension'
-errtxt[15]='is not an axis variable so VAR_DEPEND value cannot be self-referencing'
-errtxt[16]='Type conversion error. Entry is not a valid number: '
-proname='Set_Up_Structure procedure: '
-
-lc=0 & vc=0 & writeonce=0 & bounderror=0
-allowable_data_types=['BYTE','SHORT','INTEGER','LONG64','REAL','DOUBLE','STRING']
-idl_data_type=[1,2,3,14,4,5,7] ;IDL type identifiers matching allowable_data_types
-num_atts=['VAR_VALID_MIN','VAR_VALID_MAX','VAR_FILL_VALUE']
-nna=N_ELEMENTS(num_atts)
-
-ON_IOERROR,TypeConversionError
-
-;Do a search for a DATA_TEMPLATE value. If not present will do all .BOUNDARIES checks, otherwise
-;these checks will be done by the Template QA program - interim until .BOUNDARIES dimension ordering
-;is consistent for all templates (FTIR retains original convention as of 20140521)
-dtfound=0 ;default to template not present
-dti=WHERE(STRMID(meta_arr,0,13) EQ 'DATA_TEMPLATE',dtcnt)
-IF dtcnt NE 0 THEN BEGIN ;check there is a value
- res=STRSPLIT(meta_arr[dti[0]],' =;',/Extract,COUNT=rescnt)
- IF rescnt EQ 2 THEN dtfound=1 ;valid DATA_TEMPLATE value found
-ENDIF
-
-;Determine a set of possible Axis Variables to be tested (i.e. all VAR_DEPEND values that aren't CONSTANT or INDEPENDENT)
-vdi=WHERE(STRMID(meta_arr,0,10) EQ 'VAR_DEPEND',vdcnt)
-vardeptest=[''] & vardepndim=INTARR(vdcnt) & vardepvals=STRARR(8,vdcnt)
-IF vdcnt NE 0 THEN BEGIN
- FOR i=0,vdcnt-1 DO BEGIN
- res=STRSPLIT(meta_arr[vdi[i]],' =;',/Extract,COUNT=rescnt)
- vardepndim[i]=rescnt-1
- IF rescnt GT 1 THEN BEGIN ;VAR_DEPEND values found
- FOR j=1,rescnt-1 DO BEGIN
- resuc=STRUPCASE(res[j]) & vardepvals[j-1,i]=res[j]
- ni=WHERE(resuc EQ STRUPCASE(vardeptest),ncnt)
- IF (ncnt EQ 0) AND (resuc NE 'INDEPENDENT') AND (resuc NE 'CONSTANT') THEN $
- vardeptest=[vardeptest,res[j]] ;Add VAR_DEPEND value to list of axis-variables to be checked
- ENDFOR
- ENDIF
- ENDFOR
-ENDIF
-;Remove '' from vardeptest
-gi=WHERE(vardeptest NE '',n_axv)
-IF n_axv NE 0 THEN BEGIN
- vardeptest=vardeptest[gi] & vardepndl=lonarr(n_axv)
- vni=WHERE(STRMID(meta_arr,0,8) EQ 'VAR_NAME',vncnt)
- ndm=LONARR(vncnt,9) & vu=STRARR(vncnt) ;just needed, at thu stage as dummy inputs to EXTRACT_DATA
- IF vncnt NE 0 THEN BEGIN
- vn=STRARR(vncnt) & vserror=vn ;vserror needed for dummy input to EXTRACT_DATA
- FOR i=0,vncnt-1 DO BEGIN
- res=STRSPLIT(meta_arr[vni[i]],' =',/Extract,COUNT=rescnt)
- IF rescnt EQ 2 THEN vn[i]=res[1]
- ENDFOR
- multidim=0B
- FOR i=0,n_axv-1 DO BEGIN
- vcx=WHERE(vardeptest[i] EQ vn,vcxcnt)
- IF vcxcnt NE 0 THEN ndimx=vardepndim[vcx[0]] ELSE ndimx=0
- IF (vcxcnt NE 0) AND (ndimx LE 1) THEN BEGIN
- dtest=['-1'] ;-1 identifies to the EXTRACT_DATA routine that ndl is all that is required
- EXTRACT_DATA,SDS,inf,vcx,dtest,ndl
- vardepndl(i)=ndl
- ENDIF ELSE IF ndimx GT 1 THEN multidim=1B ;axis variable has more than one dependency e.g. VAR_DEPEND=DATETIME;ALTITUDE for VAR_NAME=ALTITUDE
- ENDFOR
- IF multidim THEN BEGIN
- FOR i=0,n_axv-1 DO BEGIN
- vcx=WHERE(vardeptest[i] EQ vn,vcxcnt)
- IF vcxcnt NE 0 THEN ndimx=vardepndim[vcx[0]] ELSE ndimx=0
- IF (vcxcnt NE 0) AND (ndimx GT 1) THEN BEGIN
- dtest=['-1'] ;-1 identifies to the EXTRACT_DATA routine that ndl is all that is required
- EXTRACT_DATA,SDS,inf,vcx,dtest,ndl
- ;Determine actual number of dataset values
- FOR j=0,ndimx-1 DO BEGIN
- IF vardepvals[j,vcx[0]] NE vn[vcx[0]] THEN BEGIN
- di=WHERE(vardepvals[j,vcx[0]] EQ vardeptest,dcnt)
- IF dcnt NE 0 THEN ndl=ndl/vardepndl[di[0]]
- ENDIF
- ENDFOR
- vardepndl(i)=ndl
- ENDIF
- ENDFOR
- ENDIF
- ENDIF
-ENDIF
-
-;At this point have got an array of size n_axv containing the axis variable names (vardeptest) and their VAR_SIZE (vardepndl)
-
-notlong=0B ;Boolean to check whether VAR_SIZE values are of integer data type
-WHILE lc LT N_ELEMENTS(meta_arr) DO BEGIN
- valid=0 ;set to test for successful string conversion to a number
- IF STRMID(meta_arr[lc],0,14) EQ 'DATA_VARIABLES' THEN BEGIN ;determine number of variables reported
- lcdv=lc
- vn=STRSPLIT(meta_arr[lc],' =;',/Extract) ;string containing reported variables
- nvn=N_ELEMENTS(vn)-1 ;total number of variables reported
- vn=vn[1:nvn] ;remove the DATA_VARIABLES label
- vu=STRARR(nvn) ;holding the VAR_UNITS values
- vfvd=DBLARR(nna,nvn) ;holding array for floating point num_atts array values
- vfvl=LON64ARR(nna,nvn) ;holding array for integer num_atts array values
- ndm=LONARR(nvn,9) ;to contain the dimensions and data type for each dataset
- vserror=STRARR(nvn) ;to hold variable names for datasets where the VAR_SIZE values cannot be determined
- ENDIF
-
- IF STRMID(meta_arr[lc],0,8) EQ 'VAR_NAME' THEN BEGIN
- vname=meta_arr[lc] ;used in case of error
- lcz=lc ;VAR_NAME index value
- res=STRSPLIT(meta_arr[lc],' =',/Extract)
- IF N_ELEMENTS(res) NE 2 THEN res=[res[0],'-1']
- IF res[1] NE vn[vc] THEN BEGIN
- STOP_WITH_ERROR,o3[3]+proname+vname+': ',errtxt[0]+vn[vc],lu & RETURN
- ;Does not match the equivalent VAR_NAME in DATA_VARIABLES
- ENDIF
-
- REPEAT lc=lc+1 UNTIL STRMID(meta_arr[lc],0,8) EQ 'VAR_SIZE'
- holdvs=meta_arr[lc] & lcx=lc ;do checks after determining VAR_DATA_TYPE
- REPEAT lc=lc+1 UNTIL STRMID(meta_arr[lc],0,10) EQ 'VAR_DEPEND'
- holdvd=meta_arr[lc] & lcy=lc ;do checks after determining VAR_DATA_TYPE
- REPEAT lc=lc+1 UNTIL STRMID(meta_arr[lc],0,13) EQ 'VAR_DATA_TYPE'
- res=STRSPLIT(STRUPCASE(meta_arr[lc]),' =;',/Extract)
- IF N_ELEMENTS(res) EQ 2 THEN gi=WHERE(res[1] EQ allowable_data_types,gcnt) $ ;Check VAR_DATA_TYPE is valid
- ELSE gcnt=0
- IF gcnt NE 1 THEN BEGIN
- STOP_WITH_ERROR,o3[3]+proname+vname+': ',errtxt[6]+meta_arr[lc],lu & RETURN
- ;Data type not BYTE, SHORT, INTEGER, LONG64, REAL, DOUBLE, or STRING
- ENDIF
- IF gi[0] EQ 3 THEN BEGIN
- IF writeonce EQ 0 THEN BEGIN
- writeonce=1
- infotxt='2 64-bit LONG Data Type is not currently supported|. '
- infotxt=infotxt+'Will attempt to write data in 32-bit INTEGER Data Type'
- INFOTXT_OUTPUT,infotxt
- ENDIF
- gi[0]=2 & meta_arr[lc]='VAR_DATA_TYPE=INTEGER'
- ENDIF
- ndm[vc,8]=gi[0] ;index number representing the data type
-
- ;Do VAR_SIZE and VAR_DEPEND checks
- resvs=STRSPLIT(holdvs,' =;',/Extract,COUNT=nvs) ;VAR_SIZE entries
- resvd=STRSPLIT(holdvd,' =;',/Extract,COUNT=nd) ;VAR_DEPEND entries
- nvs=nvs-1 & nd=nd-1 ;Number of Dimensions based on number of VAR_SIZE/VAR_DEPEND values
- vsadded=0
-
- IF (nvs EQ 0) AND (nd NE 0) THEN BEGIN
- ;Determine resvs values from vardeptest and vardepndl
- FOR k=1,nd DO BEGIN
- mi=WHERE(resvd[k] EQ vardeptest,mcnt)
- IF mcnt NE 0 THEN BEGIN
- resvs=[resvs,strtrim(vardepndl[mi[0]],2)]
- nvs++
- IF k NE nd THEN sctxt=';' ELSE sctxt=''
- holdvs=holdvs+strtrim(vardepndl[mi[0]],2)+sctxt
- vsadded=1
- ENDIF
- ENDFOR
- meta_arr[lcx]=holdvs
- ENDIF
- IF nvs EQ 0 THEN ndmok=0 ELSE ndmok=1
- ;If there are no VAR_SIZE value entries, need to determine from Data file or structure or
- ;from the VAR_SIZE values of the VAR_DEPEND dependencies
- IF ((nvs NE nd) AND (ndmok EQ 1)) OR (nd EQ 0) THEN BEGIN
- STOP_WITH_ERROR,o3[3]+proname+vname+': ',errtxt[2],lu & RETURN
- ;VAR_DEPEND attribute value missing or does not match the number of VAR_SIZE values
- ENDIF ELSE IF nd GT 8 THEN BEGIN
- STOP_WITH_ERROR,o3[3]+proname+vname+': ',errtxt[1],lu & RETURN
- ;Number of dimensions too high to make HDF file (max. allowed 8)
- ENDIF ELSE IF ndmok EQ 1 THEN BEGIN
- ndm[vc,0:nd-1]=LONG(resvs[1:nvs]) ;save VAR_SIZE values in ndm array
-
- ;Check that original input values are of integer data type
- FOR k=1,nvs DO BEGIN
- IF STRTRIM(resvs[k],2) NE STRTRIM(LONG(resvs[k]),2) THEN BEGIN
- resvs[k]=STRTRIM(LONG(resvs[k]),2)
- notlong=1B
- ENDIF
- ENDFOR
- IF notlong THEN BEGIN
- FOR k=1,nvs DO IF k EQ 1 THEN vshold=resvs[k] ELSE vshold=vshold+';'+resvs[k]
- meta_arr[lcx]='VAR_SIZE='+vshold
- holdvs=meta_arr[lcx]
- ENDIF
- ENDIF
-
- vshold=LONARR(nd) ;to hold VAR_SIZE of the dependencies
-
- ;Do checks on VAR_DEPEND values
- ci=WHERE((STRUPCASE(resvd) EQ 'INDEPENDENT') OR (STRUPCASE(resvd) EQ 'CONSTANT'),ccnt)
- IF ccnt NE 0 THEN resvd[ci]=STRUPCASE(resvd[ci])
- ;Rules for .BOUNDARY VAR_NAME
- IF STRPOS(vn[vc],'.BOUNDARIES') NE -1 THEN BEGIN
- test1=0 & test2=0
- IF ((nd EQ 2) OR (nd EQ 3)) AND (dtfound EQ 0) THEN BEGIN
- ;First VAR_DEPEND value must be INDEPENDENT
- test1=resvd[1] NE 'INDEPENDENT'
- ;Second VAR_DEPEND value must be present in the Variable Name e.g. ALTITUDE for ALTITUDE.BOUNDARIES
- secvdsplt=STRSPLIT(resvd[2],'.',/Extract)
- test2=STRPOS(vn[vc],secvdsplt[0]) EQ -1
- ENDIF ELSE IF (nd LE 1) OR (nd GE 4) THEN test2=1 ;incorrect number of VAR_DEPEND values
- IF nd EQ 3 THEN IF resvd[3] NE 'DATETIME' THEN test1=1 ;Third VAR_DEPEND value must be DATETIME
- IF (test1) OR (test2) THEN BEGIN
- IF test1 EQ 1 THEN itxt=' in IDL/Fortran Dimension Ordering' ELSE itxt=''
- infotxt='3 '+holdvd+' not valid for '+vname+itxt
- INFOTXT_OUTPUT,infotxt
- bounderror=1
- ;STOP_WITH_ERROR,o3[3]+proname+vname+': ',holdvd+errtxt[11]+' for the given VAR_NAME'+itxt,lu
- ;VAR_DEPEND value(s) not valid for the given VAR_NAME
- ;RETURN
- ENDIF
- ENDIF
-
- FOR i=1,nd DO BEGIN
- IF i EQ 1 THEN BEGIN
- ;Check for and Apply GEOMS rule changes for INDEPENDENT and axis variables
- vdval=resvd[i] & vnval=vn[vc]
- GEOMS_RULE_CHANGES,2,vardeptest,vnval,vdval,holdvd
- IF vdval NE resvd[i] THEN BEGIN ;First VAR_DEPEND value has been changed
- ;Rebuild VAR_DEPEND entry
- holdvd='VAR_DEPEND='+vdval
- IF nd GT 1 THEN FOR j=2,nd DO holdvd=holdvd+';'+resvd[j]
- ENDIF
- IF STRMID(holdvd,0,1) EQ 'E' THEN BEGIN ;error found while attempting to identify axis variables
- CASE 1 OF
- holdvd EQ 'E1': STOP_WITH_ERROR,o3[3]+proname+vname+': ',errtxt[13]+vdval,lu
- holdvd EQ 'E3': STOP_WITH_ERROR,o3[3]+proname+vname+' ',errtxt[15],lu
- ENDCASE
- RETURN
- ENDIF
- resvd[i]=vdval
- meta_arr[lcy]=holdvd
- ENDIF
- test1=(resvd[i] EQ 'CONSTANT') AND ((i GT 1) OR (nd GE 2)) ;CONSTANT can be the only VAR_DEPEND
- test2=(resvd[i] EQ vn[vc]) AND ((i GT 1) OR (nd GT 2)) ;e.g. ALTITUDE;DATETIME are possible dependencies for ALTITUDE
- test3=0B ;(resvd[i] EQ 'INDEPENDENT') AND ((i GT 1) OR (nd GE 2)) AND (STRPOS(vn[vc],'.BOUNDARIES') EQ -1) ;remove test as too specific
- test4=(resvd[i] NE 'CONSTANT') AND (resvd[i] NE 'INDEPENDENT') AND (resvd[i] NE vn[vc])
- ;Check for CONSTANT,INDEPENDENT or self-referencing and more than one VAR_DEPEND value
- IF (test1) OR (test2) OR (test3) THEN BEGIN
- STOP_WITH_ERROR,o3[3]+proname+vname+': ',holdvd+errtxt[11],lu & RETURN
- ;VAR_DEPEND value(s) not valid
- ENDIF
- ;Check for valid dependencies, and hold VAR_SIZE values of the dependencies if required
- ;Section modified 20231218 to allow axis variable to be after the variable that depends on it
- CASE 1 OF
- (vc EQ 0) AND (nvn GE 2): gi=WHERE(resvd[i] EQ vn[1:nvn-1],gcnt)
- (vc EQ nvn-1) AND (nvn GE 2): gi=WHERE(resvd[i] EQ vn[0:nvn-2],gcnt)
- nvn GE 3: BEGIN
- gi=WHERE(resvd[i] EQ vn[0:vc-1],gcnt)
- IF gcnt EQ 0 THEN BEGIN
- gi=WHERE(resvd[i] EQ vn[vc+1:nvn-1],gcnt)
- IF gcnt NE 0 THEN gi[0]=gi[0]+vc+1
- ENDIF
- END
- ELSE: gcnt=0
- ENDCASE
- ;Original check
- ;IF vc GE 1 THEN gi=WHERE(resvd[i] EQ vn[0:vc-1],gcnt) ELSE gcnt=0
- IF (gcnt EQ 0) AND (test4) THEN BEGIN
- STOP_WITH_ERROR,o3[3]+proname+vname+': ',errtxt[3]+resvd[i],lu & RETURN
- ;VAR_DEPEND value not CONSTANT,INDEPENDENT nor Self-referencing, and doesn't match any VAR_NAME
- ENDIF
- IF gcnt NE 0 THEN BEGIN ;do remaining checks on VAR_DEPEND values
- errfound=0
- gi=WHERE(meta_arr EQ 'VAR_NAME='+resvd[i])
- ;check VAR_DEPEND value of the VAR_DEPEND variable name
- vi=WHERE(STRPOS(meta_arr[gi[0]:N_ELEMENTS(meta_arr)-1],'VAR_DEPEND') NE -1)
- vex=STRSPLIT(STRUPCASE(meta_arr[gi[0]+vi[0]]),' =;',/Extract,COUNT=nvex)
- IF nvex NE 2 THEN vex=[vex,''] ;missing VAR_DEPEND value so add dummy value
- IF (vex[1] NE 'CONSTANT') AND (vex[1] NE 'INDEPENDENT') AND (vex[1] NE resvd[i]) THEN BEGIN
- vert_var=['ALTITUDE','ALTITUDE.GPH','PRESSURE','DEPTH']
- res=WHERE(resvd[i] EQ vert_var,vcnt) ;does VAR_DEPEND variable name reference a vertical axis
- res=WHERE(vex[1] EQ vert_var,xcnt) ;does VAR_DEPEND of the VAR_DEPEND variable name reference a vertical axis
- IF (vcnt NE 0) AND (xcnt EQ 0) THEN BEGIN ;Vertical axis so should be self-referencing
- infotxt='2 '+meta_arr[gi[0]+vi[0]]+' should be self-referencing for axis variable'
- infotxt=infotxt+' VAR_NAME='+resvd[i]+'|. VAR_DEPEND value changed in metadata'
- INFOTXT_OUTPUT,infotxt
- meta_arr[gi[0]+vi[0]]='VAR_DEPEND='+resvd[i]
- IF nvex GT 2 THEN FOR j=2,nvex-1 DO meta_arr[gi[0]+vi[0]]=meta_arr[gi[0]+vi[0]]+';'+vex[j]
- ENDIF ELSE BEGIN
- ;VAR_DEPEND value of the VAR_DEPEND variable name must be CONSTANT or an axis variable
- infotxt=STRARR(2)
- infotxt[0]='3 '+meta_arr[lcy]+' variable name(s) for '+vname+' must have VAR_DEPEND'
- infotxt[1]=' value of CONSTANT or be self-referencing'
- INFOTXT_OUTPUT,infotxt
- ENDELSE
- ENDIF
- IF vsadded EQ 0 THEN BEGIN
- ;check VAR_SIZE value of the VAR_DEPEND variable name
- vi=WHERE(STRPOS(meta_arr[gi[0]:N_ELEMENTS(meta_arr)-1],'VAR_SIZE') NE -1)
- vex=STRSPLIT(meta_arr[gi[0]+vi[0]],' =;',/Extract,COUNT=nvex)
-
- IF (nvex LT 2) OR (nvex GT 3) THEN errfound=1 $
- ELSE IF (ndmok EQ 1) AND (LONG(vex[1]) NE ndm[vc,i-1]) THEN errfound=1
-
- IF errfound EQ 1 THEN BEGIN
- STOP_WITH_ERROR,o3[3]+proname+vname+': '+'VAR_DEPEND='+resvd[i]+': ', $
- errtxt[5]+STRTRIM(ndm[vc,i-1],2)+'): '+meta_arr[gi[0]+vi[0]],lu
- ;VAR_DEPEND variable name must have matching VAR_SIZE value
- RETURN
- ENDIF ELSE vshold[i-1]=LONG(vex[1])
- ENDIF
- ENDIF
- ;If i eq 2 and value is INDEPENDENT then check that the corresponding VAR_SIZE value is 2
- ;(if present), o/w generate errors
- IF (i EQ 2) AND (STRUPCASE(resvd[i]) EQ 'INDEPENDENT') THEN BEGIN
- IF (ndmok EQ 1) AND (ndm[vc,i-1] NE 2L) THEN BEGIN
- STOP_WITH_ERROR,o3[3]+proname+vname+': '+holdvs+': ',errtxt[12],lu & RETURN
- ;Second VAR_SIZE value should be '2' for this VAR_NAME
- ENDIF
- vshold[i-1]=2L ;set VAR_SIZE value in the event that it is not part of the Metadata
- ENDIF
- ENDFOR
-
- IF (ndmok EQ 0) OR (vsadded EQ 1) THEN BEGIN ;no VAR_SIZE value so need to determine from the data or the vshold array
- IF (vshold[0] EQ 0L) AND (vsadded EQ 0) THEN BEGIN
- ;VAR_DEPEND is Self-Referencing or CONSTANT so determine VAR_SIZE from the Data file or structure
- dtest=['-1'] ;-1 identifies to the EXTRACT_DATA routine that ndl is all that is required
- EXTRACT_DATA,SDS,inf,vc,dtest,ndl
- IF STRLEN(rerr[0]) GT 2 THEN RETURN
- ;Check if any other VAR_SIZE values have already been determined
- ;This could happen in the case VAR_DEPEND=ALTITUDE;DATETIME for VAR_NAME=ALTITUDE for e.g.
- gi=WHERE(vshold NE 0L,gcnt)
- ndltot=' ('+STRTRIM(ndl,2)+')'
- IF gcnt NE 0 THEN BEGIN
- FOR i=0,gcnt-1 DO ndl=ndl/FLOAT(vshold[gi[i]])
- ENDIF
- ;Make up VAR_SIZE string based on available values
- IF vshold[0] EQ 0L THEN vsstr=' (VAR_SIZE=x' ELSE vsstr='VAR_SIZE='+STRTRIM(vshold[0],2)
- IF nd GT 1 THEN FOR i=1,nd-1 DO $
- IF vshold[i] EQ 0L THEN vsstr=vsstr+';x' ELSE vsstr=vsstr+';'+STRTRIM(vshold[i],2)
- vsstr=vsstr+') '
- vshold[0]=LONG(ndl)
- bi=WHERE(vshold EQ 0,bcnt) ;Expected VAR_SIZE values are missing if bcnt not equal to 0
- IF (ndl NE vshold[0]) OR (bcnt NE 0) THEN BEGIN
- infotxt=STRARR(2)
- infotxt[0]='3 Unable to determine missing VAR_SIZE value(s) for '+vname+' based on'
- infotxt[1]=' '+holdvd+vsstr+'and the total number of dataset values'+ndltot
- INFOTXT_OUTPUT,infotxt
- vserror[vc]=vn[vc]
- ENDIF
- meta_arr[lcx]='VAR_SIZE='+STRTRIM(vshold[0],2)
- ndm[vc,0:nd-1]=vshold
- IF nd GT 1 THEN FOR i=1,nd-1 DO meta_arr[lcx]=meta_arr[lcx]+';'+STRTRIM(vshold[i],2)
- ENDIF ELSE IF vsadded EQ 0 THEN BEGIN ;can determine VAR_SIZE from the vshold array
- meta_arr[lcx]='VAR_SIZE='
- FOR i=0,nd-1 DO BEGIN
- IF i EQ 0 THEN meta_arr[lcx]=meta_arr[lcx]+STRTRIM(vshold[i],2) $
- ELSE meta_arr[lcx]=meta_arr[lcx]+';'+STRTRIM(vshold[i],2)
- ndm[vc,i]=vshold[i] ;EQ array dimensions from VAR_SIZE of the dependencies
- ENDFOR
- ENDIF
- ;Check to see if VAR_SIZE calculation is valid or not based on contents of vserror
- vsok=1
- FOR i=1,nd DO BEGIN
- ei=WHERE(resvd[i] EQ vserror,ecnt)
- IF ecnt NE 0 THEN vsok=0
- ENDFOR
- IF (vsok EQ 1) OR (vsadded EQ 1) THEN BEGIN
- IF qa_yes THEN qtxt=' not recorded. Calculated as '+meta_arr[lcx] ELSE qtxt=' added: '+meta_arr[lcx]
- infotxt='2 VAR_SIZE value(s) for '+vname+qtxt
- INFOTXT_OUTPUT,infotxt
- ENDIF
- ENDIF
-
- IF (resvd[1] EQ 'CONSTANT') AND (TOTAL(ndm[vc,0:7]) GT 1.) THEN BEGIN
- infotxt='3 Dataset '+vn[vc]+' must only have a single value for VAR_DEPEND=CONSTANT: '+meta_arr[lcx]
- INFOTXT_OUTPUT,infotxt
- ENDIF
-
- ;Do DATETIME/MJD2000 checks
- REPEAT lc=lc+1 UNTIL STRMID(meta_arr[lc],0,9) EQ 'VAR_UNITS'
- resv=STRSPLIT(meta_arr[lc],' =',/Extract,Count=rcnt)
- IF rcnt EQ 1 THEN resv=[resv,'[MISSING]'] ;add filler for checks
- test1=(vn[vc] EQ 'DATETIME') OR (vn[vc] EQ 'DATETIME.START') OR (vn[vc] EQ 'DATETIME.STOP')
- IF (test1) AND (STRUPCASE(resv[1]) NE 'MJD2K') THEN BEGIN
- lch=lc ;hold setting for VAR_UNITS
- ;Find VAR_SI_CONVERSION values and test for correct values for MJD2K
- REPEAT lch=lch+1 UNTIL STRMID(meta_arr[lch],0,17) EQ 'VAR_SI_CONVERSION'
- res=STRSPLIT(meta_arr[lch],' =;',/EXTRACT)
- test1=(STRLOWCASE(resv[1]) EQ 's') AND (FIX(res[1]) EQ 0) AND (LONG(res[2]) EQ 86400L) AND $
- (STRLOWCASE(res[3]) EQ 's')
- IF (resv[1] EQ '[MISSING]') OR (resv[1] EQ '1') OR (STRLOWCASE(resv[1]) EQ 'd') OR (test1) THEN BEGIN
- errid='2 ' & itxt='|. Value changed in Metadata'
- ENDIF ELSE BEGIN
- errid='3 ' & itxt=''
- ENDELSE
- infotxt=errid+meta_arr[lc]+' must be VAR_UNITS=MJD2K for '+vname+itxt
- INFOTXT_OUTPUT,infotxt
- meta_arr[lc]='VAR_UNITS=MJD2K'
- resv=['VAR_UNITS','MJD2K']
- ;Change VAR_SI_CONVERSION value also
- CASE 1 OF
- ndm[vc,8] LE 3: meta_arr[lch]='VAR_SI_CONVERSION=0;86400;s
- ndm[vc,8] LE 5: meta_arr[lch]='VAR_SI_CONVERSION=0.0;86400.0;s'
- ELSE: meta_arr[lch]='VAR_SI_CONVERSION='
- ENDCASE
- ENDIF
- vu[vc]=resv[1] ;used to check for MJD2K units when reading in data as well as NCSA units value
- IF N_ELEMENTS(resv) GT 2 THEN $
- FOR i=2,N_ELEMENTS(resv)-1 DO vu[vc]=vu[vc]+' '+resv[i]
-
- ;Do checks on the numeric attributes
- FOR i=0,nna-1 DO BEGIN
- REPEAT lc=lc+1 UNTIL STRMID(meta_arr[lc],0,STRLEN(num_atts[i])) EQ num_atts[i]
- resv=STRSPLIT(meta_arr[lc],' =;',/Extract,COUNT=nresv)
- test1=(STRUPCASE(vu[vc]) NE 'MJD2K') OR (num_atts[i] EQ 'VAR_FILL_VALUE')
- test2=(nresv NE 2) AND (ndm[vc,8] NE 6)
- IF (test1) AND (test2) THEN BEGIN
- infotxt=STRARR(2)
- infotxt[0]='3 Unexpected number of ATTRIBUTE values extracted for '+vname+':'
- infotxt[1]=' '+meta_arr[lc]
- INFOTXT_OUTPUT,infotxt
- resv=[resv,''] ;ensure at least 2 values
- meta_arr[lc]=resv[0]+'='+resv[1] & nresv=2
- ;STOP_WITH_ERROR,o3[3]+proname+vname+': ',errtxt[7]+meta_arr[lc],lu & RETURN
- ;Unexpected number of ATTRIBUTE values extracted
- ENDIF
- IF (ndm[vc,8] NE 6) AND (nresv EQ 2) THEN BEGIN
- ;If VAR_UNITS=MJD2000 then check for ISO8601 format and change to MJD2000 if necessary
- IF (STRUPCASE(vu[vc]) EQ 'MJD2K') AND (STRPOS(STRUPCASE(resv[1]),'Z') NE -1) THEN BEGIN
- mjd2000=JULIAN_DATE(resv[1],/I,/M) ;return time in MJD2000 format
- IF mjd2000 EQ -99999.d THEN BEGIN
- STOP_WITH_ERROR,o3[3]+proname+vn[vc]+': ',errtxt[10]+meta_arr[lc],lu & RETURN
- ENDIF
- vfvd[i,vc]=mjd2000
- infotxt=STRARR(2)
- meta_arr[lc]=resv[0]+'='+STRTRIM(STRING(format='(d18.9)',mjd2000),2)
- IF qa_yes THEN qtxt=' must be in MJD2K format|' ELSE qtxt=' replaced with MJD2K formatted value '
- infotxt[0]='2 '+resv[0]+'='+resv[1]+' for VAR_NAME='+vn[vc]
- infotxt[1]=qtxt+meta_arr[lc]
- INFOTXT_OUTPUT,infotxt
- ENDIF ELSE BEGIN
- ;Check that the VAR_VALID_MIN/MAX or VAR_FILL_VALUE is numeric and save to vfvl or vfvd arrays
- lcx=lc
- IF ndm[vc,8] LE 3 THEN BEGIN ;value is integer type
- IF mv_lng[lc] NE 0LL THEN vfvl[i,vc]=mv_lng[lc] ELSE vfvl[i,vc]=LONG64(DOUBLE(resv[1]))
- ENDIF ELSE BEGIN ;value is floating point type
- IF mv_dbl[lc] NE 0.D THEN vfvd[i,vc]=mv_dbl[lc] ELSE vfvd[i,vc]=DOUBLE(resv[1])
- ENDELSE
- ENDELSE
- ENDIF
- ENDFOR
- vc=vc+1
- ENDIF
- lc=lc+1 & valid=1
-ENDWHILE
-
-IF notlong THEN BEGIN ;write out INFORMATION or ERROR to log
- IF qa_yes THEN itxt=' must be ' ELSE itxt=' changed to '
- infotxt='2 VAR_SIZE values'+itxt+'integer format'
- INFOTXT_OUTPUT,infotxt
-ENDIF
-
-;Save numeric metadata values to common arrays
-mv_lng=vfvl & mv_dbl=vfvd
-
-IF qa_yes THEN BEGIN
- ON_IOERROR, NULL
- ;Check VAR_VALID_MIN, VAR_VALID_MAX, VAR_FILL_VALUE and Dataset data types match VAR_DATA_TYPE
- s_dims=SIZE(sds,/DIMENSIONS)
- pchkd=PTR_VALID(sds.data)
- pchkv=PTR_VALID(sds.va_v)
- pchkl=PTR_VALID(sds.va_l)
-
- IF s_dims[0] GE 1 THEN BEGIN
- attnamechk=['VAR_NAME','VAR_DATA_TYPE','VAR_VALID_MIN','VAR_VALID_MAX','VAR_FILL_VALUE']
- n_anc=N_ELEMENTS(attnamechk)
- FOR i=0,s_dims[0]-1 DO BEGIN
- vdtx=BYTARR(n_anc)+255B ;VAR_DATA_TYPE for the attnamechk values
- vnh=''
- FOR j=0,s_dims[1]-1 DO BEGIN
- IF (pchkl[i,j]) AND (pchkv[i,j]) THEN BEGIN
- res=*sds[i,j].va_l & res=STRTRIM(STRUPCASE(res),2)
- ai=WHERE(res EQ attnamechk,acnt)
- IF acnt NE 0 THEN BEGIN
- attval=*sds[i,j].va_v
- IF ai[0] EQ 0 THEN vnh=STRTRIM(STRUPCASE(attval),2) $
- ELSE IF ai[0] EQ 1 THEN BEGIN
- vdth=STRTRIM(STRUPCASE(attval),2)
- vi=WHERE(vdth EQ allowable_data_types,vcnt)
- IF vcnt NE 0 THEN vdtx[1]=idl_data_type[vi[0]]
- ENDIF ELSE vdtx[ai[0]]=SIZE(*sds[i,j].va_v,/TYPE)
- ENDIF
- ENDIF
- ENDFOR
- IF pchkd[i,0] THEN dvdtx=SIZE(*sds[i,0].data,/TYPE) ELSE dvdtx=255B ;Data type of the dataset
- ;PRINT,vnh,dvdtx,vdtx[1:4]
- IF vdtx[1] NE 255B THEN BEGIN
- IF (dvdtx NE vdtx[1]) AND (dvdtx NE 255B) THEN BEGIN
- vi=WHERE(dvdtx EQ idl_data_type,vcnt)
- IF vcnt NE 0 THEN svdt=allowable_data_types[vi[0]] ELSE svdt=''
- IF svdt EQ '' THEN $
- infotxt='3 '+vnh+' dataset has data type that does not match VAR_DATA_TYPE='+vdth $
- ELSE $
- infotxt='3 '+vnh+' dataset has data type '+svdt+', which does not match VAR_DATA_TYPE='+vdth
- INFOTXT_OUTPUT,infotxt
- ENDIF
- FOR j=2,n_anc-1 DO BEGIN
- IF vdtx[j] NE vdtx[1] THEN BEGIN
- vi=WHERE(vdtx[j] EQ idl_data_type,vcnt)
- IF vcnt NE 0 THEN svdt=allowable_data_types[vi[0]] ELSE svdt=''
- IF svdt EQ '' THEN $
- infotxt='3 '+vnh+' '+attnamechk[j]+ $
- ' value has data type that does not match VAR_DATA_TYPE='+vdth $
- ELSE $
- infotxt='3 '+vnh+' '+attnamechk[j]+' value has data type '+svdt+ $
- ', which does not match VAR_DATA_TYPE='+vdth
- INFOTXT_OUTPUT,infotxt
- ENDIF
- ENDFOR
- ENDIF
- ENDFOR
- ENDIF
-ENDIF
-
-;Define the HDF storage structure
-ds_set={data: PTR_NEW()} ;SDS data array
-ds=REPLICATE(ds_set,nvn) ;Dimension the structure to the size of the SDS datasets
-
-TypeConversionError:
-IF valid EQ 0 THEN BEGIN
- STOP_WITH_ERROR,o3[3]+proname+vname+': ',errtxt[16]+meta_arr[lcx],lu & RETURN
-ENDIF
-END ;Procedure Set_Up_Structure
-
-
-
-PRO check_string_datatype, vc, dtest, ndl
-;Procedure to check that the data, and Variable Attributes are correctly configured when the
-;VAR_DATA_TYPE=STRING, including VAR_UNITS, VAR_VALID_MIN/MAX, and VAR_FILL_VALUE. The data
-;values are returned in the string datatype.
-; ----------
-;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
-;
-; History:
-; 20091203: Introduced to IDLCR8HDF routine - Version 3.08
-; 20101120: Account for GEOMS rule changes from v3.0 to v4.0 - Version 4.0b0
-; 20111220: Check dataset entries for non-ISO646-US ASCII Characters - Version 4.0b7
-; 20130116: Allow for a string dataset with maximum string length of 0, i.e. a set
-; of empty values - Version 4.0b15
-; 20150127: Use right padding to make strings the correct length (previously defaulted to
-; left padding); Save maximum string length of dataset in mv_str array
-; - Version 4.0b25
-; 20150217: VAR_FILL_VALUE reverts to being an empty space (made same length as the longest
-; dataset string in 4.0b25); improve checks on variable attribute values; Make
-; the first dataset entry equal to the maximum string length if writing to H5
-; due to apparent bug when writing datasets which means that the string
-; length is determined by the first entry (white space is removed when writing to
-; the H5 file) - Version 4.0b26
-; 20220805: Improve Information/Error message when attributes for string datasets are not
-; valid - Version 4.0b60
-;
-; Inputs: meta_arr - a string array containing the Global and Variable Attributes
-; vc - the index value of the vn array holding the VARIABLE_NAME being searched for in the data
-; file, or the index value of the sds structure which holds the data
-; dtest - the set of data values to be tested
-; ndl - The total number of data values i.e. the product of the VAR_SIZE values
-;
-; Outputs: meta_arr - in the event that the values being tested are changed, the array will be updated
-; dtest - correctly formatted dataset
-;
-; Called by: READ_DATA
-;
-; Subroutines Called: INFOTXT_OUTPUT
-; Information Conditions (when the program is able to make changes):
-; 1. The Metadata value is not valid for VAR_DATA_TYPE=STRING [2645]
-
-COMMON METADATA
-COMMON DATA
-COMMON WIDGET_WIN
-
-vi=WHERE(meta_arr EQ 'VAR_NAME='+vn[vc])
-dtestx=STRTRIM(dtest,0) ;remove trailing blanks only, for testing that string datasets are left justified
-dtest=STRTRIM(dtest,2)
-
-;Check for non ISO646-US ASCII Characters
-GEOMS_RULE_CHANGES,10,dtest,vn[vc]
-
-IF ~ARRAY_EQUAL(dtest,dtestx) THEN BEGIN ;format of the string dataset has been changed
- IF qa_yes THEN $
- infotxt='2 Dataset '+vn[vc]+' incorrectly formatted (text must be left-justified)' $
- ELSE infotxt='2 Dataset '+vn[vc]+' reformatted so that text is left-justified'
- INFOTXT_OUTPUT, infotxt
-ENDIF
-
-;Determine maximum string length of the dataset
-mxdatalen=0L ;default value of string length
-testdatalen=MAX(STRLEN(dtest))
-IF testdatalen GT mxdatalen THEN mxdatalen=testdatalen
-;Format the first data value to the maximum string length for H5 only (white space removed when writing to the file)
-IF (mxdatalen NE 0L) AND (STRPOS(o3[0],'H5') NE -1) THEN BEGIN
- ;Done for H5 only because H5S_CREATE_SIMPLE function uses string length of the 1st entry to set length for all entries
- vfsx='(A-'+STRTRIM(mxdatalen,2)+')' ;A- does left justified text instead of right justified
- dtest[0]=STRING(format=vfsx,dtest[0])
-ENDIF
-
-label=['VAR_UNITS','VAR_SI_CONVERSION','VAR_VALID_MIN','VAR_VALID_MAX','VAR_FILL_VALUE']
-n_lab=N_ELEMENTS(label)
-mxlablen=MAX(STRLEN(label)) ;determine max label string length
-metahold=STRMID(meta_arr,0,mxlablen) ;subset of the meta_arr strings, containing first portion of the Metadata
-
-;Perform checks on the Attributes
-FOR i=0,n_lab-1 DO BEGIN
- li=WHERE(STRPOS(metahold[vi[0]:N_ELEMENTS(metahold)-1],label[i]) NE -1)
- lvi=li[0]+vi[0]
- res=STRSPLIT(meta_arr[lvi],'=',/Extract,COUNT=nres)
- IF nres EQ 1 THEN res=[res,'']
- IF res[1] EQ ' ' THEN res[1]=''
- IF (nres GT 2) OR (res[1] NE '') THEN BEGIN
- ;print,'"'+res[1]+'"/"'+itxt+'"/'+strtrim(mxdatalen,2)
- IF qa_yes THEN itxt='must be [' ELSE itxt='changed to ['
- infotxt='2 '+vn[vc]+' '+meta_arr[lvi]+' attribute '+itxt+res[0]+'= ]'
- INFOTXT_OUTPUT, infotxt
- mv_str[vc]=mxdatalen
- ENDIF
- meta_arr[lvi]=res[0]+'= ' ;+string(ltxt)
- IF res[0] EQ 'VAR_UNITS' THEN vu[vc]=' '
-ENDFOR
-
-END ;procedure Check_String_Datatype
-
-
-
-PRO check_min_max_fill, vc, dtest, ndl
-;Procedure to check that the data, VAR_VALID_MIN/MAX, and VAR_FILL_VALUE fit within the given data
-;type. That the VAR_FILL_VALUE falls within the GEOMS definition and that the data values are within
-;the valid minimum and maximum values (or the fill value). The VAR_VALID_MIN value is less than or
-;equal to the VAR_VALID_MAX value. The data values are returned in the data type given by VAR_DATA_TYPE.
-; ----------
-;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
-;
-; History:
-; 20050802: Original IDLCR8HDF routine - Version 1.0
-; 20050909: If VAR_NAME=DATETIME then checks that the first data value is not a fill value, and is
-; the lowest value; Change the methods for determining if a given data value falls within
-; the range allowed by integer or long data types; Add fix for the case where a dataset
-; consists of only fill values - Version 1.1
-; 20051107: Include VIS_SCALE_MIN/MAX values in the checks; Add fix to remove unwanted precision (to
-; prevent rounding errors), by converting values to formatted strings then back to numeric
-; values - Version 1.11
-; 20061012: Check that all DATETIME values are in chronological order (if more than one value), and
-; contain no fill values; ISO8601 DATETIME conversions to MJD2000 format (if necessary) moved
-; to the READ_DATA routine; Add additional checks when extracting the VIS_FORMAT values for
-; testing; Common variable definition WIDGET_WIN added - Version 2.0
-; 20080302: Fix bug so that unwanted precision from all data and relevant metadata values is removed
-; before doing QA checks, to avoid unnecessary error calls - Version 3.0
-; 20100205: Add RETURN command after all STOP_WITH_ERROR calls, which allows program to return to the
-; calling program if the reterr argument is included in the idlcr8hdf call - Version 3.09
-; 20101120: Account for GEOMS rule changes v3.0 to v4.0; Remove checks involving VIS_FORMAT; Modify
-; method for checking data type limitations - Version 4.0
-; 20120420: Fix CHECK_MATH check so that combined floating point errors are ignored for long integer
-; values being tested (error 160) - Version 4.0b9
-; 20120620: Remove CHECK_MATH checks, and just apply Type conversion checks, as it was not possible for
-; CHECK_MATH to accurately indicate the problem numeric value - Version 4.0b10
-; 20120829: Fix call to ROUND when the value being rounded is a string (ROUND can only be applied to
-; numeric datatypes) - Version 4.0b12
-; 20130327: Add additional check that VAR_VALID_MIN is less than or equal to the VAR_VALID_MAX in the
-; event that the dataset only contains fill values - Version 4.0b20
-; 20141110: Check for NaN in the dataset and change to the Fill Value - Version 4.0b24
-; 20170331: Add checks for VAR_VALID_MIN and MAX values for Latitude, Longitude, Zenith, Azimuth and
-; Wind direction datasets; check that the maximum dataset value for Azimuth datasets is
-; less than 360.0 degrees - Version 4.0b42
-; 20171130: Remove ds from STOP_WITH_ERROR parameters - Version 4.0b44
-; 20180901: Also check that DATETIME.START and DATETIME.STOP values are in chronological order -
-; Version 4.0b48
-; 20201211: Fix bug that caused an IDL Math Error (Floating Illegal Operand) when converting large
-; floating point values to integers - Version 4.0b58
-; 20220805: Allow program to continue if data type errors in the dataset and attributes are detected -
-; Version 4.0b60
-;
-; Inputs: meta_arr - a string array containing the Global and Variable Attributes
-; vc - the index value of the vn array holding the VARIABLE_NAME being searched for in the data
-; file, or the index value of the sds structure which holds the data
-; dtest - the set of data values to be tested
-; ndl - The total number of data values i.e. the product of the VAR_SIZE values
-;
-; Outputs: meta_arr - in the event that the VIS_FORMAT value string or the precision of the attribute
-; values being tested are changed, the array will be updated
-; dtest - dataset returned in correct data type (VAR_FILL_VALUE/VAR_VALID_MIN/VAR_VALID_MAX
-; also appended to the start of the array)
-;
-; Called by: READ_DATA
-;
-; Subroutines Called: STOP_WITH_ERROR (if error state detected)
-; INFOTXT_OUTPUT
-; Possible Conditions for STOP_WITH_ERROR call (plus [line number] where called):
-; 1. Value is not valid, or does not match VAR_DATA_TYPE [2826]
-; 2. A data value falls outside the range given by VAR_VALID_MIN or VAR_VALID_MAX [2846, 2851]
-; 3. The VAR_FILL_VALUE is not outside the valid minimum or maximum values - No longer a requirement
-; 4. The metadata or data value being tested is outside the data type range. For example,
-; if a data value is 40000, but VAR_DATA_TYPE=INTEGER (maximum allowable value of 32767)
-; [2786,2793,2800,2807,2813,2819]
-; 5. The DATETIME value cannot be a VAR_FILL_VALUE. Because DATETIME is an axis variable
-; it should not contain Fill Values [2862]
-; 6. The DATETIME values are not in chronological order [2868]
-; 7. Type conversion error. The program cannot convert a data value to a valid number [2874]
-;
-; Information Conditions (when the program is able to make changes):
-; 1. Fill value falls within the data range and is defined as a Default value [2836]
-; 2. NaN value in the dataset replaced with the fill value
-
-COMMON METADATA
-COMMON DATA
-COMMON WIDGET_WIN
-
-;Error messages for this procedure
-ON_IOERROR,TypeConversionError
-proname='Check_Min_Max_Fill procedure: '
-errtxt2=STRARR(7) & lu=-1
-errtxt2[0]=' not valid, or does not match VAR_DATA_TYPE'
-errtxt2[1]=' data value outside VAR_VALID_'
-errtxt2[2]='Fill data value not outside VAR_VALID_MIN or VAR_VALID_MAX: '
-errtxt2[3]=' outside the data type range: '
-errtxt2[4]='DATETIME value cannot be a VAR_FILL_VALUE. '
-errtxt2[5]='DATETIME values not in chronological order'
-errtxt2[6]='Type conversion error. Entry is not a valid value'
-
-vi=WHERE(meta_arr EQ 'VAR_NAME='+vn[vc])
-
-;Determine Var_Fill_Value, Var_Valid_Min, Var_Valid_Max, and data values,
-;and check that they fall within the range allowed by VAR_DATA_TYPE
-label=['VAR_VALID_MIN','VAR_VALID_MAX','VAR_FILL_VALUE','data value']
-n_lab=N_ELEMENTS(label)
-mxlablen=MAX(STRLEN(label[0:n_lab-2])) ;determine max label string length
-metahold=STRMID(meta_arr,0,mxlablen) ;subset of the meta_arr strings, containing first portion of the Metadata
-lvi=LONARR(n_lab-1) ;to hold index values of the VAR_FILL_VALUE/VALID_MIN/VALID_MAX attributes
-
-CASE ndm[vc,8] OF
- 0: vhold=BYTARR(ndl+(n_lab-1L))
- 1: vhold=INTARR(ndl+(n_lab-1L))
- 2: vhold=LONARR(ndl+(n_lab-1L))
- 3: vhold=LON64ARR(ndl+(n_lab-1L))
- 4: vhold=FLTARR(ndl+(n_lab-1L))
- 5: vhold=DBLARR(ndl+(n_lab-1L))
-ENDCASE
-
-IF SIZE(dtest,/TYPE) EQ 7 THEN convertstr=1 ELSE convertstr=0 ;dtest is in ASCII string format
-IF ndm[vc,8] LE 3 then convtolng=1 else convtolng=0
-
-nanfound=0 ;boolean setting so information/error message only written once
-FOR i=0L,ndl+(n_lab-2L) DO BEGIN
- valid=0 ;set-up for possible Type Conversion Error
- IF i LE n_lab-2L THEN BEGIN
- lin=i ;label index
- li=WHERE(STRPOS(metahold[vi[0]:N_ELEMENTS(metahold)-1],label[lin]) NE -1)
- lvi[i]=li[0]+vi[0]
- res=STRSPLIT(meta_arr[lvi[i]],'=',/Extract)
- IF convtolng THEN lv=mv_lng[i,vc] ELSE lv=mv_dbl[i,vc]
- tcerr=': '+meta_arr[lvi[i]] ;used for type conversion error
- ENDIF ELSE BEGIN ;check individual data values
- res=['',''] & lin=n_lab-1L ;label index
- tcerr=' in the Data File: '+STRTRIM(dtest[i-lin],2) ;used for type conversion error
- IF convertstr THEN BEGIN
- IF convtolng THEN lv=ROUND(DOUBLE(dtest[i-lin]),/L64) ELSE lv=DOUBLE(dtest[i-lin]) ;LONG64(DOUBLE(dtest[i-lin])) ELSE lv=DOUBLE(dtest[i-lin])
- ENDIF ELSE lv=dtest[i-lin]
- ENDELSE
- IF N_ELEMENTS(res) EQ 2 THEN mok=0 ELSE mok=-1
-
- IF mok EQ 0 THEN BEGIN
- ;test if value is 'NaN'. If so then change to the fill value if already determined
- IF (FINITE(lv,/NAN)) AND (i GT 2L) THEN BEGIN ;data value is NaN
- lv=vhold[2]
- IF nanfound EQ 0 THEN BEGIN
- infotxt='2 '+vn[vc]+' dataset contains NaN value(s)|. Replaced with Fill Value(s)'
- INFOTXT_OUTPUT,infotxt
- nanfound=1
- ENDIF
- ENDIF
- CASE ndm[vc,8] OF
- 0:BEGIN ;BYTE unsigned 8-bit integer (0 to 255)
- ;do difference test to account for possible rounding errors
- IF lv NE BYTE(lv) THEN BEGIN
- infotxt='3 '+vn[vc]+' '+label[lin]+' outside 8-bit unsigned (BYTE) range: '+STRTRIM(lv,2)
- INFOTXT_OUTPUT,infotxt
- ENDIF ELSE vhold[i]=BYTE(lv)
- END
- 1:BEGIN ;SHORT signed 16-bit integer (-32,768 to 32,767)
- ;do difference test to account for possible rounding errors
- IF lv NE FIX(DOUBLE(lv)) THEN BEGIN
- infotxt='3 '+vn[vc]+' '+label[lin]+' outside 16-bit signed (SHORT) range: '+STRTRIM(lv,2)
- INFOTXT_OUTPUT,infotxt
- ENDIF ELSE vhold[i]=FIX(DOUBLE(lv))
- END
- 2:BEGIN ;INTEGER signed 32-bit integer (-2.147e+9 to 2.147e+9)
- ;do difference test to account for possible rounding errors
- IF lv NE LONG(DOUBLE(lv)) THEN BEGIN
- infotxt='3 '+vn[vc]+' '+label[lin]+' outside 32-bit signed (INTEGER) range: '+STRTRIM(lv,2)
- INFOTXT_OUTPUT,infotxt
- ENDIF ELSE vhold[i]=LONG(DOUBLE(lv))
- END
- 3:BEGIN ;INTEGER signed 64-bit integer (-9.223e+18 to 9.223e+18)
- ;do difference test to account for possible rounding errors
- IF ABS(DOUBLE(lv)-DOUBLE(lv)) GE 1. THEN BEGIN
- infotxt='3 '+vn[vc]+' '+label[lin]+' outside 64-bit signed (LONG) range: '+STRTRIM(lv,2)
- INFOTXT_OUTPUT,infotxt
- ENDIF ELSE vhold[i]=LONG64(DOUBLE(lv))
- END
- 4:BEGIN
- IF NOT FINITE(FLOAT(lv)) THEN BEGIN
- infotxt='3 '+vn[vc]+' '+label[lin]+' outside 32-bit floating point (REAL) range: '+STRTRIM(lv,2)
- INFOTXT_OUTPUT,infotxt
- ENDIF ELSE vhold[i]=FLOAT(lv)
- END
- 5:BEGIN
- IF NOT FINITE(DOUBLE(lv)) THEN BEGIN
- infotxt='3 '+vn[vc]+' '+label[lin]+' outside 64-bit floating point (DOUBLE) range: '+STRTRIM(lv,2)
- INFOTXT_OUTPUT,infotxt
- ENDIF ELSE vhold[i]=DOUBLE(lv)
- END
- ENDCASE
- ENDIF ELSE BEGIN
- STOP_WITH_ERROR,o3[3]+proname+vn[vc],label[lin]+errtxt2[0]+tcerr,lu & RETURN
- ENDELSE
- valid=1 ;type conversion OK if got to here
-ENDFOR
-
-;Check if VAR_FILL_VALUE falls within VAR_VALID_MIN/MAX - if so it is defined
-;as a default, rather than missing/error value
-mnv=vhold[0] & mxv=vhold[1] & fv=vhold[2]
-IF (fv GE mnv) AND (fv LE mxv) THEN BEGIN
- infotxt='0 Fill value for '+meta_arr[vi[0]]+' falls within the data'
- infotxt=infotxt+' range and is defined as a ''Default'' value'
- INFOTXT_OUTPUT,infotxt
-ENDIF
-
-dtest=vhold[n_lab-1L:ndl+(n_lab-2L)]
-
-;Do checks on VAR_VALID_MIN and MAX values for Latitude, Longitide, Zenith, Azimuth and Wind Direction datasets
-IF STRPOS(vn[vc],'LATITUDE') NE -1 THEN BEGIN
- IF (mnv LT -90.0) OR (mxv GT 90.0) THEN BEGIN
- itxt='-90.0 (South) and +90.0 (North) degrees'
- infotxt='3 Valid Minimum and Maximum range values for '+vn[vc]+' are '+itxt
- INFOTXT_OUTPUT,infotxt
- ENDIF
-ENDIF ELSE IF STRPOS(vn[vc],'LONGITUDE') NE -1 THEN BEGIN
- IF (mnv LT -180.0) OR (mxv GT 180.0) THEN BEGIN
- itxt='-180.0 (West) and +180.0 (East) degrees'
- infotxt='3 Valid Minimum and Maximum range values for '+vn[vc]+' are '+itxt
- INFOTXT_OUTPUT,infotxt
- ENDIF
-ENDIF ELSE IF (STRPOS(vn[vc],'ZENITH') NE -1) AND (STRPOS(vn[vc],'ANGLE') NE -1) THEN BEGIN
- IF (mnv LT 0.0) OR (mxv GT 180.0) THEN BEGIN
- itxt='0.0 and 180.0 degrees'
- infotxt='3 Valid Minimum and Maximum range values for '+vn[vc]+' are '+itxt
- INFOTXT_OUTPUT,infotxt
- ENDIF
-ENDIF ELSE IF (STRPOS(vn[vc],'AZIMUTH') NE -1) AND (STRPOS(vn[vc],'ANGLE') NE -1) THEN BEGIN
- IF (mnv LT 0.0) OR (mxv GT 360.0) THEN BEGIN
- itxt='0.0 (North) and 360.0 degrees (East = 90 degrees and so on)'
- infotxt='3 Valid Minimum and Maximum range values for '+vn[vc]+' are '+itxt
- INFOTXT_OUTPUT,infotxt
- ENDIF
-ENDIF ELSE IF STRPOS(vn[vc],'WIND.DIRECTION') NE -1 THEN BEGIN
- IF (mnv NE 0.0) OR (mxv NE 360.0) THEN BEGIN
- itxt='0.0 (Calm) and 360.0 degrees (use WMO definition)'
- infotxt='3 Valid Minimum and Maximum range values for '+vn[vc]+' are '+itxt
- INFOTXT_OUTPUT,infotxt
- ENDIF
-ENDIF
-
-;determine whether minimum and maximum data values are within VAR_VALID_MIN/MAX
-nfvi=WHERE(dtest NE fv,nfvcnt)
-IF nfvcnt NE 0 THEN BEGIN ;i.e. there are non-fill values in the data
- minv=MIN(dtest(nfvi),minvi,MAX=maxv,SUBSCRIPT_MAX=maxvi)
- IF minv LT mnv THEN BEGIN ;Minimum data value is less than VAR_VALID_MIN
- IF ndm[vc,8] EQ 0 THEN itxt=STRTRIM(FIX(dtest[nfvi[minvi]]),2) $
- ELSE itxt=STRTRIM(dtest[nfvi[minvi]],2)
- infotxt='3 Minimum data value of '+itxt+' is outside '+meta_arr[lvi[0]]
- infotxt=infotxt+' for dataset '+vn[vc]
- INFOTXT_OUTPUT,infotxt
- ;STOP_WITH_ERROR,o3[3]+proname+vn[vc]+': ','Minimum'+errtxt2[1]+ $
- ; 'MIN: '+STRTRIM(dtest[nfvi[minvi]],2)+' ('+meta_arr[lvi[0]]+').',lu,ds
- ;RETURN
- ENDIF
- IF maxv GT mxv THEN BEGIN ;Maximum data value is greater than VAR_VALID_MAX
- IF ndm[vc,8] EQ 0 THEN itxt=STRTRIM(FIX(dtest[nfvi[maxvi]]),2) $
- ELSE itxt=STRTRIM(dtest[nfvi[maxvi]],2)
- infotxt='3 Maximum data value of '+itxt+' is outside '+meta_arr[lvi[1]]
- infotxt=infotxt+' for dataset '+vn[vc]
- INFOTXT_OUTPUT,infotxt
- ;STOP_WITH_ERROR,o3[3]+proname+vn[vc]+': ','Maximum'+errtxt2[1]+ $
- ; 'MAX: '+STRTRIM(dtest[nfvi[maxvi]],2)+' ('+meta_arr[lvi[1]]+').',lu,ds
- ;RETURN
- ENDIF ELSE IF (STRPOS(vn[vc],'AZIMUTH') NE -1) AND (STRPOS(vn[vc],'ANGLE') NE -1) THEN BEGIN
- ;also check whether maximum dataset value = 360.0 degrees
- IF LONG(maxv) EQ 360L THEN BEGIN
- infotxt='3 Maximum data value of 360.0 degrees is an invalid value for dataset '+vn[vc]+' (North = 0.0)'
- INFOTXT_OUTPUT,infotxt
- ENDIF
- ENDIF
-ENDIF ELSE BEGIN ;dataset consists of fill values so check that VAR_VALID_MIN LE VAR_VALID_MAX
- IF mnv GT mxv THEN BEGIN
- infotxt='3 VAR_VALID_MIN value is greater than the VAR_VALID_MAX value for dataset '+vn[vc]
- INFOTXT_OUTPUT,infotxt
- ENDIF
-ENDELSE
-
-;Do Checks with DATETIME attributes
-IF STRPOS(vn[vc],'DATETIME') EQ 0 THEN BEGIN ;i.e. DATETIME, DATETIME.START or DATETIME.STOP
- ;Does DATETIME contain any fill values?
- di=WHERE(dtest EQ fv,dcnt)
- IF (vn[vc] EQ 'DATETIME') AND (dcnt NE 0) THEN BEGIN
- ;'DTFVOK' = DATETIME fill value is OK
- IF (o3[5] EQ 'DTFVOK') AND (dcnt NE ndl) THEN itxt='1 ' ELSE itxt='3 '
- infotxt=itxt+'DATETIME contains fill value(s)'
- INFOTXT_OUTPUT,infotxt
- ;STOP_WITH_ERROR,o3[3]+proname,errtxt2[4]+STRTRIM(dtest[di[0]],2),lu,ds & RETURN
- ENDIF
- di=WHERE(dtest NE fv,dcnt)
- IF dcnt GT 1 THEN BEGIN
- ;Are DATETIME values in chronological order?
- dsort=SORT(dtest[di])
- IF ARRAY_EQUAL(dtest[di],dtest[di[dsort]]) EQ 0 THEN BEGIN
- IF o3[5] EQ 'DTFVOK' THEN itxt='1 ' ELSE itxt='3 '
- infotxt=itxt+vn[vc]+' values are not in chronological order'
- INFOTXT_OUTPUT,infotxt
- ;STOP_WITH_ERROR,o3[3]+proname,errtxt2[5],lu,ds & RETURN
- ENDIF
- ENDIF
-ENDIF
-
-TypeConversionError:
-IF valid EQ 0 THEN STOP_WITH_ERROR,o3[3]+proname+'VAR_NAME='+vn[vc]+': ',errtxt2[6]+tcerr,lu
-
-END ;procedure Check_Min_Max_Fill
-
-
-
-PRO read_data, sds, inf
-;Procedure to perform checks on the data file or structure. It looks for missing or invalid
-;VAR_VALID_MIN/MAX attribute values for datasets which have VAR_UNITS=MJD2K (including
-;start and stop times), and determines these values according to values in the data. It
-;also adds a comment to VAR_NOTES if averaging kernel data is present, to indicate the
-;proper array order (if this option is chosen on program start-up).
-; ----------
-;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
-;
-; History:
-; 20050802: Original IDLCR8HDF routine - Version 1.0
-; 20050909: Check that the input data file is not empty; test for the correct number of
-; datalines in the dataset being searched (e.g. there is not an additional data
-; line in the input file); Bug fix to save dtest as an array when there is only
-; one data value, and the input data is in a structure - Version 1.1
-; 20061012: Portion of routine originally extracting a dataset from the file or structure
-; moved to the EXTRACT_DATA routine; Code added to determine missing or invalid
-; minimum or maximum metadata values for DATETIME, or START/STOP.TIME attributes;
-; Code added to append a comment to VAR_NOTES if averaging kernel data is present,
-; to indicate proper array order (if /AVK option selected); Checked data written
-; to the ds structure instead of a series of arrays set-up according to data type;
-; Common variable definition WIDGET_WIN added - Version 2.0
-; 20080302: dtest initialized for the EXTRACT_DATA call - Version 3.0
-; 20100205: Add test to allow for RETURN to the calling program after STOP_WITH_ERROR calls
-; made by the external routines used by this this procedure - Version 3.09
-; 20101120: Account for new GEOMS rules regarding DATETIME.START/STOP and MJD2K; Do check for
-; averaging kernel vector instead of matrix, in which case do not add AVK comment,
-; if /AVK option chosen - Version 4.0
-; 20150127: Create mv_str array to hold maximum string length of dataset - Version 4.0b25
-; 20150217: Add check for mv_str so it is only created if input is via files; Change AVK
-; comment so it refers to the initial altitude level rather than lowest altitude
-; level - Version 4.0b26
-; 20150409: Strip trailing and leading spaces of VAR_VALID_MAX and VAR_VALID_MIN values
-; when doing checks for missing values for DATETIME datasets - Version 4.0b28
-; 20150811: Initialize infowrite value for call to EXTRACT_DATA - Version 4.0b29
-; 20150924: Add checks on ALTITUDE/PRESSURE ordering plus ordering of corresponding BOUNDARY
-; datasets (only if it is an axis variable). Also perform checks on variables that
-; should be grouped (e.g. LATITUDE and LONGITUDE) - Version 4.0b30
-; 20151012: Add ALTITUDE.GPH and DEPTH to the vertical variables to be checked. Check for FTIR
-; template version before doing some of the .BOUNDARIES checks - Version 4.0b31
-; 20151109: Allow for the possibility of missing variable attributes (i.e. VAR_NOTES) when
-; checking for variable attribute values based on the attr_arr_data array
-; ordering; Checks that VAR_NOTES label is present if the /AVK option is selected,
-; if not then create log message; Do check on *.BOUNDARIES INDEPENDENT dimension
-; regardless of whether the related vertical variable is an axis variable or not
-; - Version 4.0b34
-; 20170331: Update comment appended to VAR_NOTES to clarify AVK dimension ordering
-; -Version 4.0b42
-; 20171121: Change check identifying FTIR templates with incorrect .BOUNDARIES ordering (FTIR
-; changed to FTIR-0 to avoid conflict with other FTIR based templates); Change conditions
-; for identifying invalid wind speed or direction values - Version 4.0b43
-; 20201020: Add check for negative random uncertainty values (standard or relative only)
-; - Version 4.0b56
-;
-; Inputs: sds - Either a structure containing the Variable Attributes and Data, or the input
-; data file
-; inf - flag identifying SDS type where, 0=structure; 1=data file
-; meta_arr - a string array containing the Global and Variable Attributes
-;
-; Outputs: meta_arr - completes missing or invalid metadata values for attributes with
-; VAR_UNITS=MJD2K, as required, and also adds comments to VAR_NOTES
-; for averaging kernel data to indicate proper array order, if this
-; option is chosen
-; ds - checked datasets are written to the ds structure with correct dimensions and
-; data type
-;
-; Called by: IDLCR8HDF
-;
-; Subroutines Called: EXTRACT_DATA
-; CHECK_MIN_MAX_FILL
-
-COMMON METADATA
-COMMON DATA
-COMMON WIDGET_WIN
-
-;Go through variable names and check for DATETIME, DATETIME.START, or DATETIME.STOP
-;values and determine representative minimum and maximum datetime values
-mint=90000.D & maxt=-90000.D ;initialize min and max datetimes
-valfound=0 ;count variable that increments when DATETIME and related datasets are found
-errorfound=1 ;variable to indicate error in DATETIME checks
-infowrite=0 ;do not write ISO8601 to MJD2K information message in EXTRACT_DATA routine (will write on last EXTRACT_DATA call)
-;set-up holding array to hold number of bytes in string VAR_FILL_VALUE (note already created if input is via session memory)
-IF mv_str[0] EQ -1L THEN mv_str=LONARR(nvn)
-vchk=['VAR_VALID_MIN','VAR_VALID_MAX']
-nvchk=N_ELEMENTS(vchk) & mmi=INTARR(nvchk)
-FOR i=0,nvchk-1 DO BEGIN
- mi=WHERE(attr_arr_data EQ vchk[i]) & mmi[i]=mi[0]
-ENDFOR
-vavk=INTARR(nvn) ;holding array for AVK attribute (used below)
-
-;Determine number of DATETIME values, and set arrays for DATETIME checks
-dtfv=DBLARR(3)-mint
-dti=WHERE(vn EQ 'DATETIME',dtcnt)
-IF dtcnt NE 0 THEN BEGIN
- dtest=['0'] ;initializing dtest array for EXTRACT_DATA call
- EXTRACT_DATA,sds,inf,dti[0],dtest,ndl,infowrite ;return the dataset
- IF STRLEN(rerr[0]) GT 2 THEN RETURN
- dtchk=N_ELEMENTS(dtest)
- dtvals=DBLARR(3,dtchk)-mint
- errorfound=0
-ENDIF ELSE BEGIN
- dti=WHERE(vn EQ 'DATETIME.START',dtcnt)
- dtsi=WHERE(vn EQ 'DATETIME.STOP',dtscnt)
- IF (dtcnt NE 0) AND (dtscnt NE 0) THEN BEGIN
- dtest=['0'] ;initializing dtest array for EXTRACT_DATA call
- EXTRACT_DATA,sds,inf,dti[0],dtest,ndl,infowrite ;return the dataset
- IF STRLEN(rerr[0]) GT 2 THEN RETURN
- dtchk=N_ELEMENTS(dtest)
- dtest=['0'] ;initializing dtest array for EXTRACT_DATA call
- EXTRACT_DATA,sds,inf,dtsi[0],dtest,ndl,infowrite ;return the dataset
- IF STRLEN(rerr[0]) GT 2 THEN RETURN
- IF dtchk EQ N_ELEMENTS(dtest) THEN BEGIN
- dtvals=DBLARR(3,dtchk)-mint
- errorfound=0
- ENDIF
- ENDIF
-ENDELSE
-
-FOR vc=0,nvn-1 DO BEGIN
- vnspl=STRSPLIT(vn[vc],'_',/Extract)
- vnspl=[vnspl,'x','x'] ;ensures vnspl has a minimum of 3 values
- IF (vnspl[1] EQ 'AVK') OR (vnspl[2] EQ 'AVK') THEN vavk[vc]=1 ;identifies a dataset containing AVK data
- IF ((vn[vc] EQ 'DATETIME') OR (vn[vc] EQ 'DATETIME.START') OR $
- (vn[vc] EQ 'DATETIME.STOP')) THEN BEGIN
- dtest=['0'] ;initializing dtest array for EXTRACT_DATA call
- EXTRACT_DATA,sds,inf,vc,dtest,ndl,infowrite ;return the dataset
- IF STRLEN(rerr[0]) GT 2 THEN RETURN
- IF errorfound EQ 0 THEN BEGIN
- dtnel=N_ELEMENTS(dtest)
- CASE 1 OF
- vn[vc] EQ 'DATETIME.START': BEGIN
- IF dtnel EQ dtchk THEN dtvals[0,*]=DOUBLE(dtest) $
- ELSE IF dtnel EQ 1 THEN dtvals[0,0]=DOUBLE(dtest) $
- ELSE errorfound=1
- dtfv[0]=mv_dbl[2,vc]
- END
- vn[vc] EQ 'DATETIME': BEGIN
- dtvals[1,*]=DOUBLE(dtest) & dtfv[1]=mv_dbl[2,vc]
- END
- ELSE: BEGIN
- IF dtnel EQ dtchk THEN dtvals[2,*]=DOUBLE(dtest) $
- ELSE IF dtnel EQ 1 THEN dtvals[2,dtchk-1]=DOUBLE(dtest) $
- ELSE errorfound=1
- dtfv[2]=mv_dbl[2,vc]
- END
- ENDCASE
- ENDIF
- gi=WHERE(DOUBLE(dtest) NE mv_dbl[2,vc],gcnt) ;edit out fill values
- IF gcnt NE 0 THEN BEGIN
- minth=MIN(DOUBLE(dtest[gi]),MAX=maxth)
- IF minth LT mint THEN mint=minth
- IF maxth GT maxt THEN maxt=maxth
- ;IF vn[vc] EQ 'DATETIME.START' THEN mint=minth $
- ;ELSE IF vn[vc] EQ 'DATETIME.STOP' THEN maxt=maxth
- ENDIF
- valfound=valfound+1
- ENDIF
-ENDFOR
-
-IF (errorfound EQ 0) AND (valfound GT 1) THEN BEGIN
- ;Make all missing values equal to fill values
- FOR i=0,2 DO BEGIN
- mi=WHERE(dtvals[i,*] EQ -90000.D,mcnt)
- IF mcnt NE 0 THEN dtvals[i,mi]=dtfv[i]
- ENDFOR
- ;Check that DATETIME.START LE DATETIME LE DATETIME.STOP
- i=0L
- WHILE (errorfound EQ 0) AND (i LE dtchk-1L) DO BEGIN
- test1=(dtvals[0,i] NE dtfv[0]) AND (dtvals[1,i] NE dtfv[1])
- test2=(dtvals[1,i] NE dtfv[1]) AND (dtvals[2,i] NE dtfv[2])
- test3=(dtvals[0,i] NE dtfv[0]) AND (dtvals[2,i] NE dtfv[2])
- IF (test1) AND (dtvals[0,i] GT dtvals[1,i]) THEN errorfound=1
- IF (test2) AND (dtvals[1,i] GT dtvals[2,i]) THEN errorfound=2
- IF (test3) AND (dtvals[0,i] GT dtvals[2,i]) THEN errorfound=3
- i=i+1L
- ENDWHILE
- IF errorfound NE 0 THEN BEGIN
- v0=STRTRIM(STRING(format='(d18.9)',dtvals[0,i-1L]),2)
- v1=STRTRIM(STRING(format='(d18.9)',dtvals[1,i-1L]),2)
- v2=STRTRIM(STRING(format='(d18.9)',dtvals[2,i-1L]),2)
- CASE 1 OF
- errorfound EQ 1: itxt=['DATETIME.START='+v0,'DATETIME='+v1]
- errorfound EQ 2: itxt=['DATETIME='+v1,'DATETIME.STOP='+v2]
- ELSE: itxt=['DATETIME.START='+v0,'DATETIME.STOP='+v2]
- ENDCASE
- infotxt='3 '+itxt[0]+' is greater than '+itxt[1]
- INFOTXT_OUTPUT,infotxt
- ENDIF
-ENDIF
-
-valfound=0 ;Boolean to indicate whether missing VAR_VALID_MIN/MAX values have been added
-;Add minimum and maximum values to Metadata as required
-di=WHERE(STRUPCASE(vu) EQ 'MJD2K',dcnt)
-IF dcnt NE 0 THEN BEGIN
- FOR vc=0,dcnt-1 DO BEGIN
- mi=WHERE(meta_arr EQ 'VAR_NAME='+vn[di[vc]])
- FOR i=0,nvchk-1 DO BEGIN
- IF STRMID(meta_arr[mi[0]+mmi[i]],0,STRLEN(vchk[i])) NE vchk[i] THEN acti=mmi[i]-1L ELSE acti=mmi[i] ;depends if VAR_NOTES is present or not
- res=STRSPLIT(STRTRIM(meta_arr[mi[0]+acti],2),'=',/Extract,COUNT=nres)
- IF nres EQ 1 THEN BEGIN ;no value so add Min or Max value to this attribute
- IF i MOD 2 EQ 0 THEN BEGIN
- IF mint NE 90000.d THEN BEGIN
- meta_arr[mi[0]+acti]=vchk[i]+'='+STRTRIM(STRING(format='(d18.9)',mint),2)
- valfound=1
- ENDIF
- mv_dbl[i,di[vc]]=mint
- ENDIF ELSE BEGIN
- IF maxt NE -90000.d THEN BEGIN
- meta_arr[mi[0]+acti]=vchk[i]+'='+STRTRIM(STRING(format='(d18.9)',maxt),2)
- valfound=1
- ENDIF
- mv_dbl[i,di[vc]]=maxt
- ENDELSE
- IF valfound EQ 1 THEN BEGIN
- infotxt=STRARR(2)
- infotxt[0]='2 Missing '+vchk[i]+' value for '+meta_arr[mi[0]]+'| added based on available'
- infotxt[1]=' DATETIME[.START][.STOP] values: '+meta_arr[mi[0]+acti]
- INFOTXT_OUTPUT,infotxt
- ENDIF
- ENDIF
- ENDFOR
- ENDFOR
-ENDIF
-
-IF o3[1] EQ 'AVK' THEN BEGIN ;append sentence to VAR_NOTES in the AVK attribute
- di=WHERE(vavk EQ 1,dcnt)
- vnc=STRARR(nvn) ;holding array for comment
- IF dcnt NE 0 THEN BEGIN
- FOR vc=0,dcnt-1 DO BEGIN
- mi=WHERE(meta_arr EQ 'VAR_NAME='+vn[di[vc]])
- ;check to see if AVK is a matrix or vector (no need for comment if it us a vector)
- add_comm=0 ;default is for no comment
- xi=WHERE(attr_arr_data EQ 'VAR_DEPEND')
- IF STRMID(meta_arr[mi[0]+xi[0]],0,10) NE 'VAR_DEPEND' THEN acti=xi[0]-1L ELSE acti=xi[0] ;depends if VAR_NOTES is present or not
- res=STRSPLIT(meta_arr[mi[0]+acti],'=; ',/EXTRACT,COUNT=nres)
- res=STRUPCASE(res)
- IF nres GE 3 THEN BEGIN
- FOR i=1,nres-1 DO BEGIN
- ri=WHERE(res[i] EQ res,rcnt) ;i.e. repeated dependencies indicate an AVK matrix
- IF rcnt GE 2 THEN add_comm=1
- ENDFOR
- ENDIF
- IF add_comm THEN BEGIN
- IF acti NE xi[0] THEN BEGIN ;No VAR_NOTES attribute present so generate error
- infotxt='3 VAR_NOTES attribute not present under '+meta_arr[mi[0]]+' so unable to append Averaging Kernel information'
- INFOTXT_OUTPUT,infotxt
- ENDIF ELSE BEGIN
- ;find relevant VAR_NOTES attribute label
- xi=WHERE(attr_arr_data EQ 'VAR_NOTES')
- eqpos=STRPOS(meta_arr[mi[0]+xi[0]],'=')
- vnc[di[vc]]=STRTRIM(STRMID(meta_arr[mi[0]+xi[0]],eqpos+1),2) ;existing VAR_NOTES comment, if any
- ;if comment exists, check to see if it ends with a period, if not add one
- IF vnc[di[vc]] NE '' THEN BEGIN
- IF STRMID(vnc[di[vc]],STRLEN(vnc[di[vc]])-1,1) NE '.' THEN $
- vnc[di[vc]]=vnc[di[vc]]+'. ' ELSE vnc[di[vc]]=vnc[di[vc]]+' '
- ENDIF
- ;determine the number of dimensions
- ni=WHERE(ndm[di[vc],0:7] NE 0,ndim)
- IF ndim GT 2 THEN vncx='for the first measurement are:' ELSE vncx='are:'
- ;append comment
- vnc[di[vc]]=vnc[di[vc]]+'First three values of the initial averaging kernel '+vncx
- ;append meta_arr index to comment
- vnc[di[vc]]=STRTRIM(mi[0]+xi[0],2)+'_'+vnc[di[vc]]
- ENDELSE
- ENDIF
- ENDFOR
- ENDIF
-ENDIF
-
-;If required do checks on the vertical dimension plus correspondng .BOUNDARIES
-;Only do checks if variable is an axis variable - so must be in ascending or descending order
-bchks=['ALTITUDE','PRESSURE','ALTITUDE.GPH','DEPTH']
-sdata=ndm(vc,8) EQ 6
-
-;Do check for DATA_TEMPLATE and do not do all the .BOUNDARIES checks for FTIR 001 or 002 templates
-ti=WHERE(STRMID(meta_arr,0,13) EQ 'DATA_TEMPLATE',tcnt)
-ftirchk=0 ;default is that it is not an FTIR-001 or 002 measurement
-IF tcnt EQ 1 THEN BEGIN
- res=STRSPLIT(meta_arr[ti[0]],'=',/EXTRACT,COUNT=nres)
- IF nres EQ 2 THEN BEGIN
- IF (STRPOS(res[1],'FTIR-0') NE -1) AND ((STRPOS(res[1],'001') NE -1) OR (STRPOS(res[1],'002') NE -1)) THEN ftirchk=1
- ENDIF
-ENDIF
-
-FOR vc=0,N_ELEMENTS(bchks)-1 DO BEGIN
- bci=WHERE(vn EQ bchks[vc],bccnt) & bbci=WHERE(vn EQ bchks[vc]+'.BOUNDARIES',bbccnt)
- IF bccnt EQ 1 THEN BEGIN ;ALTITUDE or PRESSURE present so extract VAR_DEPEND and VAR_SIZE values
- mi=WHERE(meta_arr EQ 'VAR_NAME='+vn[bci[0]])
- xi=WHERE(attr_arr_data EQ 'VAR_DEPEND')
- IF STRMID(meta_arr[mi[0]+xi[0]],0,10) NE 'VAR_DEPEND' THEN acti=xi[0]-1L ELSE acti=xi[0] ;depends if VAR_NOTES is present or not
- res=STRSPLIT(meta_arr[mi[0]+acti],'=; ',/EXTRACT,COUNT=nres)
- vdvals=STRUPCASE(res[1:nres-1])
- di=WHERE(ndm[bci[0],0:7] NE 0L,ndim)
- vsvals=ndm[bci[0],di]
- bi=WHERE(vdvals EQ bchks[vc],bcnt) ;determine if the variable of interest is also self-referencing
- nbi=WHERE(vdvals NE bchks[vc],nbcnt) ;determine if there are any other dependencies e.g. DATETIME
- sdata=ndm[bci[0],8] EQ 6 ;test for string datatype
- writeonce=[0,0] ;initialize array for writing information/error messages
- IF (bcnt EQ 1) AND (ndim LE 2) AND (~sdata) THEN BEGIN ;it is self-referencing and there are 2 or less dependencies so do the checks
- ascending=-1 ;set default value - will change to 0 for descending order and 1 for ascending order
- writeonce=[1,1] ;ensure log text is only written once
- EXTRACT_DATA,sds,inf,bci[0],dtest,ndl,infowrite
- IF STRLEN(rerr[0]) GT 2 THEN RETURN
- bvals=DOUBLE(REFORM(dtest,ndm[bci[0],di])) ;put into array order and make numeric
- IF nbcnt EQ 1 THEN n_sec=ndm[bci[0],nbi[0]] ELSE n_sec=1L ;Number of datasets to be read through
- IF bi[0] NE 0 THEN bvals=TRANSPOSE(bvals) ;make ALTITUDE or PRESSURE array the first index
- FOR i=0L,n_sec-1L DO BEGIN ;e.g. number of DATETIME values or single loop if only one dimension
- bvtest=bvals[*,i] ;set of PRESSURES or ALTITUDES to be tested
- asc=-1
- IF ARRAY_EQUAL(bvtest,bvtest[SORT(bvtest)]) THEN asc=1 $ ;ascending order
- ELSE IF ARRAY_EQUAL(bvtest,bvtest[REVERSE(SORT(bvtest))]) THEN asc=0 ;descending order
- IF ascending EQ -1 THEN ascending=asc
- IF (asc EQ -1) OR (ascending NE asc) THEN BEGIN ;altitudes/pressures not in any order
- IF (ascending NE -1) AND (writeonce[1] EQ 1) THEN BEGIN ;more than one dimension and error in one or more of the sets
- IF ascending EQ 1 THEN itxt='increasing' ELSE itxt='decreasing'
- infotxt='3 '+bchks[vc]+' values not in '+itxt+' order for every '+vdvals[nbi[0]]+' for '+meta_arr[mi[0]]
- INFOTXT_OUTPUT,infotxt
- writeonce[1]=0
- ENDIF ELSE IF writeonce[0] EQ 1 THEN BEGIN
- infotxt='3 '+bchks[vc]+' values not monotonically increasing or decreasing for '+meta_arr[mi[0]]
- INFOTXT_OUTPUT,infotxt
- writeonce[0]=0
- ENDIF
- ENDIF
- ENDFOR
- ENDIF
-
- IF bbccnt EQ 1 THEN BEGIN ;*.BOUNDARIES present so check that INDEPENDENT index has VAR_SIZE=2
- mi=WHERE(meta_arr EQ 'VAR_NAME='+vn[bbci[0]])
- xi=WHERE(attr_arr_data EQ 'VAR_DEPEND')
- IF STRMID(meta_arr[mi[0]+xi[0]],0,10) NE 'VAR_DEPEND' THEN acti=xi[0]-1L ELSE acti=xi[0] ;depends if VAR_NOTES is present or not
- res=STRSPLIT(meta_arr[mi[0]+acti],'=; ',/EXTRACT,COUNT=nres)
- vdvals=STRUPCASE(res[1:nres-1])
- di=WHERE(ndm[bbci[0],0:7] NE 0L,ndim)
- vsvals=ndm[bbci[0],di]
- bi=WHERE(vdvals EQ bchks[vc],bcnt) ;determine index of the variable of interest is also self-referencing
- IF bcnt EQ 1 THEN n_alt=ndm[bbci[0],bi[0]]
- ibi=WHERE(vdvals EQ 'INDEPENDENT',ibcnt)
- IF ibcnt EQ 1 THEN BEGIN
- n_ind=ndm[bbci[0],ibi[0]]
- IF n_ind NE 2 THEN BEGIN
- infotxt='3 INDEPENDENT index must have VAR_SIZE=2 for '+meta_arr[mi[0]]
- INFOTXT_OUTPUT,infotxt
- ENDIF
- ENDIF
- ENDIF
-
- oksofar=(writeonce[0] EQ 1) AND (writeonce[1] EQ 1)
- IF (bbccnt EQ 1) AND (oksofar) THEN BEGIN ;Also need to check that xx.BOUNDARIES is in the same order (ascending or descending)
- ;Note: Only do checks if checks on axis variable were all OK
- nbi=WHERE((vdvals NE bchks[vc]) AND (vdvals NE 'INDEPENDENT'),nbcnt) ;determine if there are any other dependencies e.g. DATETIME
- IF nbcnt EQ 1 THEN n_sec=ndm[bbci[0],nbi[0]] ELSE n_sec=1
- sdata=(ndm[bbci[0],8] EQ 6) ;test for string datatype
- IF (bcnt EQ 1) AND ((ndim EQ 2) OR (ndim EQ 3)) AND (n_ind EQ 2) AND (~sdata) THEN BEGIN
- ;it is dependent on ALTITUDE/PRESSURE and there are 2 or less other dependencies so do the checks
- EXTRACT_DATA,sds,inf,bbci[0],dtest,ndl,infowrite
- IF STRLEN(rerr[0]) GT 2 THEN RETURN
- bvals=DOUBLE(REFORM(dtest,ndm[bbci[0],di])) ;put into array order and make numeric
-
- ;Determine the actual ordering of the array
- ;Option 1 = 2-D e.g. ALTITUDE;INDEPENDENT or INDEPENDENT;ALTITUDE
- ;Option 2 = 3-D with axis variable as first or last index and INDEPENDENT is the middle index e.g. ALT;IND;DAT or DAT;IND;ALT
- ;Option 3 = 3-D with axis variable as first or last index and INDEPENDENT is the first or last index e.g. ALT;DAT;IND or IND;DAT;ALT
- ;Option 4 = 3-D with INDEPENDENT variable as first or last index and ALTITUDE is the middle index e.g. IND;ALT;DAT or DAT;ALT;IND
- IF (bcnt EQ 1) AND (ibcnt EQ 1) AND (ndim EQ 2) THEN BEGIN
- option=1
- IF bi[0] NE 0 THEN bvals=TRANSPOSE(bvals) ;make ALTITUDE or PRESSURE array the first index
- ENDIF ELSE IF (bcnt EQ 1) AND (ibcnt EQ 1) AND (ndim EQ 3) THEN BEGIN
- IF ((bi[0] EQ 0) OR (bi[0] EQ 2)) AND (ibi[0] EQ 1) THEN BEGIN
- option=2
- IF bi[0] EQ 2 THEN bvals=TRANSPOSE(bvals) ;make ALTITUDE or PRESSURE array the first index
- ENDIF ELSE IF ((bi[0] EQ 0) OR (bi[0] EQ 2)) AND (nbi[0] EQ 1) THEN BEGIN
- option=3
- IF bi[0] EQ 2 THEN bvals=TRANSPOSE(bvals) ;make ALTITUDE or PRESSURE array the first index
- ENDIF ELSE BEGIN
- option=4
- IF ibi[0] EQ 2 THEN bvals=TRANSPOSE(bvals) ;make INDEPENDENT array the first index (ALTITUDE or PRESSURE is the 2nd index)
- ENDELSE
- ENDIF ELSE option=0
-
- IF option NE 0 THEN BEGIN
- writeonceb=[1,1]
- FOR i=0L,n_sec-1L DO BEGIN
- FOR j=0,n_ind-1 DO BEGIN ;check that order of the boundary sets matches the axis variable ordering
- CASE option of ;set of PRESSURES or ALTITUDES to be tested
- 1: bvtest=bvals[*,j]
- 2: bvtest=bvals[*,j,i]
- 3: bvtest=bvals[*,i,j]
- 4: bvtest=bvals[j,*,i]
- ENDCASE
- asc=-1
- IF ARRAY_EQUAL(bvtest,bvtest[SORT(bvtest)]) THEN asc=1 $ ;ascending order
- ELSE IF ARRAY_EQUAL(bvtest,bvtest[REVERSE(SORT(bvtest))]) THEN asc=0 ;descending order
-
- IF (ascending NE asc) AND (writeonceb[0] EQ 1) THEN BEGIN
- ;order of the boundaries dataset does not match axis dataset
- IF ascending EQ 1 THEN itxt='increasing' ELSE itxt='decreasing'
- infotxt='3 '+bchks[vc]+'.BOUNDARIES order does not match '+bchks[vc]+' '+itxt+' ordering for '+meta_arr[mi[0]]
- INFOTXT_OUTPUT,infotxt
- writeonceb[0]=0
- ENDIF
- ENDFOR
- ENDFOR
- ;Check that the boundary ordering is also the same as the axis variable for every ALTITUDE/PRESSURE
- FOR i=0L,n_sec-1L DO BEGIN
- FOR j=0L,n_alt-1L DO BEGIN
- CASE option OF ;set of boundary pairs to be tested (INDEPENDENT index)
- 1: bvtest=bvals[j,*]
- 2: bvtest=bvals[j,*,i]
- 3: bvtest=bvals[j,i,*]
- 4: bvtest=bvals[*,j,i]
- ENDCASE
- asc=-1
- IF ARRAY_EQUAL(bvtest,bvtest[SORT(bvtest)]) THEN asc=1 $ ;ascending order
- ELSE IF ARRAY_EQUAL(bvtest,bvtest[REVERSE(SORT(bvtest))]) THEN asc=0 ;descending order
-
- IF (ascending NE asc) AND (writeonceb[1] EQ 1) AND (ftirchk EQ 0) THEN BEGIN
- ;order of the boundaries values does not match axis dataset
- IF ascending EQ 1 THEN itxt='increasing' ELSE itxt='decreasing'
- infotxt='3 Boundary index order does not match '+bchks[vc]+' '+itxt+' ordering for '+meta_arr[mi[0]]
- INFOTXT_OUTPUT,infotxt
- writeonceb[1]=0
- ENDIF
- ENDFOR
- ENDFOR
- ENDIF
- ENDIF
- ENDIF
- ENDIF
-ENDFOR
-
-;Test that random uncertainty values are positive
-posun=['UNCERTAINTY.RANDOM.STANDARD','UNCERTAINTY.RANDOM.STANDARD.RELATIVE']
-n_p=N_ELEMENTS(posun)
-vnx=vn
-;vnx consists of only the last sub-name of vn
-FOR i=0,nvn-1 DO $
- IF STRPOS(vnx[i],'_') NE -1 THEN vnx[i]=STRMID(vnx[i],STRPOS(vnx[i],'_',/REVERSE_SEARCH)+1)
-FOR i=0,n_p-1 DO BEGIN ;number of values to check
- pi=WHERE(vnx EQ posun[i],pcnt)
- IF pcnt NE 0 THEN BEGIN
- FOR j=0,pcnt-1 DO BEGIN ;test for negative data values
- sdata=(ndm[pi[j],8] EQ 6) ;test for string datatype
- IF ~sdata THEN BEGIN
- EXTRACT_DATA,sds,inf,pi[j],dtest,ndl,infowrite
- IF STRLEN(rerr[0]) GT 2 THEN RETURN
-
- CASE ndm[pi[j],8] OF ;identifies the VAR_DATA_TYPE
- 0:BEGIN
- fv=BYTE(mv_lng[2,pi[j]]) & urd=BYTE(dtest) & zero=0
- END
- 1:BEGIN
- fv=FIX(mv_lng[2,pi[j]]) & urd=FIX(dtest) & zero=0
- END
- 2:BEGIN
- fv=LONG(mv_lng[2,pi[j]]) & urd=LONG(dtest) & zero=0L
- END
- 3:BEGIN
- fv=mv_lng[2,pi[j]] & urd=LONG64(dtest) & zero=0LL
- END
- 4:BEGIN
- fv=FLOAT(mv_dbl[2,pi[j]]) & urd=FLOAT(dtest) & zero=0.
- END
- 5:BEGIN
- fv=mv_dbl[2,pi[j]] & urd=DOUBLE(dtest) & zero=0.d
- END
- ENDCASE
- ni=WHERE((urd LT zero) AND (urd NE fv),ncnt)
- IF ncnt NE 0 THEN BEGIN
- infotxt='3 '+vn[pi[j]]+' values cannot be negative'
- INFOTXT_OUTPUT,infotxt
- ENDIF
- ENDIF
- ENDFOR
- ENDIF
-ENDFOR
-
-;Test 'grouped' datasets, i.e. datasets that must both be in the file if one of them is present
-;Note that there might be more than one variable in a particular group e.g. LATITUDE and LATITUDE.INSTRUMENT
-grouped=[['LATITUDE','LONGITUDE'],['WIND.DIRECTION','WIND.SPEED']]
-g_dim=SIZE(grouped,/DIMENSIONS)
-FOR i=0,g_dim(1)-1 DO BEGIN ;number of grouped pairs
- p1i=WHERE(STRPOS(STRMID(vn,0,STRLEN(grouped[0,i])),grouped[0,i]) NE -1,p1cnt)
- p2i=WHERE(STRPOS(STRMID(vn,0,STRLEN(grouped[1,i])),grouped[1,i]) NE -1,p2cnt)
- IF p1cnt NE 0 THEN BEGIN
- FOR j=0,p1cnt-1 DO BEGIN ;test all variations
- extrapart=STRMID(vn[p1i[j]],STRLEN(grouped[0,i])) & extrapart=STRTRIM(extrapart,2)
- mi=WHERE(vn EQ grouped[1,i]+extrapart,mcnt)
- IF mcnt EQ 0 THEN BEGIN ;matching variable name not found
- infotxt='3 VAR_NAME='+grouped[1,i]+extrapart+' must also be present together with VAR_NAME='+vn[p1i[j]]
- INFOTXT_OUTPUT,infotxt
- ENDIF ELSE BEGIN
- ;Do tests on grouped values
- ;1. VAR_SIZE values must be the same. If this is the case then for wind direction and wind speed,
- ;2. If wind direction EQ 0 then corresponding wind speed must be 0
- ;3. If wind direction NE 0 then corresponding wind speed can't be 0
- IF ~ARRAY_EQUAL(ndm[p1i[j],0:7],ndm[mi[0],0:7]) THEN BEGIN
- infotxt='3 VAR_SIZE values for VAR_NAME='+vn[p1i[j]]+' must be the same as VAR_SIZE values for VAR_NAME='+vn[mi[0]]
- INFOTXT_OUTPUT,infotxt
- ENDIF ELSE BEGIN ;arrays match up so do wind direction/speed checks
- sdata=(ndm[p1i[j],8] EQ 6) OR (ndm[mi[0],8] EQ 6) ;test for string datatype
- IF (grouped[0,i] EQ 'WIND.DIRECTION') AND (~sdata) THEN BEGIN
- EXTRACT_DATA,sds,inf,p1i[j],dtest,ndl,infowrite
- IF STRLEN(rerr[0]) GT 2 THEN RETURN
- wdd=DOUBLE(dtest)
- EXTRACT_DATA,sds,inf,mi[0],dtest,ndl,infowrite
- IF STRLEN(rerr[0]) GT 2 THEN RETURN
- wsd=DOUBLE(dtest)
- ni=WHERE(wdd EQ 0.d,ncnt)
- IF ncnt NE 0 THEN BEGIN
- si=WHERE(wsd[ni] GT 0.d,scnt)
- IF scnt NE 0 THEN BEGIN ;Wind direction = 0.0 but wind speed NE 0.0
- infotxt='3 '+vn[mi[0]]+' values must be 0.0 for corresponding '+vn[p1i[j]]+' 0.0 values'
- INFOTXT_OUTPUT,infotxt
- ENDIF
- ENDIF
- ni=WHERE((wdd GT 0.d) AND (wdd LE 360.d),ncnt)
- IF ncnt NE 0 THEN BEGIN
- si=WHERE(wsd[ni] EQ 0.d,scnt)
- IF scnt NE 0 THEN BEGIN ;Wind direction NE 0.0 but wind speed = 0.0
- infotxt='3 '+vn[p1i[j]]+' values must be 0.0 for corresponding '+vn[mi[0]]+' 0.0 values'
- INFOTXT_OUTPUT,infotxt
- ENDIF
- ENDIF
- ENDIF
- ENDELSE
- ENDELSE
- ENDFOR
- ENDIF
- IF p2cnt NE 0 THEN BEGIN ;now do the reverse check
- FOR j=0,p2cnt-1 DO BEGIN ;test all variations
- extrapart=STRMID(vn[p2i[j]],STRLEN(grouped[1,i])) & extrapart=STRTRIM(extrapart,2)
- mi=WHERE(vn EQ grouped[0,i]+extrapart,mcnt)
- IF mcnt EQ 0 THEN BEGIN ;matching variable name not found
- infotxt='3 VAR_NAME='+grouped[0,i]+extrapart+' must also be present together with VAR_NAME='+vn[p2i[j]]
- INFOTXT_OUTPUT,infotxt
- ENDIF
- ENDFOR
- ENDIF
-ENDFOR
-
-FOR vc=0,nvn-1 DO BEGIN
- dtest=['0'] ;initializing dtest array for EXTRACT_DATA call
- infowrite=1 ;write information message in EXTRACT_DATA routine if required
- ;Extract the dataset
- EXTRACT_DATA,sds,inf,vc,dtest,ndl,infowrite
- IF STRLEN(rerr[0]) GT 2 THEN RETURN
- ;perform checks on the attribute and data values
- IF ndm[vc,8] EQ 6 THEN CHECK_STRING_DATATYPE,vc,dtest $ ;Test for String Datatype
- ELSE BEGIN
- CHECK_MIN_MAX_FILL,vc,dtest,ndl ;Test for Numeric Datatype
- IF STRLEN(rerr[0]) GT 2 THEN RETURN
- ENDELSE
-
- ;if AVK dataset extracted then add comment to VAR_NOTES if required
- IF o3[1] EQ 'AVK' THEN BEGIN
- IF (vnc[vc] NE '') AND (ndm[vc,0] GE 3) THEN BEGIN
- res=STRSPLIT(vnc[vc],'_',/Extract) & mi=FIX(res[0])
- meta_arr[mi]='VAR_NOTES='+res[1]
- FOR i=0,2 DO meta_arr[mi]=meta_arr[mi]+' '+STRTRIM(dtest[i],2)
- ENDIF
- ENDIF
-
- ;Copy data to the data structure with correct dimensions and data type
- arrchk=SIZE(dtest) ;put it into array form if it is a scalar
- IF arrchk[0] EQ 0 THEN dtest=[dtest]
- gi=WHERE(ndm[vc,0:7] NE 0)
- vs=TRANSPOSE(ndm[vc,gi])
- dtest=REFORM(dtest,vs,/Overwrite)
- ds[vc].data=PTR_NEW(dtest)
-ENDFOR
-
-END ;Procedure Read_Data
-
-
-
-PRO find_hdf_filename, hdffilename
-;Procedure to do the following:
-;1. Compare the DATA_START_DATE and DATA_STOP_DATE with the first and last entries under
-; DATETIME. If necessary create/change DATA_START_DATE and DATA_STOP_DATE to match the
-; DATETIME entries, and put in ISO8601 format.
-;2. Compare filename created by the program from the DATASET_ATTRIBUTES with that under the
-; FILE_NAME entry (if present). If necessary create/change the FILE_NAME entry to match the
-; created filename
-;3. Create/change the FILE_GENERATION_DATE and ensure it is in ISO8601 format.
-; ----------
-;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
-;
-; History:
-; 20050802: Original IDLCR8HDF routine - Version 1.0
-; 20050909: Remove requirement for a 'v' to be at the start of the DATA_FILE_VERSION
-; (e.g. v1.0 or 1.0 acceptable); Change the format that DATA_START_DATE and
-; FILE_GENERATION_DATE values are saved in the HDF file from MJD2000 to
-; ISO8601 - Version 1.1
-; 20051107: Move VIS_SCALE_MIN/MAX checks, for correct format of DATETIME values, to the
-; CHECK_MIN_MAX_FILL routine - Version 1.11
-; 20061012: Move the FILE_GENERATION_DATE checks to the main loop which determines the file
-; name to facilitate the future option of including the file generation date in the
-; file name (this option is not currently implemented by the AVDC); Common variable
-; definition WIDGET_WIN added - Version 2.0
-; 20080302: Do search for either DATA_LEVEL or DATA_TYPE, depending on whether an AVDC style
-; TAV file or original Envisat table.dat file has been read in; Common variable
-; definition TABLEDATA added (tab_type used to differentiate between the databases)
-; - Version 3.0
-; 20090610: Allow for the Global Variable DATA_STOP_DATE and calculate the value if required
-; - Version 3.07
-; 20100205: Add RETURN command after all STOP_WITH_ERROR calls, which allows program to return to the
-; calling program if the reterr argument is included in the idlcr8hdf call - Version 3.09
-; 20101120: Account for new GEOMS changes to filename composition - Version 4.0
-; 20151104: Check that ISO8601 format time is all uppercase - Version 4.0b33
-; 20151109: Check for seconds value of 60 in ISO8601 datetime value - Version 4.0b34
-; 20151215: Fix bug if DATE_TIME_START or DATE_TIME_STOP value is missing - Version 4.0b36
-; 20161130: Allow the program to continue running if an ISO8601 error is found (previously
-; called STOP_WITH_ERROR) - Version 4.0b40
-; 20171130: Remove ds from STOP_WITH_ERROR parameters; Change some error and information text
-; statements - Version 4.0b44
-; 20180314: Fix bug when checking DATA_START|STOP_DATE values. Criteria for difference with
-; MJD2K values change to GT 0.99d/86400.d instead of GE 1.0d/86400.d -
-; Version 4.0b46
-; 20190807: Allow .nc and .nc4 filename extensions when doing QA on a file that has been read
-; into session memory using the HDF5 routines (idlcr8ascii set up to use H5 routines
-; to read netCDF4 files as well as HDF5) - Version 4.0b51
-; 20190824: Fix bug in 4.0b51 that caused the file name to be written to the DATA_FILE_VERSION
-; attribute - Version 4.0b52
-; 20231218: Always recalculate FILE_GENERATION_DATE if creating a GEOMS file so that it aligns
-; with the FILE_META_VERSION - Version 4.0b61
-;
-; Inputs: meta_arr - a string array containing the Global and Variable Attributes
-; hfdfilename - a string holding the extension of the eventual HDF filename, either
-; '.hdf' for HDF4 or '.h5' for HDF5 or '.nc' for netCDF3/4
-;
-; Outputs: meta_arr - DATA_START_DATE, FILE_GENERATION_DATE and FILE_NAME values will be
-; (re)written to the array based on values computed by the routine
-; hdffilename - a string holding the full HDF file name determined by the routine
-;
-; Called by: IDLCR8HDF
-;
-; Subroutines Called: STOP_WITH_ERROR (if error state detected); INFOTXT_OUTPUT
-; Possible Conditions for STOP_WITH_ERROR call (plus [line number] where called):
-; 1. An incorrect number of sub-values found in meta_arr for DATA_DISCIPLINE,
-; DATA_SOURCE, DATA_LOCATION, or DATA_FILE_VERSION [3129]
-; 2. Error encountered when converting ISO8601 datetime values to MJD2000 format [3138,3179]
-; 3. Type conversion error encountered when a datetime value is expected to be in
-; MJD2000 format, but is not numeric [3216]
-;
-; Information Conditions (when the program is able to make changes):
-; 1. DATA_START_DATE or DATA_STOP_DATE changed to match relevant DATETIME entries [3167]
-; 2. Filename determined from DATASETS attributes [3208]
-
-COMMON TABLEDATA
-COMMON METADATA
-COMMON DATA
-COMMON WIDGET_WIN
-
-;Possible error messages for this procedure
-ON_IOERROR,TypeConversionError
-errtxt=STRARR(4) & lu=-1L
-proname='Find_HDF_Filename procedure: '
-errtxt[0]='Number of sub-values under DATA_'
-errtxt[1]=' should be '
-errtxt[2]=' ISO8601 format not valid: '
-errtxt[3]='Type conversion error. MJD2K entry is not valid: '
-
-fni=WHERE(attr_arr_glob EQ 'FILE_NAME')
-res=STRSPLIT(meta_arr[fni[0]],' =',/Extract)
-IF N_ELEMENTS(res) EQ 1 THEN res=[res,'-1']
-fn_val=res[1]
-
-IF (qa_yes) AND (hdffilename EQ '.h5') AND (fn_val NE '-1') THEN BEGIN
- ;netCDF4 files read in using H5 routines so .h5 extension is assumed. Allow files to
- ;also have .nc or .nc4 extensions for files being QA'd
- hext=['.h5','.nc','.nc4']
- fn_ext=STRMID(fn_val,STRPOS(fn_val,'.',/REVERSE_SEARCH))
- ei=WHERE(fn_ext EQ hext,ecnt)
- IF ecnt NE 0 THEN hdffilename=hext[ei[0]]
-ENDIF
-
-fhold='' & iso='' & mjd2000=0.d
-dtmjd=0.d & dtiso='' & itxt=' '
-dt=['DISCIPLINE','SOURCE','LOCATION','START_DATE','STOP_DATE','FILE_GENERATION_DATE','FILE_VERSION']
-valid=0 ;test for Type conversion errors
-FOR i=0,N_ELEMENTS(dt)-1 DO BEGIN
- IF i EQ 0 THEN nei=4 ELSE nei=2
- IF i EQ 5 THEN ai=WHERE(attr_arr_glob EQ dt[i]) $
- ELSE ai=WHERE(attr_arr_glob EQ 'DATA_'+dt[i])
-
- res=STRSPLIT(meta_arr[ai[0]],' =;',/Extract,COUNT=cres)
- IF (cres NE nei) AND ((i LE 2) OR (i GE 6)) THEN BEGIN
- ;infotxt='3 '+errtxt[0]+dt[i]+errtxt[1]+nes
- ;INFOTXT_OUTPUT,infotxt
- STOP_WITH_ERROR,o3[3]+proname+meta_arr[ai[0]]+': ',errtxt[0]+dt[i]+errtxt[1]+STRTRIM(nei-1,2),lu
- RETURN
- ENDIF
-
- IF i EQ 6 THEN BEGIN ;Do DATA_FILE_VERSION checks
- dfvv=res[1]
- GEOMS_RULE_CHANGES,4,dfvv
- res[1]=dfvv & meta_arr[ai[0]]='DATA_FILE_VERSION='+dfvv
- ENDIF
-
- IF (i EQ 3) OR (i EQ 4) THEN BEGIN ;calculate the ISO or MJD2000 value as required
- ;Find lowest or highest DATETIME values from Variable Attributes
- IF i EQ 3 THEN BEGIN
- di=WHERE(vn EQ 'DATETIME.START',dcnt)
- itxt= ' first ' & etxt='DATETIME.START '
- ENDIF ELSE BEGIN
- di=WHERE(vn EQ 'DATETIME.STOP',dcnt)
- itxt= ' last ' & etxt='DATETIME.STOP '
- ENDELSE
- IF dcnt NE 0 THEN BEGIN
- darr=*ds[di[0]].data ;extract the DATETIME values
- ;remove any fill values
- fi=WHERE(darr NE mv_dbl[2,di[0]],dcnt)
- ENDIF
- IF dcnt EQ 0 THEN BEGIN
- di=WHERE(vn EQ 'DATETIME',dcnt) & etxt='DATETIME '
- IF dcnt NE 0 THEN BEGIN
- darr=*ds[di[0]].data ;extract the DATETIME values
- ;remove any fill values
- fi=WHERE(darr NE mv_dbl[2,di[0]],dcnt)
- ENDIF
- ENDIF
- IF dcnt NE 0 THEN BEGIN
- darr=darr[fi]
- IF N_ELEMENTS(darr) EQ 1 THEN itxt=' '
- IF i EQ 3 THEN dtmjd=MIN(darr) ELSE dtmjd=MAX(darr)
- dtiso=JDF_2_DATETIME(dtmjd,/M,/S) ;returns datetime in ISO8601 format
- dtisochk=0B ;boolean to check for 60 as seconds value
- ;Extract DATA_START/STOP_DATE from the Global Attributes, if present
- IF N_ELEMENTS(res) EQ nei THEN BEGIN ;a date/time value is present
- IF STRPOS(STRUPCASE(res[1]),'Z') NE -1 THEN BEGIN
- IF (STRPOS(res[1],'Z') EQ -1) OR (STRPOS(res[1],'T') EQ -1) THEN BEGIN ;ISO8601 string must be in upper case
- IF qa_yes then qtxt=' must be ' ELSE qtxt=' made '
- infotxt='2 '+res[0]+' entry'+qtxt+'uppercase for ISO8601 compliance'
- INFOTXT_OUTPUT,infotxt
- ENDIF
- res[1]=STRUPCASE(res[1])
- mjd2000=JULIAN_DATE(res[1],/I,/M) ;return ISO8601 time in MJD2000 format
- IF mjd2000 EQ -99999.d THEN BEGIN ;conversion error
- infotxt='3 '+res[0]+errtxt[2]+res[1]
- INFOTXT_OUTPUT,infotxt
- ;STOP_WITH_ERROR,o3[3]+proname,res[0]+errtxt[2]+res[1],lu,ds & RETURN
- ENDIF
- dtisochk=STRMID(res[1],13,2) EQ '60' ;existing time in ISO8601 format
- ENDIF ELSE BEGIN
- mjd2000=DOUBLE(res[1])
- iso=JDF_2_DATETIME(mjd2000,/M,/S) ;return MJD2000 time in ISO8601 format
- res[1]=iso
- IF qa_yes THEN qtxt=' is not in ' ELSE qtxt= ' converted to '
- infotxt='2 '+meta_arr[ai[0]]+qtxt+'ISO8601 format'
- INFOTXT_OUTPUT,infotxt
- ENDELSE
- res1=res[1]
- ENDIF ELSE BEGIN
- mjd2000=-99999.0D & res1='-99999.0'
- ENDELSE
- ;Check that DATA_START_DATE matches the first value under DATETIME.START or DATETIME
- ;and DATA_STOP_DATE matches the last value under DATETIME.STOP or DATETIME (if present)
- IF (ABS(mjd2000-dtmjd) GT 0.99D/86400.D) OR ((dtisochk) AND (dtiso NE res1)) THEN BEGIN
- IF N_ELEMENTS(res) EQ nei THEN BEGIN
- IF qa_yes THEN qtxt=' must match' ELSE qtxt=' changed to match'
- infotxt='2 Date in DATA_'+dt[i]+qtxt+itxt+etxt+'entry: '+res[1]+' -> '+dtiso
- ENDIF ELSE BEGIN
- IF qa_yes THEN qtxt='should be '+dtiso ELSE qtxt=dtiso+' added'
- infotxt='2 Missing DATA_'+dt[i]+' value '+qtxt+' based on'+itxt+etxt+'entry'
- ENDELSE
- INFOTXT_OUTPUT, infotxt
- res=[res[0],dtiso] ;new ISO time for filename
- ENDIF
- meta_arr[ai[0]]=res[0]+'='+res[1] ;rewrite (correct) value in meta_arr in ISO8601 format
- ENDIF ELSE IF N_ELEMENTS(res) EQ 1 THEN res=[res,'[MISSING]']
- ENDIF
- IF i EQ 5 THEN BEGIN
- ;check format of FILE_GENERATION_DATE value and create/change if necessary
- IF ~qa_yes THEN BEGIN
- ;Not doing QA so calculate new FILE_GENERATION_DATE of file creation so it aligns with FILE_META_VERSION (added 20231218)
- mjd2000=SYSTIME(/Julian,/UTC)-2451544.5D
- mjds=JDF_2_DATETIME(mjd2000,/M,/S) ;returns datetime in ISO8601 format
- meta_arr[ai[0]]=res[0]+'='+mjds ;write date in ISO8601 format
- infotxt='0 '+res[0]+' updated with current system time'
- INFOTXT_OUTPUT,infotxt
- IF cres EQ 1 THEN res=[res,mjds] ELSE res[1]=mjds
- ENDIF ELSE BEGIN
- IF N_ELEMENTS(res) EQ 1 THEN res=[res,'-1']
- IF STRPOS(STRUPCASE(res[1]),'Z') NE -1 THEN BEGIN
- IF (STRPOS(res[1],'Z') EQ -1) OR (STRPOS(res[1],'T') EQ -1) THEN BEGIN ;ISO8601 string must be in upper case
- IF qa_yes THEN qtxt=' must be ' ELSE qtxt=' made '
- infotxt='2 '+res[0]+' entry'+qtxt+'uppercase for ISO8601 compliance'
- INFOTXT_OUTPUT,infotxt
- ENDIF
- mjd2000=JULIAN_DATE(res[1],/I,/M) ;return time in MJD2000 format
- IF mjd2000 EQ -99999.d THEN res=[res[0],'-2'] $ ;conversion error
- ELSE IF STRMID(res[1],13,2) EQ '60' THEN res=[res[0],'-3'] $
- ELSE meta_arr[ai[0]]=res[0]+'='+STRTRIM(STRUPCASE(res[1]),2) ;rewrite date in ISO8601 format
- ENDIF
- IF STRPOS(STRUPCASE(res[1]),'Z') EQ -1 THEN BEGIN
- ;FILE_GENERATION_DATE not present or not in ISO8601 format so recalculate
- mjd2000=SYSTIME(/Julian,/UTC)-2451544.5D
- mjds=JDF_2_DATETIME(mjd2000,/M,/S) ;returns datetime in ISO8601 format
- meta_arr[ai[0]]=res[0]+'='+mjds ;write date in ISO8601 format
- IF res[1] EQ '-1' THEN itxt='not present' $
- ELSE IF res[1] EQ '-3' THEN itxt='must have seconds value between 0 and 59' $
- ELSE itxt='not in ISO8601 format'
- infotxt='2 '+res[0]+' value '+itxt+'|. Recalculated using the system time'
- INFOTXT_OUTPUT,infotxt
- res[1]=mjds
- ENDIF
- ENDELSE
- ;Test to see if DATA_STOP_DATE is greater than FILE_GENERATION_DATE
- IF dtmjd GT mjd2000 THEN BEGIN
- infotxt='3 Calculated DATA_STOP_DATE='+dtiso+' is later than '+meta_arr[ai[0]]
- INFOTXT_OUTPUT,infotxt
- ENDIF
- ENDIF
- IF i NE 5 THEN BEGIN
- ;i.e. don't include file generation date in the filename
- IF fhold EQ '' THEN fhold=fhold+STRLOWCASE(res[nei-1]) $
- ELSE fhold=fhold+'_'+STRLOWCASE(res[nei-1])
- ENDIF
-ENDFOR
-hdffilename=fhold+hdffilename
-
-;check to see if the resulting filename is the same as that under FILE_NAME
-IF hdffilename NE fn_val THEN BEGIN
- IF fn_val NE '-1' THEN BEGIN
- infotxt=STRARR(3)
- infotxt[0]='2 FILE_NAME entry under File Attributes does not match the filename'
- infotxt[0]=infotxt[0]+' determined from the Global Attributes'
- infotxt[1]=' '+fn_val+' -> '+hdffilename+'|'
- infotxt[2]=' Filename determined from Global Attributes used'
- ENDIF ELSE BEGIN
- IF qa_yes THEN qtxt='should be '+hdffilename ELSE qtxt=hdffilename+' added'
- infotxt=STRARR(2)
- infotxt[0]='2 Missing FILE_NAME value '+qtxt
- infotxt[1]=' based on Global Attribute values'
- ENDELSE
- INFOTXT_OUTPUT, infotxt
- meta_arr[fni[0]]='FILE_NAME='+hdffilename ;change FILE_NAME entry
-ENDIF
-valid=1 ;Type conversions performed OK
-
-TypeConversionError:
-IF valid EQ 0 THEN BEGIN
- infotxt='3 '+errtxt[3]+res[1]
- INFOTXT_OUTPUT, infotxt
- ;STOP_WITH_ERROR,o3[3]+proname+res[0]+': ',errtxt[3]+res[1],lu & RETURN
-ENDIF
-
-END ;Procedure Find_HDF_Filename
-
-
-
-PRO makean, hdf_fn, obj_id, obj_type, label, value
-;Procedure to read the contents of a file given as an attribute value and write
-;it directly to the HDF Scientific Dataset or to the annotation file description
-;section (depending on its size). Currently set up to always write the contents
-;of the file to HDF_SD as the text file size limit is also 4096 bytes.
-; ----------
-;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
-;
-; History:
-; 20050802: Original IDLCR8HDF routine - Version 1.0
-;
-; Inputs: hdf_fn - The file handle identifier for the HDF file
-; obj_id - The Scientific Dataset (SD) identifier of the HDF file or a newly
-; created dataset
-; obj_type - a string identifying the type of annotation (set to 'FILE')
-; label - a string containing the Attribute label from the metadata
-; value - a string containing the Attribute value from the metadata
-;
-; Output: Nil
-;
-; Called by: AVDC_HDF_WRITE
-;
-; Subroutines Called: None
-
-nchar=4096 ;strlen maximum for writing directly to SD (instead of AN)
-
-;Extract name of file
-pos1=STRPOS(value,'"')
-pos2=STRPOS(value,'"',/REVERSE_SEARCH)
-data_notes_file=STRMID(value,pos1+1,pos2-pos1-1)
-OPENR,lu,data_notes_file,/GET_LUN
-
-;read each line and count number of characters in file
-dum='' & text_in_file=''
-i=0
-WHILE NOT EOF(lu) DO BEGIN
- READF,lu,dum
- text_in_file=text_in_file+dum+STRING(10B) ;+STRING(13B)
- i=i+1
-ENDWHILE
-FREE_LUN,lu
-n1=STRLEN(text_in_file)
-
-;The text-buffer contains n1 characters which will be written either
-;to the annotation file description section or directly to the HDF file as
-;the value of the current attribute
-IF n1 LE nchar THEN HDF_SD_ATTRSET,obj_id,label,text_in_file,n1,/DFNT_CHAR $
-ELSE BEGIN
- ;open the HDF output file for AN write operations
- ;Initialize the HDF AN interface for the specified file
- an_id=HDF_AN_START(hdf_fn)
-
- IF obj_type EQ 'FILE' THEN BEGIN
- ;Get id for the file label annotation
- an_labl_id=HDF_AN_CREATEF(an_id,2)
- ;Get id for the file description annotation
- an_desc_id=HDF_AN_CREATEF(an_id,3)
- ENDIF ELSE BEGIN
- ;Get id for the data object label annotation
- an_labl_id=HDF_AN_CREATE(an_id,DFTAG_NDG,HDF_SD_IDTOREF(obj_id),0)
- ;Get id for the data object description annoatation
- an_desc_id=HDF_AN_CREATE(an_id,DFTAG_NDG,HDF_SD_IDTOREF(obj_id),1)
- ENDELSE
-
- ;Write the label annotation
- status=HDF_AN_WRITEANN(an_labl_id,label)
- ;Terminate access to the label annotation
- HDF_AN_ENDACCESS,an_labl_id
-
- ;Write the description annotation
- status=HDF_AN_WRITEANN(an_desc_id,text_in_file)
- ;Terminate access to the description annotation
- HDF_AN_ENDACCESS,an_desc_id
-
- ;Terminate access to the AN interface
- HDF_AN_END,an_id
-ENDELSE
-
-END ;Procedure MakeAN
-
-
-
-PRO avdc_hdf5_write, hdffilename, natts, av, geoms_output
-;IDL subroutine to write AVDC-type global attributes and datasets to the Root
-;Group in an HDF5 file.
-;##########################################################################
-;Note: In versions of IDL earlier than 6.2, this procedure may not compile
-;and the HDF5 write option will not be available.
-;##########################################################################
-; ----------
-;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
-;
-; History:
-; 20061012: Introduced to IDLCR8HDF - Version 2.0
-; 20080302: Removed portion of the code which created the DIMENSIONLIST attribute
-; containing the object reference pointers to the variables referred to
-; in the VAR_DEPEND values - Version 3.0
-; 20150127: For string datasets, change the VAR_FILL_VALUE to an empty string that
-; is the same length as the individual strings - Version 4.0b25
-; 20150217: Change rule from 4.0b25 so the VAR_FILL_VALUE is an empty string; For
-; string datasets do a STRTRIM of the dataset when writing to the file, as
-; the first dataset entry may have had whitespace added so that the
-; maximum dataset length is correctly determined - Version 4.0b26
-; 20181221: Allow compression and shuffle options when creating HDF5 files
-; - Version 4.0b49
-;
-; Inputs: hdffilename - a string holding the name of the HDF file being created
-; and written to
-; natts - an integer giving the number of Variable Attributes in each
-; dataset (i.e. the number of elements in attr_arr_data)
-; av - a string array of size (nvn,natts+7) containing the attribute and
-; NCSA values for each dataset
-; ds - structure containing the data
-; meta_arr - a string array containing the Global and Variable Attributes
-; vn - a string array containing the VAR_NAME values
-; ndm - a long array describing the dimensions of each dataset and the
-; VAR_DATA_TYPE
-;
-; Outputs: An HDF5 file with the name hdffilename
-;
-; Called by: AVDC_HDF_WRITE
-;
-; Subroutines Called: None
-
-COMMON METADATA
-COMMON DATA
-
-;Check for compression setting
-comp_fac=0
-IF STRLEN(geoms_output) EQ 4 THEN BEGIN
- val_chk=STRMID(geoms_output,3,1)
- IF IS_A_NUMBER_HDF(val_chk) THEN comp_fac=FIX(val_chk)
-ENDIF
-
-;Create an HDF5 file
-hdf_file_id=H5F_CREATE(hdffilename)
-H5F_CLOSE, hdf_file_id
-
-;open the HDF5 file for writing.
-hdf_file_id=H5F_OPEN(hdffilename,/WRITE)
-;The H5G_OPEN function opens an existing group within an HDF5 file
-sd_id=H5G_OPEN(hdf_file_id,'/')
-
-;Write the Global Attributes to file
-;Section 4.1 (Bojkov et al., 2002): Originator attributes
-;Section 4.2 (Bojkov et al., 2002): Dataset attributes
-;Section 4.3 (Bojkov et al., 2002): File attributes
-FOR i=0,N_ELEMENTS(attr_arr_glob)-1 DO BEGIN
- ;separate out the Meta Attribute entry
- eqpos=STRPOS(meta_arr[i],'=')
- IF eqpos EQ STRLEN(meta_arr[i])-1 THEN attribute_value=' ' $
- ELSE attribute_value=STRMID(meta_arr[i],eqpos+1)
- ;get attribute type and space, needed to create the attribute
- atype_id=H5T_IDL_CREATE(attribute_value)
- aspace_id=H5S_CREATE_SCALAR()
- ;create attribute in the output file
- aset_id=H5A_CREATE(sd_id,attr_arr_glob[i],atype_id,aspace_id)
- ;write attribute to dataset
- H5A_WRITE,aset_id,attribute_value
- ;close all open identifiers
- H5A_CLOSE,aset_id
- H5S_CLOSE,aspace_id
- H5T_CLOSE,atype_id
-ENDFOR
-
-;Write the Datasets (SDS) to file
-;Section 5.1 (Bojkov et al, 2002): Variable description attributes.
-al=STRARR(natts)
-al[0:natts-1]=attr_arr_data
-FOR i=0,nvn-1 DO BEGIN
- CASE ndm[i,8] OF ;identifies the VAR_DATA_TYPE
- 0:BEGIN
- valid_range=[BYTE(mv_lng[0,i]),BYTE(mv_lng[1,i])]
- fill_value=BYTE(mv_lng[2,i])
- END
- 1:BEGIN
- valid_range=[FIX(mv_lng[0,i]),FIX(mv_lng[1,i])]
- fill_value=FIX(mv_lng[2,i])
- END
- 2:BEGIN
- valid_range=[LONG(mv_lng[0,i]),LONG(mv_lng[1,i])]
- fill_value=LONG(mv_lng[2,i])
- END
- 3:BEGIN
- valid_range=[mv_lng[0,i],mv_lng[1,i]]
- fill_value=mv_lng[2,i]
- END
- 4:BEGIN
- valid_range=[FLOAT(mv_dbl[0,i]),FLOAT(mv_dbl[1,i])]
- fill_value=FLOAT(mv_dbl[2,i])
- END
- 5:BEGIN
- valid_range=[mv_dbl[0,i],mv_dbl[1,i]]
- fill_value=mv_dbl[2,i]
- END
- 6:BEGIN
- valid_range=[' ',' ']
- fill_value=' ' ;STRING(format='(A-'+STRTRIM(mv_str[i],2)+')','') ;bytarr(mv_str[i])
- END
- ENDCASE
-
- ;Make up data array for writing to HDF5 file
- data=*ds[i].data
- ;may have lost array VAR_SIZE information if the last value is '1' e.g. 41,1
- gi=WHERE(ndm[i,0:7] NE 0L) & vs=TRANSPOSE(ndm[i,gi]) ;transpose to get array values in a vector
- ;get dataset type and space, needed to create the dataset
- dtype_id=H5T_IDL_CREATE(data)
- dspace_id=H5S_CREATE_SIMPLE(vs)
- ;create dataset in the output file
- IF comp_fac EQ 0 THEN dset_id=H5D_CREATE(sd_id,vn[i],dtype_id,dspace_id) $
- ELSE BEGIN
- ;Perform compression and shuffling of the dataset
- dset_id=H5D_CREATE(sd_id,vn[i],dtype_id,dspace_id,CHUNK_DIMENSIONS=vs,GZIP=comp_fac,/SHUFFLE)
- ENDELSE
- ;write data to dataset - for string datasets remove unwanted white space
- IF ndm[i,8] EQ 6 THEN H5D_WRITE,dset_id,STRTRIM(data,2) ELSE H5D_WRITE,dset_id,data
-
- ;write out variable attributes
- ninc=0 & j=0
- mi=WHERE(meta_arr EQ attr_arr_data[0]+'='+vn[i])
- WHILE j LE natts-1 DO BEGIN
- ma=mi[0]+ninc
- ;Extract Attribute Label
- eqpos=STRPOS(meta_arr[ma],'=')
- res=STRMID(meta_arr[ma],0,eqpos)
- WHILE res[0] NE al[j] DO j=j+1 ;in case of missing optional variable attributes
- CASE 1 OF
- ;attr_arr_data[j] EQ 'VAR_SIZE': athold=vs
- attr_arr_data[j] EQ 'VAR_VALID_MIN': athold=valid_range[0]
- attr_arr_data[j] EQ 'VAR_VALID_MAX': athold=valid_range[1]
- attr_arr_data[j] EQ 'VAR_FILL_VALUE': athold=fill_value
- ELSE: athold=av[i,j]
- ENDCASE
- ;get attribute type and space, needed to create the attribute
- atype_id=H5T_IDL_CREATE(athold)
- IF N_ELEMENTS(athold) EQ 1 THEN aspace_id=H5S_CREATE_SCALAR() $
- ELSE aspace_id=H5S_CREATE_SIMPLE(N_ELEMENTS(athold))
- ;create attribute in the output file
- aset_id=H5A_CREATE(dset_id,al[j],atype_id,aspace_id)
- ;write attribute to dataset
- H5A_WRITE,aset_id,athold
- ;close all open attribute identifiers
- H5A_CLOSE,aset_id
- H5S_CLOSE,aspace_id
- H5T_CLOSE,atype_id
- ninc=ninc+1 & j=j+1
- ENDWHILE
- ;close all open dataset identifiers
- H5D_CLOSE,dset_id
- H5S_CLOSE,dspace_id
- H5T_CLOSE,dtype_id
-ENDFOR
-
-;The H5G_CLOSE procedure closes the specified group and releases resources used by it.
-H5G_CLOSE,sd_id
-;The H5F_CLOSE procedure closes the HDF file associated with the given file handle.
-H5F_CLOSE,hdf_file_id
-
-END ;Procedure AVDC_HDF5_Write
-
-
-
-PRO avdc_nc_write, hdffilename, natts, av, var_depend, var_size
-;IDL subroutine to write AVDC-type global attributes and datasets to a netCDF file.
-; ----------
-;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
-;
-; History:
-; 20111120: Introduced to IDLCR8HDF - Version 4.0
-; 20130116: Account for string datasets which contain only empty string values (make
-; string length = 1) - Version 4.0b15
-; 20150127: For string datasets, change the VAR_FILL_VALUE to an empty string that
-; is the same length as the individual strings - Version 4.0b25
-; 20150217: Change rule from 4.0b25 so the VAR_FILL_VALUE is an empty string
-; - Version 4.0b26
-; 20161230: Assign a dimension ID to STRING datasets with VAR_DEPEND=CONSTANT so that
-; the string length is correctly accounted for (previously only wrote the
-; first character of the string) - Version 4.0b41
-;
-; Inputs: hdffilename - a string holding the name of the netCDF file being created
-; and written to
-; natts - an integer giving the number of Variable Attributes in each
-; dataset (i.e. the number of elements in attr_arr_data)
-; av - a string array of size (nvn,natts+7) containing the attribute and
-; NCSA values for each dataset
-; var_depend - a string array of size (nvn) holding the VAR_DEPEND attribute
-; values, used to assign DIM information
-; var_size - a string array of size (nvn) holding the VAR_SIZE attribute
-; values, used to assign DIM information
-; ds - structure containing the data
-; meta_arr - a string array containing the Global and Variable Attributes
-; vn - a string array containing the VAR_NAME values
-; ndm - a long array describing the dimensions of each dataset and the
-; VAR_DATA_TYPE
-;
-; Outputs: A netCDF file with the name hdffilename
-;
-; Called by: AVDC_HDF_WRITE
-;
-; Subroutines Called: None
-
-COMMON METADATA
-COMMON DATA
-
-;Create a netCDF file
-df_file_id=NCDF_CREATE(hdffilename,/CLOBBER)
-
-;Write the Global Attributes to file
-;Section 4.1 (Bojkov et al., 2002): Originator attributes
-;Section 4.2 (Bojkov et al., 2002): Dataset attributes
-;Section 4.3 (Bojkov et al., 2002): File attributes
-FOR i=0,N_ELEMENTS(attr_arr_glob)-1 DO BEGIN
- ;separate out the Meta Attribute entry
- eqpos=STRPOS(meta_arr[i],'=')
- IF eqpos EQ STRLEN(meta_arr[i])-1 THEN attribute_value=' ' $
- ELSE attribute_value=STRMID(meta_arr[i],eqpos+1)
- ;get attribute type and space, needed to create the attribute
- NCDF_ATTPUT,df_file_id,/GLOBAL,attr_arr_glob[i],attribute_value
-ENDFOR
-
-;Write the Datasets (SDS) to file
-;Section 5.1 (Bojkov et al, 2002): Variable description attributes.
-al=STRARR(natts)
-al[0:natts-1]=attr_arr_data
-vdh=['CONSTANT','INDEPENDENT']
-dim_id_list=LONARR(2)-999L
-constfound=0
-
-FOR i=0,nvn-1 DO BEGIN
-
- ;Make up data array for writing to netCDF file
- data=*ds[i].data
- ;may have lost array VAR_SIZE information if the last value is '1' e.g. 41,1
- gi=WHERE(ndm[i,0:7] NE 0L) & vs=TRANSPOSE(ndm[i,gi]) ;transpose to get array values in a vector
-
- ;Determine Dimension IDs for the Variable
- vdl=STRSPLIT(var_depend[i],' ;',/EXTRACT,COUNT=vdcount)
- vsl=STRSPLIT(var_size[i],' ;',/EXTRACT) & vsl=LONG(vsl)
- IF ndm[i,8] EQ 6 THEN BEGIN ;need to include string length dimension variable
- vdcount++
- IF MAX(STRLEN(data)) EQ 0 THEN mxstrlen=1 ELSE mxstrlen=MAX(STRLEN(data))
- vdl=['STRLEN',vdl] & vsl=[mxstrlen,vsl]
- ENDIF
- dims=LONARR(vdcount)-999L & const=0
- FOR j=0,vdcount-1 DO BEGIN
- ndi=WHERE(STRUPCASE(vdl[j]) EQ vdh,ndc)
- IF vdl[j] EQ 'STRLEN' THEN BEGIN
- ;dataset is of type string so add string length dimension variable to vdh array
- dim_id=NCDF_DIMDEF(df_file_id,vn[i]+'_STRLEN',vsl[j])
- vdh=[vdh,STRUPCASE(vn[i])+'_STRLEN'] ;add dimension variable to vdh array
- dim_id_list=[dim_id_list,dim_id]
- dims[j]=dim_id
- ENDIF ELSE IF ndc EQ 0 THEN BEGIN ;new dimension found for file
- vdh=[vdh,STRUPCASE(vdl[j])] ;add dimension variable to vdh array
- dim_id=NCDF_DIMDEF(df_file_id,vdl[j],vsl[j])
- dim_id_list=[dim_id_list,dim_id]
- dims[j]=dim_id
- ENDIF ELSE BEGIN ;dimension already has ID or is CONSTANT or INDEPENDENT
- IF ndi[0] EQ 0 THEN BEGIN
- ;CONSTANT so no dimension is assigned
- IF constfound EQ 0 THEN BEGIN
- dim_id=NCDF_DIMDEF(df_file_id,'CONSTANT',1)
- dim_id_list[0]=dim_id ;assign a dim_id for CONSTANT - needed for STRING datasets
- constfound=1
- ENDIF
- const=1
- dims[j]=dim_id_list[0]
- ENDIF ELSE IF ndi[0] EQ 1 THEN BEGIN
- ;INDEPENDENT or BOUNDARIES so assign INDEPENDENT/BOUNDARIES_%vs as dimension name
- IF (j EQ 1) AND (STRPOS(vn[i],'.BOUNDARIES') NE -1) THEN itxt='BOUNDARIES_' $
- ELSE itxt='INDEPENDENT_'
- dim_id=NCDF_DIMDEF(df_file_id,itxt+STRTRIM(vsl[j],2),vsl[j])
- vdh=[vdh,itxt+STRTRIM(vsl[j],2)] ;add dimension variable to vdh array
- dim_id_list=[dim_id_list,dim_id]
- dims[j]=dim_id
- ENDIF ELSE dims[j]=dim_id_list[ndi[0]]
- ENDELSE
- ENDFOR
-
- ;If multi-dimensional reverse dimension IDs as the dataset is automatically transposed
- IF N_ELEMENTS(vs) GT 1 THEN dims=REVERSE(dims)
-
- CASE ndm[i,8] OF ;identifies the VAR_DATA_TYPE
- 0:BEGIN
- valid_range=[BYTE(mv_lng[0,i]),BYTE(mv_lng[1,i])]
- fill_value=BYTE(mv_lng[2,i])
- IF const THEN var_id=NCDF_VARDEF(df_file_id,vn[i],/BYTE) $
- ELSE var_id=NCDF_VARDEF(df_file_id,vn[i],dims,/BYTE)
- END
- 1:BEGIN
- valid_range=[FIX(mv_lng[0,i]),FIX(mv_lng[1,i])]
- fill_value=FIX(mv_lng[2,i])
- IF const THEN var_id=NCDF_VARDEF(df_file_id,vn[i],/SHORT) $
- ELSE var_id=NCDF_VARDEF(df_file_id,vn[i],dims,/SHORT)
- END
- 2:BEGIN
- valid_range=[LONG(mv_lng[0,i]),LONG(mv_lng[1,i])]
- fill_value=LONG(mv_lng[2,i])
- IF const THEN var_id=NCDF_VARDEF(df_file_id,vn[i],/LONG) $
- ELSE var_id=NCDF_VARDEF(df_file_id,vn[i],dims,/LONG)
- END
- 3:BEGIN
- valid_range=[mv_lng[0,i],mv_lng[1,i]]
- fill_value=mv_lng[2,i]
- IF const THEN var_id=NCDF_VARDEF(df_file_id,vn[i],/LONG) $
- ELSE var_id=NCDF_VARDEF(df_file_id,vn[i],dims,/LONG)
- END
- 4:BEGIN
- valid_range=[FLOAT(mv_dbl[0,i]),FLOAT(mv_dbl[1,i])]
- fill_value=FLOAT(mv_dbl[2,i])
- IF const THEN var_id=NCDF_VARDEF(df_file_id,vn[i],/FLOAT) $
- ELSE var_id=NCDF_VARDEF(df_file_id,vn[i],dims,/FLOAT)
- END
- 5:BEGIN
- valid_range=[mv_dbl[0,i],mv_dbl[1,i]]
- fill_value=mv_dbl[2,i]
- IF const THEN var_id=NCDF_VARDEF(df_file_id,vn[i],/DOUBLE) $
- ELSE var_id=NCDF_VARDEF(df_file_id,vn[i],dims,/DOUBLE)
- END
- 6:BEGIN
- valid_range=[' ',' ']
- fill_value=' ' ;bytarr(mv_str[i]) ;STRING(format='(A-'+STRTRIM(mv_str[i],2)+')','')
- var_id=NCDF_VARDEF(df_file_id,vn[i],dims,/CHAR)
- END
- ENDCASE
-
- ;write out variable attributes
- ninc=0 & j=0
- mi=WHERE(meta_arr EQ attr_arr_data[0]+'='+vn[i])
- WHILE j LE natts-1 DO BEGIN
- ma=mi[0]+ninc
- ;Extract Attribute Label
- eqpos=STRPOS(meta_arr[ma],'=')
- res=STRMID(meta_arr[ma],0,eqpos)
- WHILE res[0] NE al[j] DO j=j+1 ;in case of missing optional variable attributes
- CASE 1 OF
- attr_arr_data[j] EQ 'VAR_VALID_MIN': athold=valid_range[0]
- attr_arr_data[j] EQ 'VAR_VALID_MAX': athold=valid_range[1]
- attr_arr_data[j] EQ 'VAR_FILL_VALUE': athold=fill_value
- ELSE: athold=av[i,j]
- ENDCASE
- NCDF_ATTPUT,df_file_id,var_id,attr_arr_data[j],athold
- ninc=ninc+1 & j=j+1
- ENDWHILE
-
- IF ndm[i,8] NE 6 THEN BEGIN
- NCDF_ATTPUT,df_file_id,var_id,'units',vu[i]
- NCDF_ATTPUT,df_file_id,var_id,'valid_range',valid_range
- NCDF_ATTPUT,df_file_id,var_id,'_Fillvalue',fill_value
- ENDIF
-
- NCDF_CONTROL,df_file_id,/ENDEF ;Enter Data Mode
- NCDF_VARPUT,df_file_id,var_id,data ;Write Data to file
- NCDF_CONTROL,df_file_id,/REDEF ;Enter Define Mode
-ENDFOR
-
-NCDF_CLOSE,df_file_id
-
-END ;Procedure AVDC_NC_Write
-
-
-
-PRO avdc_hdf_write, hdffilename, geoms_output
-;IDL subroutine to write AVDC-type global attributes and datasets to a Scientific
-;Dataset (SDS) in HDF4 (Bojkov et al.,2004) or HDF5. This routine creates the HDF4
-;file if requested, and calls the AVDC_HDF5_WRITE routine to create the HDF5 file if
-;that option is chosen.
-; ----------
-;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
-;
-; History:
-; 20050802: Originally split into two routines to separately write the Global
-; attributes (AVDC_HDF_GLOBAL_ATTRIBUTES) and the Variable attributes
-; and data (AVDC_HDF_SDS_VARIABLES) to an HDF4 file - Version 1.0
-; 20061012: Two routines combined into the AVDC_HDF_WRITE routine; Call to the
-; AVDC_HDF5_WRITE routine if HDF5 file option is chosen; HDF4 file not
-; closed between writing the global and variable attributes to file;
-; Data now written to file from the ds structure instead of from arrays
-; made according to VAR_DATA_TYPE; the av array made to hold attribute
-; values to write to the HDF file, instead of writing the values to
-; file directly from the meta_arr array (to avoid duplication of code
-; in AVDC_HDF5_WRITE) - Version 2.0
-; 20130116: Account for string datasets which contain only empty string values (make
-; string length = 1) - Version 4.0b15
-; 20150127: For string datasets, change the VAR_FILL_VALUE to an empty string that
-; is the same length as the individual strings - Version 4.0b25
-; 20150217: Change rule from 4.0b25 so the VAR_FILL_VALUE is an empty string
-; - Version 4.0b26
-;
-; Inputs: hdffilename - a string holding the name of the HDF file being created
-; and written to
-; ds - structure containing the data
-; meta_arr - a string array containing the Global and Variable Attributes
-; vn - a string array containing the VAR_NAME values
-; ndm - a long array describing the dimensions of each dataset and the
-; VAR_DATA_TYPE
-;
-; Outputs: An HDF4 file with the name hdffilename
-;
-; Called by: IDLCR8HDF
-;
-; Subroutines Called: MAKEAN
-; AVDC_HDF5_WRITE (if HDF5 option is chosen)
-; INFOTXT_OUTPUT
-; Information Conditions (when the program is able to make changes):
-; 1. HDF4.2R1 or earlier library and SDS name has greater than 63-characters
-; - in which case the SDS name will be truncated [3573]
-
-COMMON METADATA
-COMMON DATA
-
-nodimset=1 ;added from version 3.06 to stop dimension information being written to the HDF4 file
- ;while still retaining original code
-
-;Determine HDF format (ftype is either 'hdf', 'h5' or 'nc')
-ftype=STRMID(hdffilename,STRPOS(hdffilename,'.',/Reverse_Search)+1)
-
-;List of dim. attributes to be assigned to the SDS dimensions.
-da_list=['VAR_NAME','VAR_UNITS']
-da_att=INDGEN(N_ELEMENTS(da_list),/String) & natts=N_ELEMENTS(attr_arr_data)
-av=STRARR(nvn,natts) ;array containing attribute and NCSA values for each dataset
-var_depend=STRARR(nvn) ;holding array for the VAR_DEPEND attributes, used to assign DIM information
-var_size=STRARR(nvn) ;holding array for the VAR_SIZE attributes, used to assign netCDF DIM information
-
-;Write all Dataset Attribute values to an array
-FOR i=0,nvn-1 DO BEGIN ;nvn EQ number of variable names
- mi=WHERE(meta_arr EQ attr_arr_data[0]+'='+vn[i])
- ninc=0 & j=0
- WHILE j LE natts-1 DO BEGIN
- ma=mi[0]+ninc
- ;separate out the Meta entry into two components
- eqpos=STRPOS(meta_arr[ma],'=')
- res=STRMID(meta_arr[ma],0,eqpos)
- WHILE res NE attr_arr_data[j] DO j=j+1 ;in case optional variable attributes are not included
- IF eqpos EQ STRLEN(meta_arr[ma])-1 THEN av[i,j]=' ' $
- ELSE av[i,j]=STRMID(meta_arr[ma],eqpos+1)
- ;Reverse Values for VAR_DEPEND and VAR_SIZE to match order values will be saved in HDF
- IF (res EQ 'VAR_DEPEND') OR (res EQ 'VAR_SIZE') THEN BEGIN
- atval=STRSPLIT(av[i,j],' ;',/EXTRACT,COUNT=avcnt)
- IF avcnt GT 1 THEN BEGIN
- atval=REVERSE(atval)
- FOR k=0,avcnt-1 DO IF k EQ 0 THEN ahold=atval[k] ELSE ahold=ahold+';'+atval[k]
- av[i,j]=ahold
- ENDIF
- IF res EQ 'VAR_DEPEND' THEN var_depend[i]=av[i,j] ELSE var_size[i]=av[i,j]
- ENDIF
- ninc=ninc+1 & j=j+1
- ENDWHILE
-ENDFOR
-
-IF ftype EQ 'hdf' THEN BEGIN
-
- ;Determine HDF library used by this program
- HDF_LIB_INFO,MAJOR=mj,MINOR=mn,RELEASE=rl
- hdf4lib=(mj*100)+(mn*10)+rl
- hdf4txt='HDF'+STRTRIM(mj,2)+'.'+STRTRIM(mn,2)+'R'+STRTRIM(rl,2)
-
- ;Create and open the HDF4 file for writing
- hdf_file_id=HDF_OPEN(hdffilename,/ALL)
- ;The HDF_SD_START function opens or creates an HDF4 file and initializes the SD interface.
- sd_id=HDF_SD_START(hdffilename,/RDWR)
-
- ;Write the Global Attributes to file
- ;Section 4.1 (Bojkov et al., 2002): Originator attributes
- ;Section 4.2 (Bojkov et al., 2002): Dataset attributes
- ;Section 4.3 (Bojkov et al., 2002): File attributes
- FOR i=0,N_ELEMENTS(attr_arr_glob)-1 DO BEGIN
- ;separate out the Meta Attribute entry
- eqpos=STRPOS(meta_arr[i],'=')
- IF eqpos EQ STRLEN(meta_arr[i])-1 THEN attribute_value=' ' $
- ELSE attribute_value=STRMID(meta_arr[i],eqpos+1)
- ;check for Free text attribute, and filename entry
- res=STRMID(meta_arr[i],0,eqpos)
- fai=WHERE(res EQ attr_free,facnt)
- IF (facnt NE 0) AND (STRMID(STRUPCASE(attribute_value),0,6) EQ 'FILE("') THEN $
- MAKEAN,hdf_file_id,sd_id,'FILE',res,attribute_value $
- ELSE HDF_SD_ATTRSET,sd_id,attr_arr_glob[i],attribute_value
- ENDFOR
-
- ;Write the Datasets (SDS) to file
- ;Section 5.1 (Bojkov et al, 2002): Variable description attributes.
- FOR i=0,nvn-1 DO BEGIN ;nvn EQ number of variable names
-
- ;If hdf4 library version is less than 4.2R2 then check length of VAR_NAME
- IF hdf4lib LT 422 THEN BEGIN
- IF STRLEN(vn[i]) GT 63 THEN BEGIN
- infotxt=STRARR(4)
- infotxt[0]='2 '+hdf4txt+' truncates the dataset name if its'
- infotxt[0]=infotxt[0]+' length is greater than 63-characters.'
- infotxt[1]=' If possible update the HDF4 library to HDF4.2R2 or newer| '
- infotxt[1]=infotxt[1]+'(refer to program documentation for procedure to do this).
- infotxt[2]=' The Archive Data Center may also be set up to update the file'
- infotxt[2]=infotxt[2]+' on submission.'
- infotxt[3]=' VAR_NAME='+vn[i]
- INFOTXT_OUTPUT, infotxt
- ENDIF
- ENDIF
-
- ;create data array from structure and create and define dataset for an HDF4 file
- data=*ds[i].data
- ;may have lost array VAR_SIZE information if the last value is '1' e.g. 41,1
- gi=WHERE(ndm[i,0:7] NE 0L) & vs=TRANSPOSE(ndm[i,gi])
- type=HDF_IDL2HDFTYPE(SIZE(data,/Type))
- IF ndm[i,8] EQ 6 THEN BEGIN
- mxstrlen=MAX(STRLEN(data)) ;determine maximum stength length
- IF mxstrlen EQ 0 THEN mxstrlen=1
- sds_id_1=HDF_SD_CREATE(sd_id,vn[i],[mxstrlen,vs],HDF_TYPE=type)
- ENDIF ELSE sds_id_1=HDF_SD_CREATE(sd_id,vn[i],vs,HDF_TYPE=type)
-
- mi=WHERE(meta_arr EQ attr_arr_data[0]+'='+vn[i])
- CASE ndm[i,8] OF ;identifies the VAR_DATA_TYPE
- 0:BEGIN
- valid_range=[BYTE(mv_lng[0,i]),BYTE(mv_lng[1,i])]
- fill_value=BYTE(mv_lng[2,i])
- END
- 1:BEGIN
- valid_range=[FIX(mv_lng[0,i]),FIX(mv_lng[1,i])]
- fill_value=FIX(mv_lng[2,i])
- END
- 2:BEGIN
- valid_range=[LONG(mv_lng[0,i]),LONG(mv_lng[1,i])]
- fill_value=LONG(mv_lng[2,i])
- END
- 3:BEGIN
- valid_range=[mv_lng[0,i],mv_lng[1,i]]
- fill_value=mv_lng[2,i]
- END
- 4:BEGIN
- valid_range=[FLOAT(mv_dbl[0,i]),FLOAT(mv_dbl[1,i])]
- fill_value=FLOAT(mv_dbl[2,i])
- END
- 5:BEGIN
- valid_range=[mv_dbl[0,i],mv_dbl[1,i]]
- fill_value=mv_dbl[2,i]
- END
- 6:BEGIN
- valid_range=[' ',' ']
- fill_value=' ' ;bytarr(mv_str[i]) ;STRING(format='(A-'+STRTRIM(mv_str[i],2)+')','')
- ;Note: this seems to give the same result as using 0B and saving it as type string in the HDF_SD_ATTRSET call
- END
- ENDCASE
- ninc=0 & j=0
- WHILE j LE natts-1 DO BEGIN
- ma=mi[0]+ninc
- ;Extract Attribute Label
- eqpos=STRPOS(meta_arr[ma],'=')
- res=STRMID(meta_arr[ma],0,eqpos)
- WHILE res[0] NE attr_arr_data[j] DO j=j+1
- CASE 1 OF
- ;attr_arr_data[j] EQ 'VAR_SIZE': HDF_SD_ATTRSET,sds_id_1,attr_arr_data[j],vs
- attr_arr_data[j] EQ 'VAR_VALID_MIN': HDF_SD_ATTRSET,sds_id_1,attr_arr_data[j],valid_range[0]
- attr_arr_data[j] EQ 'VAR_VALID_MAX': HDF_SD_ATTRSET,sds_id_1,attr_arr_data[j],valid_range[1]
- attr_arr_data[j] EQ 'VAR_FILL_VALUE': HDF_SD_ATTRSET,sds_id_1,attr_arr_data[j],fill_value
- ELSE: HDF_SD_ATTRSET,sds_id_1,attr_arr_data[j],av[i,j]
- ENDCASE
- ninc=ninc+1 & j=j+1
- ENDWHILE
- ;Set information about the dataset (note no pre-defined attributes for String Datatype)
- IF ndm[i,8] NE 6 THEN $
- HDF_SD_SETINFO,sds_id_1,UNIT=vu[i],RANGE=valid_range,FILL=fill_value
-
- IF nodimset EQ 0 THEN BEGIN
- ;Assign the VAR_DEPEND dimension(s) attribute information
- vd_out=STRSPLIT(var_depend[i],' ;',/Extract) ;Read the variable dependencies
- vdim=N_ELEMENTS(vd_out)
- vd_out=STRTRIM(vd_out,2)
- ;Start search of information
- FOR j=0,vdim-1 DO BEGIN
- IF (STRUPCASE(vd_out[j]) NE 'CONSTANT') AND (STRUPCASE(vd_out[j]) NE 'INDEPENDENT') THEN BEGIN
- ;Return the index of the var_depend array SDS dataset
- index=HDF_SD_NAMETOINDEX(sd_id,vd_out[j])
- ;Access the dataset
- sds_id_tmp=HDF_SD_SELECT(sd_id,index)
- FOR k=0,N_ELEMENTS(da_list)-1 DO BEGIN
- ;Find the dataset attribute in da_list
- dindex=HDF_SD_ATTRFIND(sds_id_tmp,da_list[k])
- ;Read attribute info and assign to da_att
- HDF_SD_ATTRINFO,sds_id_tmp,dindex,DATA=att_data
- da_att[k]=att_data
- ENDFOR
- ;End access of SDS search
- HDF_SD_ENDACCESS,sds_id_tmp
-
- ;Assign dimension to the current SDS
- dim_id=HDF_SD_DIMGETID(sds_id_1,j)
- HDF_SD_DIMSET,dim_id,/BW_INCOMP,NAME=da_att[0],UNIT=da_att[1]
- ENDIF
- ENDFOR
- ENDIF
-
- ;Write data to HDF4 file
- HDF_SD_ADDDATA,sds_id_1,data
- HDF_SD_ENDACCESS,sds_id_1
- ENDFOR
-
- ;The HDF_SD_END function closes the SD interface to an HDF4 file.
- HDF_SD_END,sd_id
- ;The HDF_CLOSE procedure closes the HDF4 file associated with the given file handle.
- HDF_CLOSE,hdf_file_id
-ENDIF ELSE IF ftype EQ 'h5' THEN AVDC_HDF5_WRITE,hdffilename,natts,av,geoms_output $
-ELSE AVDC_NC_WRITE,hdffilename,natts,av,var_depend,var_size
-
-END ;Procedure AVDC_HDF_Write
-
-
-
-PRO idlcr8hdf, ga, sds, tav, odir, reterr, H5=o1, AVK=o2, LOG=o4, POPUP=o5, QA=o6, NOHDF=o7, NC=o8, DATETIME=o9, $
- C1=o11, C2=o12, C3=o13, C4=o14, C5=o15, C6=o16, C7=o17, C8=o18, C9=o19
-;Main IDL program to create HDF4 or HDF5 format files for submission to the AVDC, NDACC, or
-;NILU (ESA Envisat) databases.
-;
-;Program documentation, idlcr8hdf-v4.0_Readme.pdf, available from http://avdc.gsfc.nasa.gov.
-;
-;Program sub-version 4.0b64 (20240912)
-; ----------
-;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
-;
-; History:
-; 20050802: Original Release - Version 1.0
-; 20050909: No change to routine - Version 1.1
-; 20051107: No change to routine - Version 1.11
-; 20061012: Make the code suitable for running on a licensed version of IDL (using the .pro
-; and .sav versions of the code) and on IDL Virtual Machine (using the .sav version
-; of the code); Change command line keyword options, remove the /Help (window now
-; opened if there are no command line parameters) and /File options (now automatically
-; identifies whether input is from session memory or files), and add the /H5 (write
-; HDF5), /AVK (for Averaging Kernel Datasets add sentence to VAR_NOTES indicating
-; averaging kernel order), /Log (append input/output information to a log file), and
-; /Popup options (append input/output, error and warning information to a pop-up
-; display window); If input is from files, the program can now handle multiple data
-; files (either as a string array or as a file spec), together with a Metadata template
-; file and the TAV file; Input/output information as well as errors and warnings
-; included in the IDL DE output log window and/or to a log file; The user has the
-; option of continuing to run the program with file inputs from the 'Introduction'
-; window; Common variable definition WIDGET_WIN added - Version 2.0
-; 20080302: Remove HELP,/TRACEBACK call, which identified how the program was called. This was used
-; to stop output to the IDLDE output window in the event that IDL VM was used, but it is
-; not required as IDL VM ignores these print calls; Add DIALOG_BOX to show completion
-; of the program if program inputs are via the INTRO box, and logging window option
-; isn't requested - Version 3.0
-; 20100205: Add optional reterr parameter which allows program to return to a calling program with
-; an error message, rather than stop - Version 3.09
-; 20101120: Adopt GEOMS metadata standard; New structure format required when input is by session
-; memory; New arrays created to hold numeric metadata values (mv_lng and mv_dbl) in
-; either LONG64 or DOUBLE format (changed to their actual format before writing to file);
-; Add QA keyword, to perform QA function on HDF file read into session memory (does not
-; create an HDF file) - Version 4.0
-; 20130114: Add check for IDL being run in DEMO mode by using LMGR command. Replace PRINTF,-1
-; statements (i.e. when dux[0] eq -1) with PRINT (o/w causes DEMO mode to stop. Also disable
-; /LOG and /NC keywords in DEMO mode - Version 4.0b14
-; 20140327: Add check and log message for VAR_SIZE values not being of type string - Version 4.0b20
-; 20150127: When input is by structure, stop removing leading and trailing spaces from variable
-; attribute values when creating metafile array - Version 4.0b25
-; 20150217: Initialize mv_str if input is via files - Version 4.0b26
-; 20150302: Fix Bug which incorrectly set mv_str value if input is via structure - Version 4.0b27
-; 20151012: Create qa_yes common variable to streamline checks on whether program called in QA mode
-; or not - Version 4.0b31
-; 20160725: Fix bug so that numeric metadata sub-values read in from a structure will be written to
-; the metafile array OK - Version 4.0b38
-; 20161130: Fix issue when a metadata value is not numeric or a string e.g. a structure when the
-; inputs are in the form of a structure - Version 4.0b40
-; 20180218: Ensure variable attribute value (vav) is only a single variable when calling
-; PRE_DEFINED_ATT_CHECKS - Version 4.0b45
-; 20181221: Allow compression and shuffle options when creating HDF5 files - Version 4.0b49
-;
-; Inputs: DIALOG PROMPTS WITH IDL VIRTUAL MACHINE (File input only)
-; Metadata template file - The program will fill in missing spaces based on data file and
-; TAV inputs
-; Data file(s) - the user has the option of picking multiple data files
-; tav - the current Table Attribute Values (TAV) file
-; outdir - the directory which will hold the HDF file
-;
-; COMMAND LINE PARAMETER OPTION AVAILABLE WITH FULL IDL VERSION
-; ga - either a string array containing the Global Attributes, or a Metadata template file
-; (as above)
-; sds - either a heap structure using pointers containing the Data (DS.Data) and the Variable
-; Attributes (DS.VA_L and DS.VA_V) for a single file, or a string array or file spec of
-; file(s) containing the data (as above)
-; tav - the current Table Attribute Values (TAV) file
-; outdir - the directory to which any HDF and log files will be written
-; reterr - optional string input that, if included, returns an error message to the calling
-; program rather than stop the program. Note that the Pop-up option will be
-; deselected if present together with this argument.
-;
-; OPTIONS
-; H5 - output will be as an HDF5 file instead of the standard HDF4
-; NC - output will be as a netCDF3 file instead of the standard HDF4
-; AVK - to add a sentence to VAR_NOTES indicating array order for Averaging Kernel
-; datasets
-; Log - to append input/output information as well as warnings and errors to a log file
-; named idlcr8hdf.log, or idlcr8qa.log if QA option chosen
-; Popup - to append input/output information as well as warnings and errors to a pop-up
-; display window
-; QA - to perform QA function on an HDF file passed to the program using the session
-; memory option. The logfile is automatically generated and will be called
-; idlcr8qa.log instead of idlcr8hdf.log
-; NoHDF - an HDF file will not be created, but the logfile idlcr8hdf.log will automatically
-; be generated
-; Cn - to enable compression and shuffling of HDF5 files only (ignored for netCDF3 and HDF4)
-;
-; Output: An HDF file formatted to GEOMS standards. Keyword options can also result
-; in an output log file (idlcr8hdf.log), or a pop-up box showing log output. If the
-; reterr string input parameter is included then reterr will return an error message to the
-; calling program, if encountered.
-;
-; Called by: Main Routine
-;
-; Subroutines Called: INTRO (if program called without command line parameters)
-; READ_TABLEFILE
-; READ_METADATA
-; CHECK_METADATA
-; SET_UP_STRUCTURE
-; READ_DATA
-; FIND_HDF_FILENAME
-; AVDC_HDF_WRITE
-; STOP_WITH_ERROR (if error state detected)
-; INFOTXT_OUTPUT (if information conditions detected)
-; Possible Conditions for STOP_WITH_ERROR call:
-; 1. If Metadata, Data, or Table Attribute Values file(s), or Output Directory
-; selection is not valid
-; 2. If input is via session memory and the array holding the global attributes
-; is not of type string, or is not one dimensional
-; 3. If input is via session memory and the input heap structure is not valid
-; 4. If input is via session memory and the array sizes of the heap structures
-; do not match
-; 5. If the H5 option is chosen and the IDL version does not support HDF5 write
-; routines (less than v6.2)
-; 6. If the NC option is chosen and IDL was called in IDL DEMO mode
-;
-; Information Conditions (when the program is able to make changes):
-; 1. /POPUP keyword cannot be used together with the 'reterr' argument
-; 2. Argument 'reterr' must be a scalar variable of type string
-; 3. /LOG keyword cannot be used in IDL DEMO mode
-
-COMMON TABLEDATA
-COMMON METADATA
-COMMON DATA
-COMMON WIDGET_WIN
-
-;Possible error message for this procedure
-proname='IDLcr8HDF procedure: ' & lu=-1L
-errtxt=STRARR(6)
-errtxt[0]=' selection not valid'
-errtxt[1]='Global Attributes Array is not of type string, or is not one dimensional'
-errtxt[2]='structure is not valid ('
-errtxt[3]='Array sizes of the heap structure do not match: '
-errtxt[4]=' does not support HDF5 Write Routines'
-errtxt[5]='IDLcr8HDF requires session memory input to perform the QA function'
-
-demomode=LMGR(/DEMO) ;Boolean to check for IDL being run in demo mode (disable /LOG option)
-IF N_PARAMS() GE 2 THEN intype=SIZE(sds,/TYPE) $ ;Is SDS a structure (8) or a string (7)?
-ELSE IF demomode THEN intype=-3 $
-ELSE intype=-1
-IF (intype NE 7) AND (intype NE 8) AND (intype NE -1) AND (intype NE -3) THEN intype=-2
-;Command line parameter neither string nor structure
-dux=[-1,-1,1] ;initialize flags for output log to IDLDE Output Window and/or to file
-;dux[0] default allows output to the IDLDE output window (ignored by IDL VM)
-;dux[1] is the program assigned file unit for sending log output to file
-;dux[2] is the step between dux[0] and dux[1]
-lineno=0L ;count variable to put curser at end of text pop-up window
-rerr=['NA','NA'] ;initialize return error string
-
-IF intype LT 0 THEN BEGIN ;either no input parameters or invalid second parameter
- o3=['0','0','0','0','0','0']
- lvals=['None','1','2','3','4','5','6','7','8','9'] ;compression level for HDF5 option
- INTRO,intype ;Open Intro Box and determine HDF output format (HDF4 or HDF5)
- IF o3[0] EQ '0' THEN BEGIN
- STOP_WITH_ERROR,'','',lu & RETURN
- ENDIF
- IF o3[3] EQ 'Pop' THEN o3[3]='' ELSE o3[3]='D_'
-ENDIF ELSE BEGIN ;Set options (HDF4, HDF5, netCDF, AVK, LOGFILE, POP-UP, QA, NOHDF)
- IF KEYWORD_SET(o1) THEN BEGIN
- o3=['H5','0','0','D_','0','0']
- CASE 1 OF
- KEYWORD_SET(o11): o3[0]='H5_1'
- KEYWORD_SET(o12): o3[0]='H5_2'
- KEYWORD_SET(o13): o3[0]='H5_3'
- KEYWORD_SET(o14): o3[0]='H5_4'
- KEYWORD_SET(o15): o3[0]='H5_5'
- KEYWORD_SET(o16): o3[0]='H5_6'
- KEYWORD_SET(o17): o3[0]='H5_7'
- KEYWORD_SET(o18): o3[0]='H5_8'
- KEYWORD_SET(o19): o3[0]='H5_9'
- ELSE:
- ENDCASE
- ;PRINT,o3
- ENDIF ELSE IF KEYWORD_SET(o8) THEN o3=['NC','0','0','D_','0','0'] $
- ELSE o3=['H4','0','0','D_','0','0']
- IF KEYWORD_SET(o2) THEN o3[1]='AVK' ;AVK option
- IF KEYWORD_SET(o4) THEN o3[2]='idlcr8hdf.log' ;Log option
- IF KEYWORD_SET(o5) THEN o3[3]='' ;POP-UP option
- IF KEYWORD_SET(o6) THEN o3[2]='idlcr8qa.log' ;QA option
- IF KEYWORD_SET(o7) THEN o3[4]='NOHDF' ;NoHDF option
- IF KEYWORD_SET(o9) THEN o3[5]='DTFVOK' ;DATETIME Fill Value OK
-ENDELSE
-IF o3[0] EQ 'NC' THEN dftxt='netCDF' ELSE dftxt='HDF'
-
-;If reterr included allows error output to return to a calling program
-n_it=5
-infotxt=STRARR(2,n_it)
-IF N_PARAMS() GE 5 THEN BEGIN
- ;IF (ARG_PRESENT(reterr)) AND (SIZE(reterr,/Type) EQ 7) THEN BEGIN
- IF SIZE(reterr,/Type) EQ 7 THEN BEGIN
- rerr=['','']
- IF o3[3] EQ '' THEN BEGIN
- ;Deselect POPUP option and write INFORMATION message
- o3[3]='D_'
- infotxt[0,0]='0 /POPUP keyword cannot be used together with the ''reterr'' argument.'
- infotxt[1,0]=' The request for a POPUP window has been ignored.'
- ENDIF
- ENDIF ELSE BEGIN
- ;reterr input included but is of the incorrect type, or parameter cannot be returned
- ;to the calling program
- infotxt[0,1]='0 Argument ''reterr'' must be of type string.' ;a returnable variable of type string.'
- infotxt[1,1]=' idlcr8hdf will stop normally if an error is encountered.'
- ENDELSE
-ENDIF
-
-;Do check for /LOG and /NC keywords in IDL DEMO Mode and if necessary disable or stop the program
-IF demomode THEN BEGIN
- IF o3[2] EQ 'idlcr8hdf.log' THEN BEGIN
- o3[2]='0'
- infotxt[0,2]='0 /LOG keyword cannot be used in IDL DEMO mode and has been ignored.'
- ENDIF
- IF o3[0] EQ 'NC' THEN BEGIN
- o3[4]='NOHDF'
- infotxt[0,3]='3 NetCDF file create feature is disabled in IDL DEMO mode. '
- infotxt[1,3]=' To generate files please use the free IDL Virtual Machine or a licenced version of IDL.'
- ENDIF
-ENDIF
-
-qa_yes=STRPOS(o3[2],'idlcr8qa.log') NE -1 ;Set Boolean for doing QA checks
-
-IF (KEYWORD_SET(o1)) AND (FLOAT(!Version.Release) LT 6.2) THEN BEGIN ;No HDF5 library for IDL6.1 or less
- STOP_WITH_ERROR,'D_'+proname,'IDL Version '+!Version.Release+errtxt[4],lu
- IF STRLEN(rerr[0]) GT 2 THEN reterr=rerr[0] & RETURN
-ENDIF
-
-;Check TAV file and Output Directory inputs
-tablefile='' & outdir=''
-CD,CURRENT=path ;identify current working directory
-IF N_PARAMS() GE 3 THEN BEGIN ;check TAV file
- IF (SIZE(tav,/N_ELEMENTS) EQ 1) AND (SIZE(tav,/TYPE) EQ 7) THEN BEGIN
- file_exist=FILE_TEST(tav,/READ)
- IF (file_exist EQ 1) AND (tav NE '') THEN tablefile=tav
- ENDIF
-ENDIF
-IF N_PARAMS() GE 4 THEN BEGIN ;check output directory
- IF (SIZE(odir,/N_ELEMENTS) EQ 1) AND (SIZE(odir,/TYPE) EQ 7) THEN BEGIN
- file_exist=FILE_TEST(odir,/DIRECTORY)
- IF (file_exist EQ 1) AND (odir NE '') THEN BEGIN
- outdir=FILE_SEARCH(odir,/FULLY_QUALIFY_PATH,/MARK_DIRECTORY)
- path=outdir
- ENDIF ELSE IF intype NE 8 THEN outdir=STRUPCASE(STRTRIM(odir,2))
- ENDIF
-ENDIF
-
-IF (intype LT 0) OR (intype EQ 7) THEN BEGIN ;Input via files
- IF KEYWORD_SET(o6) THEN BEGIN
- ;session memory inputs required for QA option
- STOP_WITH_ERROR,'D_'+proname,errtxt[5],lu
- IF STRLEN(rerr[0]) GT 2 THEN reterr=rerr[0] & RETURN
- ENDIF
- metafile='' & inf=1 & mv_str=[-1L]
- IF N_PARAMS() GE 2 THEN BEGIN
- ;Check that input files exist. If not then user will be prompted for new inputs
- IF SIZE(ga,/N_ELEMENTS) EQ 1 THEN BEGIN
- file_exist=FILE_TEST(ga,/READ)
- IF (file_exist EQ 1) AND (ga NE '') THEN $
- metafile=FILE_DIRNAME(ga,/MARK_DIRECTORY)+FILE_BASENAME(ga)
- ENDIF
- arorfs=SIZE(sds,/DIMENSIONS) ;Is datafile input Filespec or String Array?
- IF arorfs[0] EQ 0 THEN BEGIN ;input is filespec (scalar)
- IF sds NE '' THEN sds=FINDFILE(sds)
- isize=SIZE(sds,/DIMENSIONS)
- IF isize[0] EQ 0 THEN sds=[''] ;no files found matching filespec
- ENDIF
- IF N_ELEMENTS(arorfs) GT 1 THEN sds=REFORM(sds,N_ELEMENTS(sds),/OVERWRITE) ;Convert to 1-D array
- file_exist=FILE_TEST(sds,/READ)
- gi=WHERE(file_exist EQ 1,nfile)
- IF nfile NE 0 THEN BEGIN
- sds=sds[gi] & sds=FILE_DIRNAME(sds,/MARK_DIRECTORY)+FILE_BASENAME(sds)
- ENDIF ELSE sds=[''] ;contains all valid datafiles
- ENDIF ELSE sds=['']
- IF metafile EQ '' THEN BEGIN
- metafile=DIALOG_PICKFILE(Filter=['*.meta','meta*.txt','*.txt'], PATH=path, $
- /MUST_EXIST,Title='Select Metadata Template File')
- IF metafile EQ '' THEN BEGIN
- STOP_WITH_ERROR,'D_'+proname,'Metadata file'+errtxt[0],lu
- IF STRLEN(rerr[0]) GT 2 THEN reterr=rerr[0] & RETURN
- ENDIF
- path=FILE_DIRNAME(metafile,/MARK_DIRECTORY)
- END
- IF sds[0] EQ '' THEN BEGIN
- datafile=DIALOG_PICKFILE(Filter=['*.data','*.dat','*.txt'], PATH=path, $
- /MUST_EXIST,/Multiple_Files,Title='Select Data File(s)')
- gi=WHERE(datafile NE '',nfile)
- IF nfile EQ 0 THEN BEGIN
- STOP_WITH_ERROR,'D_'+proname,'Data file(s)'+errtxt[0],lu
- IF STRLEN(rerr[0]) GT 2 THEN reterr=rerr[0] & RETURN
- ENDIF ELSE BEGIN
- datafile=datafile[gi] & dsort=SORT(datafile) & sds=datafile[dsort]
- path=FILE_DIRNAME(sds[0],/MARK_DIRECTORY)
- ENDELSE
- ENDIF
- IF outdir EQ 'M' THEN outdir=FILE_DIRNAME(metafile,/MARK_DIRECTORY) $
- ELSE IF outdir EQ 'D' THEN outdir=FILE_DIRNAME(sds[0],/MARK_DIRECTORY)
-ENDIF ELSE BEGIN ;SDS is a data structure, so do initial checks on parameters
- ;check GA array
- inf=0 & nfile=1
- as=SIZE(ga) & n_ga=N_ELEMENTS(ga)
- IF (as[0] NE 1) OR (as[2] NE 7) THEN BEGIN
- STOP_WITH_ERROR,'D_'+proname,errtxt[1],lu
- IF STRLEN(rerr[0]) GT 2 THEN reterr=rerr[0] & RETURN
- ENDIF
-
- vdtype=[1,2,3,4,5,7,12,13,14,15] ;valid IDL data types
- badlabel=[''] ;list of attribute labels where the values are not valid (to avoid repeated log messages)
- it5=[''] ;array holding infotxt[*,5] error messages
-
- ;check pointer array (heap structure)
- IF N_TAGS(sds) EQ 0 THEN BEGIN
- STOP_WITH_ERROR,'D_'+proname,'Input heap '+errtxt[2]+'expecting VA_L, VA_V, and DATA tags).',lu
- IF STRLEN(rerr[0]) GT 2 THEN reterr=rerr[0] & RETURN
- ENDIF
- tagnames=STRUPCASE(TAG_NAMES(sds))
- si=WHERE((tagnames EQ 'VA_L') OR (tagnames EQ 'VA_V') OR (tagnames EQ 'DATA'),scnt)
- IF scnt NE 3 THEN BEGIN
- STOP_WITH_ERROR,'D_'+proname,'Input heap '+errtxt[2]+'missing VA_L, VA_V, and/or DATA tags).',lu
- IF STRLEN(rerr[0]) GT 2 THEN reterr=rerr[0] & RETURN
- ENDIF
- ;Check for number and size of dimensions, and invalid pointer arguments for the data
- FOR i=0,2 DO BEGIN
- CASE 1 OF
- i EQ 0: BEGIN
- pchk=PTR_VALID(sds.data) & sdstxt='SDS.DATA '
- END
- i EQ 1: BEGIN
- pchk=PTR_VALID(sds.va_v) & sdstxt='SDS.VA_V '
- pchkv=pchk
- END
- ELSE: BEGIN
- pchk=PTR_VALID(sds.va_l) & sdstxt='SDS.VA_L '
- END
- ENDCASE
- ;Check that the SDS structure has two dimensions
- s_ndim=SIZE(pchk,/N_DIMENSIONS) & s_dim=SIZE(pchk,/DIMENSIONS)
- IF s_ndim NE 2 THEN BEGIN
- STOP_WITH_ERROR,'D_'+proname,sdstxt+errtxt[2]+'two dimensions expected).',lu
- IF STRLEN(rerr[0]) GT 2 THEN reterr=rerr[0] & RETURN
- ENDIF ELSE IF i EQ 0 THEN BEGIN
- pi=WHERE(pchk[*,0] NE 1,pcnt)
- IF pcnt NE 0 THEN BEGIN
- STOP_WITH_ERROR,'D_'+proname,sdstxt+errtxt[2]+'null pointer arguments).',lu
- IF STRLEN(rerr[0]) GT 2 THEN reterr=rerr[0] & RETURN
- ENDIF
- ENDIF ELSE BEGIN
- IF i EQ 2 THEN test=s_dim[1] NE s_dim0[1] ELSE test=0
- IF (s_dim[0] NE s_dim0[0]) OR (test) THEN BEGIN
- dimv='['+STRTRIM(s_dim0[0],2)+','+STRTRIM(sdim0[1],2)+']/['+ $
- STRTRIM(s_dim[0],2)+','+STRTRIM(sdim[1],2)+'].'
- STOP_WITH_ERROR,'D_'+proname,errtxt[3]+dimv,lu
- IF STRLEN(rerr[0]) GT 2 THEN reterr=rerr[0] & RETURN
- ENDIF
- ENDELSE
- s_dim0=s_dim
- ENDFOR
-
- ;create Metafile by combining global and variable attributes, also make
- ;accompanying arrays to hold any numeric metadata values
- vi=WHERE(pchk EQ 1,vatot) ;total number of Variable Attribute labels
- metafile=STRARR(n_ga+vatot) & mv_lng=LON64ARR(n_ga+vatot) & mv_dbl=DBLARR(n_ga+vatot)
- pcnt=LONG(n_ga) & mv_str=LONARR(vatot)
- metafile[0:pcnt-1L]=ga
- FOR i=0,s_dim[0]-1 DO BEGIN ;s_dim[0] is the number of datasets
- dset=*sds[i,0].data & dtype=SIZE(dset,/TYPE)
- FOR j=0,s_dim[1]-1 DO BEGIN ;s_dim[1] is the number of attributes
- IF pchk[i,j] EQ 1 THEN BEGIN
- IF pchkv[i,j] EQ 0 THEN metafile[pcnt]=*sds[i,j].va_l+'=' $
- ELSE BEGIN
- vavt=SIZE(*sds[i,j].va_v,/TYPE)
- gti=WHERE(vavt EQ vdtype,g_vavt) ;test to see if the data type is valid
- IF g_vavt EQ 1 THEN BEGIN ;it is OK to transfer the attribute value to a variable
- vav=*sds[i,j].va_v
- n_vav=N_ELEMENTS(vav)
- FOR k=0,n_vav-1 DO BEGIN
- IF k EQ 0 THEN BEGIN
- IF vavt EQ 1 THEN metafile[pcnt]=*sds[i,j].va_l+'='+STRING(FIX(vav[k])) $
- ELSE IF vavt EQ 7 THEN metafile[pcnt]=*sds[i,j].va_l+'='+STRING(vav[k]) $ ;save all values as a string representation
- ELSE metafile[pcnt]=*sds[i,j].va_l+'='+STRTRIM(vav[k],2)
- ENDIF ELSE BEGIN
- IF vavt EQ 1 THEN metafile[pcnt]=metafile[pcnt]+';'+STRING(FIX(vav[k])) $
- ELSE IF vavt EQ 7 THEN metafile[pcnt]=metafile[pcnt]+';'+STRING(vav[k]) $
- ELSE metafile[pcnt]=metafile[pcnt]+';'+STRTRIM(vav[k],2)
- ENDELSE
- ENDFOR
- IF vavt NE 7 THEN BEGIN ;i.e. if the metadata value is not of STRING datatype
-
- ;Do check for VAR_SIZE value written as a numeric type instead of string
- vs_chk=*sds[i,j].va_l
- IF (STRUPCASE(vs_chk) EQ 'VAR_SIZE') AND (infotxt[0,4] EQ '') THEN BEGIN
- ;need to generate infotxt message
- IF qa_yes THEN itxt=' must be of ' ELSE itxt=' changed to '
- infotxt[0,4]='2 VAR_SIZE attribute value(s)'+itxt+'type STRING'
- ENDIF
-
- IF n_vav EQ 1 THEN BEGIN ;save numeric values to the appropriate numeric array (LONG64 or DOUBLE)
- CASE 1 OF
- ((vavt GE 1) AND (vavt LE 3)) OR ((vavt GE 12) AND (vavt LE 14)): mv_lng[pcnt]=vav
- (vavt GE 4) AND (vavt LE 5): mv_dbl[pcnt]=vav
- ELSE:
- ENDCASE
- ENDIF
- ENDIF
- ENDIF ELSE BEGIN ;invalid data type for the attribute label
- IF qa_yes THEN itxt='' ELSE itxt=' and will be ignored'
- val=*sds[i,j].va_l & val=STRTRIM(val,2)
- bli=WHERE(val EQ badlabel,blcnt)
- IF blcnt EQ 0 THEN BEGIN
- badlabel=[badlabel,val]
- IF it5[0] EQ '' THEN it5[0]='2 Value for Attribute Label '+val+' is an invalid Data Type'+itxt $
- ELSE it5=[it5,'2 Value for Attribute Label '+val+' is an invalid Data Type'+itxt]
- ENDIF
- ENDELSE
- ENDELSE
- pcnt=pcnt+1L
- ENDIF
- ENDFOR
- ENDFOR
-ENDELSE
-
-;If necessary open DIALOG_BOXES for the TAV File and Output Directory Inputs
-IF tablefile EQ '' THEN BEGIN
- tablefile=DIALOG_PICKFILE(Filter=['table*.dat','*.dat'], PATH=path, $
- /MUST_EXIST,Title='Select Table Attribute Values File')
- IF tablefile EQ '' THEN BEGIN
- STOP_WITH_ERROR,'D_'+proname,'Table Attribute Values file'+errtxt[0],lu
- IF STRLEN(rerr[0]) GT 2 THEN reterr=rerr[0] & RETURN
- ENDIF
- path=FILE_DIRNAME(tablefile,/MARK_DIRECTORY)
-ENDIF
-IF outdir EQ '' THEN BEGIN
- outdir=DIALOG_PICKFILE(/DIRECTORY,Title='Select Directory for Output '+dftxt+' file(s)', PATH=path)
- IF outdir EQ '' THEN BEGIN
- STOP_WITH_ERROR,'D_'+proname,dftxt+' file(s) output directory'+errtxt[0],lu
- IF STRLEN(rerr[0]) GT 2 THEN reterr=rerr[0] & RETURN
- ENDIF
-ENDIF
-
-;Set dux values according to output options
-IF o3[2] NE '0' THEN BEGIN ;open idlcr8hdf/idlcr8qa.log
- o3[2]=outdir+o3[2]
- res=FILE_TEST(o3[2],/WRITE)
- IF res EQ 0 THEN OPENW,du,o3[2],/GET_LUN $
- ELSE BEGIN
- OPENW,du,o3[2],/Append,/GET_LUN & FOR i=0,1 DO PRINTF,du,''
- ENDELSE
- dux[1]=du & dux[2]=dux[1]-dux[0]
-ENDIF
-FOR i=dux[0],dux[1],dux[2] DO BEGIN
- IF i EQ -1 THEN BEGIN
- PRINT,dftxt+' File Input/Output Log - Program Started on '+SYSTIME(0) & PRINT,''
- ENDIF ELSE IF (i EQ dux[0]) OR ((i EQ dux[1]) AND (STRPOS(o3[2],'idlcr8hdf.log') NE -1)) THEN BEGIN
- PRINTF,i,dftxt+' File Input/Output Log - Program Started on '+SYSTIME(0)
- PRINTF,i,''
- ENDIF
-ENDFOR
-
-IF o3[3] EQ '' THEN BEGIN
- ;Set-up output file display widget and log file
- base=WIDGET_BASE(Title=dftxt+' File Input/Output Log',Units=2,yoffset=3,xoffset=3,Tlb_Frame_Attr=1,/COLUMN) ;,Tab_Mode=1)
- wtxt=WIDGET_TEXT(base,/Scroll,xsize=100,ysize=12)
- base2=WIDGET_BASE(base)
- b1=WIDGET_BUTTON(base2,value='Finish',uvalue='DONE',frame=3,Sensitive=0) ;Accelerator='Return')
- WIDGET_CONTROL,wtxt,/REALIZE
- WIDGET_CONTROL,base,/REALIZE
- WIDGET_CONTROL,wtxt,set_value='',/APPEND
- WIDGET_CONTROL,wtxt,set_value='Input TAV File: '+tablefile,/APPEND
-ENDIF
-
-;Call the procedure to read the TAV file (returns tab_arr)
-FOR i=dux[0],dux[1],dux[2] DO $
- IF i EQ -1 THEN PRINT,'Input TAV File: '+tablefile ELSE PRINTF,i,'Input TAV File: '+tablefile
-READ_TABLEFILE,tablefile
-IF STRLEN(rerr[0]) GT 2 THEN BEGIN
- reterr=rerr[0] & RETURN
-ENDIF
-
-;Check (and write out) any earlier infotxt statements
-FOR i=0,3 DO BEGIN
- IF infotxt[0,i] NE '' THEN BEGIN
- IF i EQ 2 THEN INFOTXT_OUTPUT,infotxt[0,i] ELSE INFOTXT_OUTPUT,infotxt[*,i]
- ENDIF
-ENDFOR
-
-FOR ndf=0,nfile-1 DO BEGIN
- IF o3[3] EQ '' THEN BEGIN
- lineno=lineno+3L
- WIDGET_CONTROL,wtxt,set_value='',/Append
- ENDIF
- FOR i=dux[0],dux[1],dux[2] DO IF i EQ -1 THEN PRINT,'' ELSE PRINTF,i,''
- IF inf EQ 1 THEN BEGIN
- IF o3[3] EQ '' THEN BEGIN
- lineno=lineno+2L
- WIDGET_CONTROL,wtxt,set_value='Input Metadata File: '+metafile,/Append
- WIDGET_CONTROL,wtxt,set_value='Input Data File: '+sds[ndf],/Append
- ENDIF
- FOR i=dux[0],dux[1],dux[2] DO BEGIN
- IF i EQ -1 THEN BEGIN
- PRINT,'Input Metadata File: '+metafile & PRINT,'Input Data File: '+sds[ndf]
- ENDIF ELSE BEGIN
- PRINTF,i,'Input Metadata File: '+metafile
- PRINTF,i,'Input Data File: '+sds[ndf]
- ENDELSE
- ENDFOR
- ENDIF ELSE BEGIN
- IF o3[3] EQ '' THEN $
- WIDGET_CONTROL,wtxt,set_value='Reading input Global Attribute array and Data Structure',/Append
- FOR i=dux[0],dux[1],dux[2] DO BEGIN
- IF i EQ -1 THEN PRINT,'Reading input Global Attribute array and Data Structure' $
- ELSE IF (i EQ dux[0]) OR ((i EQ dux[1]) AND (STRPOS(o3[2],'idlcr8hdf.log') NE -1)) THEN $
- PRINTF,i,'Reading input Global Attribute array and Data Structure'
- ENDFOR
- ;check for numeric VAR_SIZE value message
- IF infotxt[0,4] NE '' THEN INFOTXT_OUTPUT,infotxt[0,4]
-
- ;check for invalid variable attribute value data type
- IF it5[0] NE '' THEN BEGIN
- FOR i=0,N_ELEMENTS(it5)-1 DO INFOTXT_OUTPUT,it5[i]
- ENDIF
-
- ;Do stage 1 of HDF Pre-Defined attributes check (checking units,valid_range,_FillValue only)
- ncsa=['x','x','units','_fillvalue','valid_range','valid_range']
- var_atts=['VAR_NAME','VAR_DATA_TYPE','VAR_UNITS','VAR_FILL_VALUE','VAR_VALID_MIN','VAR_VALID_MAX']
- FOR i=0,s_dim[0]-1 DO BEGIN ;s_dim[0] is the number of datasets
- vdtv='' & vnx='' & verror=INTARR(2)
- FOR k=0,N_ELEMENTS(var_atts)-1 DO BEGIN
- vav='' & nav=''
- FOR j=0,s_dim[1]-1 DO BEGIN ;s_dim[1] is the number of attributes
- IF (pchk[i,j] EQ 1) AND (pchkv[i,j] EQ 1) THEN BEGIN
- CASE 1 OF
- STRUPCASE(STRTRIM(*sds[i,j].va_l,2)) EQ var_atts[k]: vav=*sds[i,j].va_v
- STRLOWCASE(STRTRIM(*sds[i,j].va_l,2)) EQ ncsa[k]: nav=*sds[i,j].va_v
- ELSE:
- ENDCASE
- ENDIF
- ENDFOR
- IF k EQ 0 THEN vnx=STRTRIM(vav[0],2) $ ;VAR_NAME
- ELSE IF k EQ 1 THEN vdtv=STRUPCASE(STRTRIM(vav[0],2)) $ ;VAR_DATA_TYPE
- ELSE IF k GE 4 THEN BEGIN ;separate out min and max valid_range values
- IF ((SIZE(nav[0],/TYPE) EQ 7) AND (STRTRIM(nav[0],2) NE '')) OR (SIZE(nav[0],/TYPE) NE 7) THEN BEGIN
- IF k EQ 4 THEN nav=nav[0] $
- ELSE IF N_ELEMENTS(nav) GT 1 THEN nav=nav[1]
- ENDIF
- vav=vav[0]
- ENDIF ELSE BEGIN
- vav=vav[0] & nav=nav[0]
- ENDELSE
- IF k GE 2 THEN PRE_DEFINED_ATT_CHECKS,k-2,nav,vav,vnx,vdtv,verror
- ENDFOR
- ENDFOR
- ENDELSE
-
- ;Call the procedure to read the metadata (returns meta_arr)
- READ_METADATA,metafile,inf
- IF STRLEN(rerr[0]) GT 2 THEN BEGIN
- reterr=rerr[0] & RETURN
- ENDIF
- ;Call Procedure to check inputs in meta_arr with those from the TAV file
- CHECK_METADATA
- IF STRLEN(rerr[0]) GT 2 THEN BEGIN
- reterr=rerr[0] & RETURN
- ENDIF
- ;Call the procedure to set up the data structure and assign arrays for VAR_NAME and VAR_UNITS
- IF inf EQ 0 THEN SET_UP_STRUCTURE,sds,inf ELSE SET_UP_STRUCTURE,sds[ndf],inf
- IF STRLEN(rerr[0]) GT 2 THEN BEGIN
- reterr=rerr[0] & RETURN
- ENDIF ;Call the procedure to read the data
- IF inf EQ 0 THEN READ_DATA,sds,inf ELSE READ_DATA,sds[ndf],inf
- IF STRLEN(rerr[0]) GT 2 THEN BEGIN
- reterr=rerr[0] & RETURN
- ENDIF
- ;Call the Procedure to construct the HDF output filename based on input metadata.
- IF STRPOS(o3[0],'H5') NE -1 THEN hdffilename='.h5' $
- ELSE IF o3[0] EQ 'NC' THEN hdffilename='.nc' $
- ELSE hdffilename='.hdf'
- FIND_HDF_FILENAME,hdffilename
- IF STRLEN(rerr[0]) GT 2 THEN BEGIN
- reterr=rerr[0] & RETURN
- ENDIF
-
- ;Create HDF file if the program is not performing QA and NOHDF keyword not selected
- IF (~qa_yes) AND (o3[4] EQ '0') THEN BEGIN
- ;Check to see if file exists - if so delete (o/w will append to existing file)
- IF FILE_TEST(hdffilename) EQ 1 THEN FILE_DELETE,hdffilename
-
- ;Call the procedure to write the Global Attributes, Variable Attributes and Data
- ;to an HDF4 or HDF5 file
- geoms_output=o3[0]
- AVDC_HDF_WRITE,hdffilename,geoms_output
- FILE_MOVE, hdffilename, outdir+hdffilename,/Allow_Same,/Overwrite
- hdffilename=outdir+hdffilename
-
- IF o3[3] EQ '' THEN $
- WIDGET_CONTROL,wtxt,set_value=hdffilename+' successfully created!',/Append,Set_text_top_line=lineno
- FOR i=dux[0],dux[1],dux[2] DO BEGIN
- IF i EQ -1 THEN PRINT,hdffilename+' successfully created!' $
- ELSE PRINTF,i,hdffilename+' successfully created!'
- ENDFOR
- ENDIF
- ;Free up memory by destroying the heap variables pointed at by its pointer arguments
- PTR_FREE,ds.data
-ENDFOR
-
-IF qa_yes THEN endtxt=dftxt+' QA function ' $
-ELSE IF o3[4] EQ '0' THEN endtxt=dftxt+' file creation ' $
-ELSE endtxt='GEOMS file testing '
-
-FOR i=dux[0],dux[1],dux[2] DO BEGIN
- IF i EQ -1 THEN BEGIN
- PRINT,'' & PRINT,endtxt+'completed - Program Ended on '+SYSTIME(0)
- ENDIF ELSE IF (i EQ dux[0]) OR ((i EQ dux[1]) AND (STRPOS(o3[2],'idlcr8hdf.log') NE -1)) THEN BEGIN
- PRINTF,i,'' & PRINTF,i,endtxt+'completed - Program Ended on '+SYSTIME(0)
- ENDIF
-ENDFOR
-IF dux[1] NE dux[0] THEN FREE_LUN,dux[1]
-
-ri=WHERE(rerr EQ '',rcnt)
-IF (rcnt EQ N_ELEMENTS(rerr)) AND (~qa_yes) AND (o3[4] EQ '0') THEN BEGIN
- IF nfile EQ 1 THEN reterr=hdffilename+' successfully created' $
- ELSE reterr=dftxt+' files successfully created'
-ENDIF ELSE IF (rerr[1] NE 'NA') AND (rerr[1] NE '') THEN reterr=rerr[1]
-
-IF o3[3] EQ '' THEN BEGIN
- WIDGET_CONTROL,b1,Sensitive=1,/Input_Focus
- WIDGET_CONTROL,wtxt,set_value='',/Append,Set_text_top_line=lineno+2L
- WIDGET_CONTROL,wtxt,set_value=endtxt+'completed - hit <Finish> to close program',/Append
- XMANAGER,'idlcr8hdf',base
-ENDIF ELSE IF intype LT 0 THEN BEGIN ;Create Finish Dialog Box
- res=DIALOG_MESSAGE(endtxt+'completed successfully!',/Information,Title='EVDC IDLcr8HDF')
-ENDIF
-
-END ;Procedure IDLcr8HDF
+;Main Program Version: idlcr8hdf.pro v4.0b65, 20241029
+; Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
+;
+;Sub-versions (refer to idlcr8hdf-v4.0_Readme.pdf for full history)
+
+PRO idlcr8hdf_common
+;Procedure to define the COMMON blocks for this program
+; ----------
+;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
+;
+;History:
+; 20050909: Introduced to IDLCR8HDF - Version 1.1
+; 20061012: Variable 'ncsa' added to METADATA; 'iarr', 'larr', 'rarr', 'darr' holding Arrays
+; removed from DATA and replaced with Structure 'ds'; Variable 'vfv' added to DATA;
+; WIDGET_WIN added for common variables associated with the Graphical User Interface
+; - Version 2.0
+; 20080302: tab_type integer added to TABLEDATA - Version 3.0
+; 20100205: rerr string added to WIDGET_WIN - Version 3.09
+; 20110401: vnchange added to DATA; mv_lng and mv_dbl added to METADATA; vserror added to DATA;
+; vfv removed from DATA; ncsa removed from METADATA - Version 4.0b1
+; 20150127: mv_str added to METADATA to hold maximum string length of string dataset entries -
+; Version 4.0b25
+; 20151012: Rename free_attr to attr_free and add qa_yes - Version 4.0b31
+;
+;Input: Nil
+;
+;Output: Nil
+;
+;Called by: N/A
+;
+;Subroutines Called: None
+
+COMMON TABLEDATA, tab_arr,tab_ver,tab_type
+COMMON METADATA, meta_arr,attr_arr_glob,attr_arr_data,attr_free,mv_lng,mv_dbl,mv_str
+COMMON DATA, ds,ndm,nvn,vn,vu,vnchange,vserror,qa_yes
+COMMON WIDGET_WIN, wtxt,b1,lineno,base,o3,dux,rerr,lvals
+
+END ;Procedure idlcr8hdf_common
+
+
+
+PRO intro_event, ev
+;Procedure to define how Events from the Start-up Introduction Window are handled
+; ----------
+;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
+;
+;History:
+; 20061012: Introduced to IDLCR8HDF - Version 2.0
+;
+;Input: ev - Selected widget event structure
+;
+;Output: o3 - Common string array defining the various options for program output
+;
+;Called by: XMANAGER in INTRO
+;
+;Subroutines Called: None
+
+COMMON WIDGET_WIN
+
+;The uservalue is retrieved from a widget when an event occurs
+WIDGET_CONTROL,ev.id,GET_UVALUE=uv
+;Assign/Remove AVK button event to/from a variable name
+IF uv EQ 'AVK' THEN IF o3[1] EQ uv THEN o3[1]='0' ELSE o3[1]=uv $
+;Assign/Remove Log Output button event to/from a variable name
+ELSE IF uv EQ 'idlcr8hdf.log' THEN IF o3[2] EQ uv THEN o3[2]='0' ELSE o3[2]=uv $
+;Assign/Remove Pop-up window for Log Output button event to/from a variable name
+ELSE IF uv EQ 'Pop' THEN IF o3[3] EQ uv THEN o3[3]='0' ELSE o3[3]=uv $
+;Assign button event to a variable name
+ELSE IF (uv EQ 'H4') OR (uv EQ 'H5') OR (uv EQ 'NC') THEN BEGIN
+ o3[0]=uv
+ IF uv EQ 'H5' THEN BEGIN
+ WIDGET_CONTROL,ev.id+1,Sensitive=1
+ WIDGET_CONTROL,ev.id+2,Sensitive=1,Set_Combobox_Select=0
+ ENDIF
+ENDIF ELSE IF uv EQ '0' THEN IF ev.str EQ 'None' THEN o3[0]='H5' ELSE o3[0]='H5_'+STRTRIM(ev.str,2) $
+ELSE o3[0]='0' ;Cancel button chosen
+IF (uv NE 'AVK') AND (uv NE 'idlcr8hdf.log') AND (uv NE 'Pop') AND (uv NE 'H5') THEN WIDGET_CONTROL,ev.top,/DESTROY
+
+END ;Intro_Event
+
+
+
+PRO intro, intype
+;Procedure which creates an Introduction Window at start-up when IDLCR8HDF is called without parameters,
+;or invalid parameters, or is called by IDL Virtual Machine. The user has a choice of continuing with
+;the program after selecting input options, or stopping the program.
+; ----------
+;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
+;
+;History:
+; 20061012: Introduced to IDLCR8HDF - Version 2.0
+; 20080302: Swapped the order of the option windows, so that the options indicated by the check
+; boxes appear above the option boxes that close the Introduction Window and continue
+; with the program - Version 3.0
+; 20100205: Include text regarding the RETERR argument which can be accepted as an input
+; parameter when using the full version of idlcr8hdf - Version 3.09
+; 20110401: Add vertxt string array to hold text which changes between versions of the code;
+; Change text to reference GEOMS compliant files instead of AVDC/EVDC/NDACC; Change
+; text regarding the format of the structure required for input - Version 4.0b1
+; 20111220: Change text and options to include netCDF; change contact details - Version 4.0b7
+; 20130114: Make insensitive /LOG and /NC options if the program is called in IDL DEMO mode
+; -Version 4.0b14
+;
+;Input: intype - Integer set to -1 or -2: -1 indicates normal state; -2 indicates that input
+; parameters at the IDLCR8HDF call were invalid.
+;
+;Output: Nil
+;
+;Called by: IDLCR8HDF
+;
+;Subroutines Called: INTRO_EVENT (via XMANAGER)
+
+COMMON WIDGET_WIN
+
+nhdr=44 & errtxt=STRARR(nhdr)
+vertxt=['idlcr8hdf-v4.0_Readme.pdf','v4.0b65 October 2024']
+errtxt[1]='Welcome to IDLcr8HDF. This program creates GEOMS compliant HDF4, HDF5 and netCDF files'
+errtxt[2]='(also refer to '+vertxt[0]+').'
+errtxt[4]='Inputs to the program (IDL Virtual Machine (VM) and IDL Licensed (LIC) Versions):'
+errtxt[5]=' METADATA TEMPLATE FILE - Containing the Global and Variable Attribute information for the site.'
+errtxt[6]=' DATA FILE(s) - Multiple selection permitted. The file(s) must show the datasets in the single column'
+errtxt[7]=' format described in the documentation.'
+errtxt[8]=' TAV FILE - the current non-encrypted Table Attribute Values file.'
+errtxt[9]=' OUTPUT DIRECTORY - Directory for any HDF, netCDF and log files created. Shortcut values of ''M'' or ''D'''
+errtxt[10]=' will output files to the Metadata or Data file directories respectively.'
+errtxt[12]='Alternative Input Option for IDL LIC Versions:'
+errtxt[13]=' GLOBAL ATTRIBUTES ARRAY (GA) - a string array containing the Global Attributes in the form'
+errtxt[14]=' ''label=value''.'
+errtxt[15]=' DATA STRUCTURE (DS) - a heap structure using pointers containing the Variable Attribute Labels'
+errtxt[16]=' (DS.VA_L), the Variable Attribute Values (DS.VA_V), and the Data (DS.Data), for a single output file.'
+errtxt[17]=' TAV FILE - the current non-encrypted Table Attribute Values file.'
+errtxt[18]=' OUTPUT DIRECTORY - Directory for any HDF, netCDF and log files created.'
+errtxt[19]=' RETERR - String variable to which any error(s) are written.'
+errtxt[21]='For IDL VM, input is by ''DIALOG_BOXES''. For IDL LIC, input can be by ''DIALOG_BOXES'' or passed'
+errtxt[22]='by calling the program with one of the following command line options:'
+errtxt[23]=' 1. idlcr8hdf (Opens this box, and allows the user the option to continue with file inputs).'
+errtxt[24]=' 2. idlcr8hdf,METAFILE,DATAFILE(s),TAVFILE,OUTDIR[,RETERR][,/H5][,/Cn][,/NC][,/AVK][,/Log][,/Popup] or'
+errtxt[25]=' idlcr8hdf,'''','''','''',''''[,/H5][,/Cn][,/NC][,/AVK][,/Log][,/Popup] (For null string, DIALOG_BOXES will prompt for input).'
+errtxt[26]=' 3. idlcr8hdf,GA,DS,TAVFILE,OUTDIR[,RETERR][,/H5][,/Cn][,/NC][,/AVK][,/Log][,/Popup] or
+errtxt[27]=' idlcr8hdf,GA,DS[,/H5][,/Cn][,/NC][,/AVK][,/Log][,/Popup] (Inputs are from session memory, DIALOG_BOX(s) will'
+errtxt[28]=' prompt for input if TAVFILE or OUTDIR are not included).'
+errtxt[29]=' /H5, /Cn, /NC, /AVK, /Log and /Popup keywords are used in place of the options given below (HDF4 is default).'
+errtxt[30]='The /Cn keyword enables compression and shuffling of HDF5 files only (C1=low, C9=high). Default is no compression.'
+errtxt[32]='Contacts -'
+errtxt[33]=' Ian Boyd, Bryan Scientific Consulting LLC (iboyd@bryanscientific.org)'
+errtxt[34]=' 9 Cambridge Terrace'
+errtxt[35]=' Masterton 5810, New Zealand'
+errtxt[37]=' Ann Mari Fjaeraa, EVDC Project Manager (amf@nilu.no)'
+errtxt[38]=' Norwegian Institute for Air Research, Instituttveien 18'
+errtxt[39]=' Postbox 100, N-2027 KJELLER, NORWAY'
+errtxt[41]='EVDC Website: Tools and documentation available from http://evdc.esa.int/'
+errtxt[43]='To continue, please choose from the options below (Note: HDF5 only available on IDL6.2 or greater).'
+errtxt=' '+errtxt
+
+;Set-up text display widget
+IF intype EQ -2 THEN xtxt=' - Command Line Input Error' ELSE xtxt=''
+IF intype EQ -3 THEN optsens=0 ELSE optsens=1
+base=WIDGET_BASE(Title='idlcr8hdf '+vertxt[1]+xtxt,Tlb_Frame_Attr=1,/Column) ;,Tab_Mode=1)
+wtxt=WIDGET_TEXT(base,xsize=102,ysize=25,/Scroll)
+base3=WIDGET_BASE(base,/Nonexclusive)
+logtext='Append log output to the file ''idlcr8hdf.log'' '
+IF intype EQ -3 THEN logtext=logtext+'(No log or netCDF file output permitted in IDL DEMO Mode)' $
+ELSE logtext=logtext+'(will create the file IF it doesn''t exist)'
+poptext='Open a Pop-Up Window to display log output'
+avktext='If Avg. Kernel data are present, append sentence to VAR_NOTES giving first three values of '
+avktext=avktext+'the first kernel'
+avktip='Sentence reads ''First three values of the first kernel are: x.xx x.xx x.xx'''
+b4=WIDGET_BUTTON(base3,value=logtext,uvalue='idlcr8hdf.log',frame=3,SENSITIVE=optsens)
+b5=WIDGET_BUTTON(base3,value=poptext,uvalue='Pop',frame=3)
+b6=WIDGET_BUTTON(base3,value=avktext,uvalue='AVK',frame=3) ;,Tooltip=AVKTip)
+base2=WIDGET_BASE(base,/Row)
+tip='Left Mouse Click or Tab to entry and hit <Spacebar>'
+b1=WIDGET_BUTTON(base2,value='HDF4',uvalue='H4',frame=3) ;,Tooltip=Tip)
+b7=WIDGET_BUTTON(base2,value='netCDF3',uvalue='NC',frame=3,SENSITIVE=optsens)
+IF FLOAT(!Version.Release) GE 6.2 THEN $
+ b2=WIDGET_BUTTON(base2,value='HDF5',uvalue='H5',frame=3) $;,Tooltip=Tip) $
+ELSE b2=WIDGET_BUTTON(base2,value='HDF5',Sensitive=0,frame=3)
+b8=Widget_Label(base2,Value=' Compression ',Sensitive=0)
+b9=Widget_Combobox(base2,Value=lvals,Sensitive=0,uvalue='0')
+
+b3=WIDGET_BUTTON(base2,value='Stop',uvalue='CANCEL',frame=3) ;,ToolTip=Tip)
+WIDGET_CONTROL,base,/Realize
+WIDGET_CONTROL,b4,/Input_Focus
+FOR i=0,N_ELEMENTS(errtxt)-1 DO $
+ WIDGET_CONTROL,wtxt,set_value=errtxt[i],/Append
+XMANAGER,'intro',base
+
+END ;Intro
+
+
+
+PRO idlcr8hdf_event, ev
+;Procedure to close the pop-up logging window after user selects 'Finish'.
+; ----------
+;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
+;
+;History:
+; 20061012: Introduced to IDLCR8HDF - Version 2.0
+;
+;Input: Selected widget event structure
+;
+;Output: Nil
+;
+;Called by: XMANAGER in IDLCR8HDF and STOP_WITH_ERROR
+;
+;Subroutines Called: None
+
+WIDGET_CONTROL,ev.top,/DESTROY
+RETALL & HEAP_GC
+
+END ;IDLcr8HDF_Event
+
+
+
+FUNCTION is_a_number_hdf, value, ind
+
+IF N_PARAMS() EQ 2 THEN BEGIN
+ ;Check each individual character is numeric
+ nlen=STRLEN(value)
+ retval=1B
+ FOR i=0,nlen-1 DO BEGIN
+ ah=STRMID(value,i,1)
+ isan=(BYTE(ah) GE 48) AND (BYTE(ah) LE 57) ;0-9
+ IF ~isan THEN retval=0B
+ ENDFOR
+ RETURN, retval
+ENDIF ELSE BEGIN
+ ON_IOERROR, ConversionError
+ IF STRTRIM(value,2) EQ '' THEN RETURN, 0B
+ n=DOUBLE(value)
+ RETURN, 1B
+ ConversionError:
+ RETURN, 0B
+ENDELSE
+END
+
+
+
+PRO stop_with_error, txt1, txt2, lu
+;Procedure called when an error in the program inputs is detected. An error message is displayed
+;and the program stopped and reset. If necessary, open files are closed, and memory associated
+;with a structure is freed. The error message is displayed in one or more of the following:
+;a Pop-up dialog window; the Pop-up logging window; the Output Logging window in the IDLDE;
+;a log file (idlcr8hdf.log)
+; ----------
+;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
+;
+;History:
+; 20050802: Original IDLCR8HDF Routine - Version 1.0
+; 20061012: Set-up so that the error output is displayed in the output window(s) dependent on
+; the options chosen by the user, the point in the program that the error is detected,
+; and the method that IDLCR8HDF is called. If txt1 is preceeded by 'D_' or is null,
+; the error output is to a Pop-up Dialog window. Other error output options are
+; determined by the values in the (Common) dux array - Version 2.0
+; 20100205: Allow routine to return to the calling routine, rather than stop the application, if
+; a 'reterr' argument is included in the call to idlcr8hdf - Version 3.09
+; 20110401: Change end text dependent on whether the program is creating a GEOMS file or doing QA
+; on a GEOMS file - Version 4.0b1
+; 20111220: Change text referring to HDF to GEOMS - Version 4.0b7
+; 20171121: Add comment advising that program stopped before all checks were completed if doing
+; QA; Add COMMON DATA to routine and remove ds from the list of input parameters
+; - Version 4.0b43
+;
+;Inputs: txt1 - the initial text line of the error message. Generally contains the name of the routine
+; where the error was detected.
+; txt2 - the second text line which generally describes the error state.
+; lu - Where applicable, the file unit that needs to be closed at the termination of the program,
+; otherwise set to -1.
+; ds - Where applicable, the name of the data structure set-up by the program (dependent on
+; the point in the program that the error is detected), so that memory associated with the
+; structure can be freed (COMMON variable).
+; dux - an integer array used to determine the output options for the error message (COMMON) variable).
+; qa_yes - a boolean to indicate whether idlcr8hdf called in QA mode or not (COMMON variable).
+;
+;Output: Error message displayed/output dependent on the requested output options.
+;
+;Called by: The routine in which the error was detected. The following routines call STOP_WITH_ERROR:
+; READ_TABLEFILE; TEST_FILE_INPUT; READ_METADATA; EXTRACT_AND_TEST; CHECK_METADATA;
+; SET_UP_STRUCTURE; CHECK_MIN_MAX_FILL; EXTRACT_DATA; FIND_HDF_FILENAME; IDLCR8HDF.
+;
+;Subroutines Called: IDLCR8HDF_EVENT (via XMANAGER)
+
+COMMON DATA
+COMMON WIDGET_WIN
+
+IF lu NE -1L THEN FREE_LUN,lu
+
+IF txt1 EQ '' THEN BEGIN ;<cancel> chosen on Intro box
+ res=DIALOG_MESSAGE('IDLcr8HDF Stopped!',/Information,Title='EVDC IDLcr8HDF')
+ENDIF ELSE BEGIN
+ ;If necessary free up memory by destroying the heap variables pointed at by its pointer arguments
+ IF N_ELEMENTS(ds) NE 0 THEN PTR_FREE,ds.data
+ ;Write error to file and/or IDL output window
+ IF STRMID(txt1,0,2) EQ 'D_' THEN txtx=STRMID(txt1,2) ELSE txtx=txt1
+ FOR i=dux[0],dux[1],dux[2] DO BEGIN
+ IF i EQ -1 THEN BEGIN
+ PRINT,' ERROR in '+txtx & PRINT,' '+txt2
+ PRINT,'' & PRINT,'IDLcr8HDF stopped - Program Ended on '+SYSTIME(0)
+ ENDIF ELSE BEGIN
+ PRINTF,i,' ERROR in '+txtx & PRINTF,i,' '+txt2
+ IF (i EQ dux[0]) OR ((i EQ dux[1]) AND (STRPOS(o3[2],'idlcr8hdf.log') NE -1)) THEN BEGIN
+ PRINTF,i,'' & PRINTF,i,'IDLcr8HDF stopped - Program Ended on '+SYSTIME(0)
+ ENDIF ELSE IF qa_yes THEN BEGIN
+ PRINTF,i,'' & PRINTF,i,' INFORMATION: QA stopped before all checks could be completed'
+ ENDIF
+ ENDELSE
+ ENDFOR
+ IF dux[1] NE dux[0] THEN FREE_LUN,dux[1]
+ IF (STRMID(txt1,0,2) EQ 'D_') AND (rerr[0] EQ 'NA') THEN BEGIN
+ ;write error to DIALOG Box instead of Popup window
+ errtxt2=STRARR(4)
+ errtxt2[0]=STRMID(txt1,2) & errtxt2[1]=txt2 & errtxt2[3]='IDLcr8HDF Stopped!'
+ res=DIALOG_MESSAGE(errtxt2,/Error,Title='EVDC IDLcr8HDF Error')
+ ENDIF ELSE IF rerr[0] EQ 'NA' THEN BEGIN ;write error to Popup window
+ IF o3[4] EQ '0' THEN endtxt='GEOMS file creation ' $
+ ELSE endtxt='GEOMS file testing '
+ lineno=lineno+4L
+ WIDGET_CONTROL,wtxt,set_value=' ERROR in '+txt1,/Append
+ WIDGET_CONTROL,wtxt,set_value=' '+txt2,/Append
+ WIDGET_CONTROL,wtxt,set_value='',/Append,Set_text_top_line=lineno
+ WIDGET_CONTROL,wtxt,set_value=endtxt+'stopped - hit <Finish> to close program',/Append
+ WIDGET_CONTROL,b1,Sensitive=1,/Input_Focus
+ XMANAGER,'stop_with_error',base,Event_Handler='idlcr8hdf_event'
+ ENDIF
+ENDELSE
+HEAP_GC
+IF rerr[0] EQ 'NA' THEN RETALL $
+ELSE BEGIN
+ IF o3[4] EQ '0' THEN endtxt='create GEOMS file' $
+ ELSE endtxt='complete GEOMS file test'
+ rerr[0]='Unable to '+endtxt+' - '+txtx+txt2
+ENDELSE
+
+END ;Procedure Stop_With_Error
+
+
+
+PRO infotxt_output, infotxt
+;Procedure called when the program makes a change to the input meta data or reports information
+;relevant to the creation of the HDF/netCDF file. This information can be reported to a Pop-up logging
+;window, IDLDE output log window, and/or the log file. Code for this output was originally
+;written directly in the affected procedures
+; ----------
+;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
+;
+;History:
+; 20090311: Introduced to IDLCR8HDF - Version 3.06
+; 20110401: infotxt INFORMATION header changed to a number, to allow changes to the comment
+; header depending on how idlcr8hdf has been called, as follows: 0 = INFORMATION;
+; 1 = NON-STANDARD COMPLIANCE NOTIFICATION (QA) o/w INFORMATION; 2 = ERROR (QA)
+; o/w INFORMATION; 3 = ERROR; 4 = DEBUG - Version 4.01
+;
+;Inputs: infotxt - the text line(s) of the information message. Can be either a scalar string
+; or string array
+;
+;Output: Message displayed/output dependent on the requested output options
+;
+;Called by: The routine in which the reported change was made. The following routines call
+; INFOTXT_OUTPUT: READ_TABLEFILE; GEOMS_RULE_CHANGES; PRE_DEFINED_ATT_CHECKS;
+; READ_METADATA; EXTRACT_AND_TEST; CHECK_METADATA; EXTRACT_DATA; SET_UP_STRUCTURE;
+; CHECK_STRING_DATATYPE; CHECK_MIN_MAX_FILL; READ_DATA; FIND_HDF_FILENAME;
+; AVDC_HDF_WRITE; IDLCR8HDF
+;
+;Subroutines Called: None
+
+COMMON DATA
+COMMON WIDGET_WIN
+
+dm=SIZE(infotxt,/N_ELEMENTS)
+qaval=FIX(STRMID(infotxt[0],0,1))
+write_rerr=0 ;Boolean to generate a return error message
+
+IF qa_yes THEN BEGIN ;program called in QA mode
+ ;Add correct message title
+ CASE 1 OF
+ qaval EQ 0: infotxt[0]=' INFORMATION:'+STRMID(infotxt[0],1)
+ qaval EQ 1: infotxt[0]=' NON-STANDARD COMPLIANCE NOTIFICATION:'+STRMID(infotxt[0],1)
+ qaval EQ 4: infotxt[0]=' DEBUG:'+STRMID(infotxt[0],1)
+ ELSE: infotxt[0]=' ERROR:'+STRMID(infotxt[0],1)
+ ENDCASE
+ ;truncate message from the '|'
+ bs_found=0
+ FOR n=0,dm-1 DO BEGIN
+ IF bs_found EQ 1 THEN infotxt[n]='' $
+ ELSE BEGIN
+ bspos=STRPOS(infotxt[n],'|')
+ IF bspos NE -1 THEN BEGIN
+ infotxt[n]=STRMID(infotxt[n],0,bspos)
+ bs_found=1
+ ENDIF
+ ENDELSE
+ ENDFOR
+ ;recalculate number of good infotxt values
+ gi=WHERE(infotxt NE '',dm)
+ENDIF ELSE BEGIN ;program called in HDF file create mode
+ CASE 1 OF
+ qaval EQ 3: BEGIN
+ infotxt[0]=' ERROR:'+STRMID(infotxt[0],1)
+ IF rerr[1] NE 'NA' THEN BEGIN ;generate return error message
+ IF rerr[1] EQ '' THEN BEGIN
+ IF o3[4] EQ '0' THEN endtxt='create GEOMS file' $
+ ELSE endtxt='complete GEOMS file test'
+ ENDIF
+ write_rerr=1
+ ENDIF
+ o3[4]='NOHDF' ;Error in Input so do not create HDF file
+ END
+ qaval EQ 4: infotxt[0]=' DEBUG:'+STRMID(infotxt[0],1)
+ ELSE: infotxt[0]=' INFORMATION:'+STRMID(infotxt[0],1)
+ ENDCASE
+ ;remove '|' from the message
+ FOR n=0,dm-1 DO BEGIN
+ bspos=STRPOS(infotxt[n],'|')
+ IF bspos NE -1 THEN BEGIN
+ infotxt[n]=STRMID(infotxt[n],0,bspos)+STRMID(infotxt[n],bspos+1)
+ ENDIF
+ ENDFOR
+ENDELSE
+
+lineno=lineno+dm
+FOR n=0,dm-1 DO BEGIN
+ IF o3[3] EQ '' THEN WIDGET_CONTROL,wtxt,set_value=infotxt[n],/Append,Set_text_top_line=lineno
+ FOR m=dux[0],dux[1],dux[2] DO BEGIN
+ IF m EQ -1 THEN PRINT,infotxt[n] ELSE PRINTF,m,infotxt[n] ;write out to log
+ ENDFOR
+ENDFOR
+
+IF write_rerr EQ 1 THEN BEGIN
+ IF rerr[1] EQ '' THEN rerr[1]='Unable to '+endtxt+' -'+STRMID(infotxt[0],8) $
+ ELSE rerr[1]=rerr[1]+';'+STRMID(infotxt[0],8)
+ IF dm GT 1 THEN FOR n=1,dm-1 DO rerr[1]=rerr[1]+STRMID(infotxt[n],3)
+ENDIF
+
+END ;Procedure InfoTxt_Output
+
+
+
+FUNCTION jdf_2_datetime, jdf, MJD2000=mjd2000, SHORTISO8601=iso, LONGISO8601=isoms
+;Computes the UT date/time from JDF/MJD2000.
+; ----------
+; Bojan R. Bojkov
+; bojan.r.bojkov@nasa.gov
+; 03/04/2004
+; 03/11/2004 bug fix
+; 05/27/2005 add ShortISO8601 and LongISO8601 switches,
+; and change seconds value to decimal seconds (Ian Boyd)
+;
+; References ----------
+; Explan. Supp. Astron. Almanac, p.604 (1992); through E. Celarier
+;
+; Caveats ----------
+; Valid for all YYYY >= -4712 (i.e. for all JD .ge. 0)
+; The true Gregorian calendar was only adopted on 15 October 1582,
+; so any dates before this are "virtual" Gregorian dates.
+;
+; Input ----------
+; jdf : double precision value
+; mjd2000 : flag if input is MJD2000
+; iso : flag if output is in ISO8601 (YYYYMMDDThhmmssZ)
+; isoms : flag if output is in ISO8601ms (YYYYMMDDThhmmss.sssZ)
+;
+; Output ---------
+; floating point array [YYYY, MM, DD, hh, mn, ss.sss],
+; or ISO8601 string format
+; External subroutines ---------
+; NONE
+
+jdf=DOUBLE(jdf)
+
+j0=2451544.5D
+IF KEYWORD_SET(mjd2000) THEN jdhold=jdf+j0 ELSE jdhold=jdf
+jdi=LONG(jdhold)
+df=jdhold-jdi
+
+;Determine hh, mm, ss, ms
+hh=(df+0.5)*24.D
+q=WHERE(hh GE 24.D)
+IF q[0] NE -1 THEN BEGIN
+ hh[q]=hh[q]-24.D
+ jdi[q]=jdi[q]+1
+ENDIF
+t1=hh
+hh=LONG(t1)
+t2=(t1-hh)*60.D
+mn=LONG(t2)
+t3=(t2-mn)*60.D
+ss=t3
+;ss=LONG(t3)
+;ms=LONG((t3-ss)*1.d3)
+
+;Determine YYYY, MM, DD
+t1=jdi+68569L
+t2=(4*t1)/146097L
+t1=t1-(146097L*t2+3L)/4L
+t3=(4000L*(t1+1L))/1461001L
+t1=t1-(1461L*t3)/4L + 31L
+t4=(80L*t1)/2447L
+
+dd=t1-(2447L*t4)/80L
+t1=t4/11L
+mm=t4+2L-12L*t1
+yyyy=100L*(t2-49L)+t3+t1
+
+dt=TRANSPOSE([[yyyy],[mm],[dd],[hh],[mn],[ss]])
+IF (KEYWORD_SET(iso)) OR (KEYWORD_SET(isoms)) THEN BEGIN
+ dts=STRARR(6)
+ IF (KEYWORD_SET(iso)) AND (ss-FIX(ss) GE 0.5) THEN BEGIN
+ ;recalculate to get correct seconds value
+ jdhold=jdf+0.000008D ;add ~0.7 secs
+ IF KEYWORD_SET(mjd2000) THEN jdhold=jdhold+j0
+ CALDAT,jdhold,mm,dd,yyyy,hh,mn,ss
+ dt=TRANSPOSE([[yyyy],[mm],[dd],[hh],[mn],[ss]])
+ ENDIF
+ dt=LONG(dt)
+ FOR i=1,5 DO $
+ IF dt[i] LT 10L THEN dts[i]='0'+STRTRIM(dt[i],2) ELSE dts[i]=STRTRIM(dt[i],2)
+ IF KEYWORD_SET(isoms) THEN BEGIN
+ ssd=STRING(format='(f6.3)',ss)
+ IF FLOAT(ssd) LT 10.0 THEN dts[5]='0'+STRTRIM(ssd,2) ELSE dts[5]=STRTRIM(ssd,2)
+ ENDIF
+ RETURN,STRTRIM(dt[0],2)+dts[1]+dts[2]+'T'+dts[3]+dts[4]+dts[5]+'Z'
+ENDIF ELSE RETURN, dt
+
+END ;Function jdf_2_datetime
+
+
+
+FUNCTION julian_date, date_ut, ISO8601=iso, MJD2000=mjd2000
+;Computes the Julian date (jd) or date in MJD2000 format in double precision.
+; ----------
+; Bojan R. Bojkov
+; bojan.r.bojkov@nasa.gov
+; 06/03/2001
+; 09/24/2001: Code cleanup - Function Version 1.0
+; 10/16/2001: Mode corrections - Function Version 1.01
+; 03/07/2002: Comment cleanup and variable change - Function Version 2.0
+; 10/08/2002: Added jdf option - Function Version 2.1
+; 07/21/2003: Converted to a function - Function Version 3.0
+; 05/25/2005: Added ISO8601 and MJD2000 keywords.
+; Input can either be a string or a numeric array
+; (see below for input details). Returns -99999.d
+; if the Input is invalid - Ian Boyd
+;
+; References ----------
+; Hughes, D.W., Yallop, B.D. and Hohenkerk, C.Y., "The Equation of Time",
+; Mon. Not. R. astr. Soc., 238, pp 1529-1535. 1989.
+;
+; Results verified using:
+; Meeus, J, "Astronomical Algorithms", William-Bell, Inc., Richmond VA,
+; p62, 1991.
+;
+; Caveats ----------
+; NONE
+;
+; Input ----------
+; date_ut: Numeric array (UT: YYYY, MM, DD, [hh, mm, ss]), or if ISO8601
+; keyword set, a string containing datetime as YYYYMMDDThhmmssZ
+;
+; Output ---------
+; jd: julian day, or if MJD2000 keyword set,
+; a double precision day in MJD2000 format
+;
+; External subroutines ---------
+; NONE
+
+ON_IOERROR,TypeConversionError
+valid=0 ;Type conversion check
+inputerror=0 ;Check that input is as expected
+;Check input format
+IF KEYWORD_SET(iso) THEN BEGIN ;input is YYYYMMDDThhmmssZ
+ IF STRLEN(date_ut) EQ 16 THEN BEGIN
+ ;test for numeric values (will return type conversion error value if not a number)
+ FOR i=0,7 DO check=FIX(STRMID(date_ut,i,1))
+ FOR i=9,14 DO check=FIX(STRMID(date_ut,i,1))
+ dt=INTARR(6)
+ dt[0]=FIX(STRMID(date_ut,0,4)) & dt[1]=FIX(STRMID(date_ut,4,2))
+ dt[2]=FIX(STRMID(date_ut,6,2)) & dt[3]=FIX(STRMID(date_ut,9,2))
+ dt[4]=FIX(STRMID(date_ut,11,2)) & dt[5]=FIX(STRMID(date_ut,13,2))
+ date_ut=dt & achk=[0,6] ;to add hhmmss component
+ ENDIF ELSE inputerror=1
+ENDIF ELSE BEGIN
+ ;check that input contains at least YMD (and at most YMDhms) information and is either
+ ;an integer, long, float or double array
+ achk=SIZE(date_ut)
+ IF (achk[1] LT 3) OR (achk[1] GT 6) OR (achk[2] LT 2) OR (achk[2] GT 5) THEN inputerror=1
+ENDELSE
+
+IF inputerror EQ 0 THEN BEGIN ;can perform JD calculation
+ IF date_ut[1] GT 2 THEN BEGIN
+ y=DOUBLE(date_ut[0])
+ m=DOUBLE(date_ut[1]-3)
+ d=DOUBLE(date_ut[2])
+ ENDIF ELSE BEGIN
+ y=DOUBLE(date_ut[0]-1)
+ m=DOUBLE(date_ut[1]+9)
+ d=DOUBLE(date_ut[2])
+ ENDELSE
+
+ ;Compute Julian date:
+ j=LONG(365.25D0*(y+4712.D0))+LONG(30.6D0*m+0.5D0)+59.D0+d-0.5D0
+
+ ;Check for Julian or Gregorian calendar:
+ IF j LT 2299159.5D0 THEN jd=j $ ;If Julian calendar.
+ ELSE BEGIN ;If Gregorian calendar.
+ gn=38.D0-LONG(3.D0*LONG(49.D0+y/100.D0)/4.D0)
+ jd=j+gn
+ ENDELSE
+
+ ;add hhmmss values if present
+ CASE 1 OF
+ achk[1] EQ 4: jd=jd+DOUBLE(date_ut[3])/24.D ;hh only
+ achk[1] EQ 5: jd=jd+(DOUBLE(date_ut[3])*3600.D + DOUBLE(date_ut[4])*60.D)/86400.D ;hhmm only
+ achk[1] EQ 6: jd=jd+(DOUBLE(date_ut[3])*3600.D + DOUBLE(date_ut[4])*60.D + $
+ DOUBLE(date_ut[5]))/86400.D ;hhmmss
+ ELSE:
+ ENDCASE
+
+ valid=1 ;Type conversion OK
+
+ ;Set to MJD2000 if required
+ IF KEYWORD_SET(mjd2000) THEN jd=jd-2451544.5D
+ENDIF
+
+TypeConversionError:
+IF valid EQ 0 THEN RETURN,-99999.D ELSE RETURN, jd
+
+END ;Function Julian_Date
+
+
+
+PRO var_units_test, vuvalue, rd, tab_type, var_si_conv, errstate
+;Procedure to perform checks on the VAR_UNITS value in the metadata and, based on the VAR_UNITS input,
+;calculate and return the VAR_SI_CONVERSION value.
+; ----------
+;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
+;
+; History:
+; 20061012: Introduced to IDLCR8HDF - Version 2.0
+; (Previously these checks were performed in the Extract_and_Test routine)
+; 20080302: var_unit_arr and unit_pre_arr added which hold, respectively, the valid VAR_UNITS and
+; corresponding VAR_SI_CONVERSION values, and the set of UNIT_PREFIXs (previously
+; values from the EVDC/AVDC TAV file have been used). This has been done so that the
+; routine can be stand-alone (i.e. not dependent on also having to read in a TAV file),
+; and also because the original Envisat table.dat file does not contain the
+; VAR_SI_CONVERSION values. The input parameters have been changed to reflect this: bu
+; and up arrays (previously containing the VAR_UNIT and UNIT_PREFIX values from the TAV
+; file), and nbu and nup (the number of elements in the bu and up arrays) are no longer
+; used. The integer flag tab_type has been added to account for the different handling of
+; some of the VAR_UNIT and VAR_SI_CONVERSION values between AVDC and original Envisat.
+; The STOP_WITH_ERROR routine is no longer called in the event of an error, but an Error
+; State string is returned instead. Bug fixed when testing for a UNIT_PREFIX - previously
+; only the first character of the VAR_UNIT value was checked for a possible UNIT_PREFIX,
+; thus 'deka' (da) was excluded. Bug Fixed when the calculated Power Value of the last
+; base SI unit shown in VAR_SI_CONVERSION is '1', this is now set so that the power value
+; does not appear - Version 3.0
+; 20090611: Galileo added to var_unit_arr (AVDC); Last VAR_SI_CONVERSION value for ppmv, ppbv,
+; pptv, and ppv changed to DIMENSIONLESS (was ppv) in var_unit_arr (AVDC);
+; VAR_SI_CONVERSION values for molec changed to 0;1.66054E-24;mol (was 0;1;molec) in
+; var_unit_arr (AVDC); VAR_SI_CONVERSION values for DU changed to 0;4.4615E-4;mol m-2
+; (was 0;2.6867E20;molec m-2) in var_unit_arr; Stop power units being added to
+; DIMENSIONLESS (e.g. for VAR_UNITS=ppmv2); Change EVDC VAR_SI_CONVERSION values in
+; var_unit_arr to match Envisat Metadata Guidelines values - Version 3.01
+; 20091117: EVDC VAR_SI_CONVERSION values set to the same as AVDC (only one var_unit_arr set);
+; Fix bug which, in some cases, does not account for repeated units in VAR_UNITS when
+; determining the VAR_SI_CONVERSION (e.g. VAR_UNITS=W m-2 sr-1 m-1); Put VAR_SI_CONVERSION
+; units in power value order; Correctly scale multiple units by the power
+; e.g. W2 = (m2 kg s-3)^2 (previously assumed only a single unit was being scaled);
+; Generate error if third VAR_SI_CONVERSION value is 'DIMENSIONLESS' or 'NONE' but also
+; includes extra values - Version 4.0b1
+; 20101001: Set up for GEOMS compliance e.g. DIMENSIONLESS changed to 1; MJD2000 changed to MJD2K;
+; NONE removed - Version 4.0b2
+; 20101221: Bug fix - need to correctly account for when units cancel each other out. Was still
+; assigned the value DIMENSIONLESS - Version 4.0b3
+; 20110303: Bug fix - when a dimensionless unit includes a power value (e.g. ppmv2), the base unit
+; in VAR_SI_CONVERSION stays as '1'; Add 'dB' to var_unit_arr - Version 4.0b4
+; 20110719: Add 'pH' - Version 4.0b5
+; 20240908: Bug fix - Using the SORT command to put units in order of power value can give different
+; results depending on the operating system when power values are identical. Change routine
+; so the MAX command is used instead to create an array in descending power order
+; - Version 4.0b6
+; 20240912: Reinstitute sorting power units with the SORT command and save this version as a second
+; var_si_conv option (var_si_conv becomes a string array of size 2). This means the ordering
+; of the units between the two var_si_conv values may differ - Version 4.0b7
+;
+; Inputs: vuvalue - a string containing the Metadata VAR_UNITS value to be checked (everything
+; after the '=')
+; rd - integer flag to indicate if VAR_DATA_TYPE is real/double (-1) or not (1). Used to
+; make VAR_SI_CONVERSION values of the same type
+; tab_type - integer flag differentiating between AVDC (0) and original Envisat (1) styles
+;
+; Output: var_si_conv - a string array of size 2 containing the VAR_SI_CONVERSION value determined
+; by the program (returns '' if an error is encountered). The 2 values may
+; differ if there are multiple units with the same power value, due to using
+; two different sort methods
+; errstate - string describing an error state encountered during testing (o/w set to '')
+;
+; Called by: CHECK_METADATA
+;
+; External Subroutines Called: None
+
+var_unit_arr=['%;0;0.01;1','A;0;1;A','C;0;1;s A','cd;0;1;cd','d;0;86400;s','deg;0;1.74533E-2;rad',$
+ 'degC;273.15;1;K','1;0;1;1','h;0;3600;s','Hz;0;1;s-1','J;0;1;m2 kg s-2','K;0;1;K',$
+ 'l;0;1E-3;m3','lm;0;1;cd sr','lx;0;1;cd sr m-2','m;0;1;m','min;0;60;s',$
+ 'MJD2K;0;86400;s','mol;0;1;mol','molec;0;1.66054E-24;mol','Np;0;1;1',$
+ 'N;0;1E3;m kg s-2','Pa;0;1;m-1 kg s-2','pH;0;1E-12;m2 kg s-2 A-2','photons;0;1;photons',$
+ 'ppbv;0;1E-9;1','ppmv;0;1E-6;1','pptv;0;1E-12;1','ppv;0;1;1','psu;0;1;psu',$
+ 'rad;0;1;rad','s;0;1;s','sr;0;1;sr','V;0;1;m2 kg s-3 A-1','W;0;1;m2 kg s-3','kg;0;1;kg',$
+ 'DU;0;4.4614E-4;mol m-2','Gal;0;1E-2;m s-2','dB;0;1;1']
+
+unit_pre_arr=['Y;yotta;1E24','Z;zetta;1E21','E;exa;1E18','P;peta;1E15','T;tera;1E12','G;giga;1E9',$
+ 'M;mega;1E6','k;kilo;1E3','h;hecto;1E2','da;deka;1E1','d;deci;1E-1','c;centi;1E-2',$
+ 'm;milli;1E-3','u;micro;1E-6','n;nano;1E-9','p;pico;1E-12','f;femto;1E-15',$
+ 'a;atto;1E-18','z;zepto;1E-21','y;yocto;1E-24']
+
+;Set up text in case an error is found in the input VAR_UNITS value
+errtxt=STRARR(2)
+IF tab_type EQ 1 THEN errtxt[0]='No match with Table.Dat BASE_UNIT/UNIT_PREFIX values' $
+ELSE errtxt[0]='No match with Table Attribute Values file VAR_UNITS/UNIT_PREFIX values'
+errtxt[1]='Not valid'
+errtxt='VAR_UNITS='+vuvalue+': '+errtxt
+var_si_conv=STRARR(2) & errstate='' ;initialize outputs
+
+nta=N_ELEMENTS(var_unit_arr)
+;extract var_unit_arr/unit_pre_arr sub-values into vu/up arrays
+vuhold=STRSPLIT(var_unit_arr[0],';',/Extract) ;test for number of sub-values
+nvu=N_ELEMENTS(vuhold) & vu=STRARR(nvu,nta)
+FOR j=0,nta-1 DO BEGIN
+ vuhold=STRSPLIT(var_unit_arr[j],';',/Extract)
+ vu[*,j]=vuhold
+ENDFOR
+nta=N_ELEMENTS(unit_pre_arr)
+vuhold=STRSPLIT(unit_pre_arr[0],';',/Extract) ;test for number of sub-values
+nup=N_ELEMENTS(vuhold) & up=STRARR(nup,nta)
+FOR j=0,nta-1 DO BEGIN
+ vuhold=STRSPLIT(unit_pre_arr[j],';',/Extract)
+ up[*,j]=vuhold
+ENDFOR
+
+;separate out metadata sub-values into component parts, and set-up holding arrays
+vp=STRSPLIT(STRTRIM(vuvalue,2),' ',/Extract) & vpn=N_ELEMENTS(vp)
+vpx=STRARR(2,vpn) ;0 holds VAR_UNIT value, 1 holds POWER component
+bpx=STRARR(vpn) ;holding string array for base unit
+ex=INTARR(vpn)+1 ;holding integer array for power value (defaults to 1)
+tm=DBLARR(vpn) ;holding array for scale factor
+j=0
+WHILE (j LE vpn-1) AND (errstate EQ '') DO BEGIN
+ ;test to see if the sub-value is a base unit in the TAV file
+ ti=WHERE(vp[j] EQ vu[0,*],tcnt)
+ IF tcnt NE 0 THEN vpx[0,j]=vp[j] $ it is a base unit
+ ELSE BEGIN ;separate out into unit and power values as required
+ vpx[0,j]=STRMID(vp[j],0,1) ;save first character of vp(j)
+ stopchk=0
+ IF (STRLEN(vp[j]) GE 2) THEN BEGIN
+ FOR k=1,STRLEN(vp[j])-1 DO BEGIN
+ ah=STRMID(vp[j],k,1)
+ test1=(BYTE(ah) GE 65) AND (BYTE(ah) LE 90) ;A-Z
+ test2=(BYTE(ah) GE 97) AND (BYTE(ah) LE 122) ;a-z
+ IF (test1[0]) OR (test2[0]) THEN BEGIN
+ IF stopchk EQ 0 THEN vpx[0,j]=vpx[0,j]+ah ELSE stopchk=2
+ ;if stopchk EQ 2 THEN this means that VAR_UNITS is not legal
+ ENDIF ELSE IF stopchk EQ 0 THEN BEGIN
+ stopchk=1 ;first non-alpha character so check for numeric or '-' character
+ test1=(BYTE(ah) EQ 45) OR ((BYTE(ah) GE 49) AND (BYTE(ah) LE 57)) ;- or 1-9
+ IF NOT test1[0] THEN stopchk=2 ELSE vpx[1,j]=ah
+ ENDIF ELSE IF stopchk EQ 1 THEN BEGIN
+ ;need to check for a numeric character only
+ test1=(BYTE(ah) GE 48) AND (BYTE(ah) LE 57) ;0-9
+ IF NOT test1[0] THEN stopchk=2 ELSE vpx[1,j]=vpx[1,j]+ah
+ ENDIF
+ ENDFOR
+ IF vpx[1,j] NE '' THEN ex[j]=FIX(vpx[1,j])
+ ENDIF
+ IF stopchk EQ 2 THEN vpx[0,j]=vp[j] ;in the event that the value is not valid, so will create error
+ ;Do TAV check on the VAR_UNIT value
+ ti=WHERE(vpx[0,j] EQ vu[0,*],tcnt)
+ ENDELSE
+
+ IF tcnt NE 0 THEN BEGIN ;VAR_UNIT is a BASE_VALUE
+ bemult=(DOUBLE(vu[nvu-2,ti[0]]))^ex[j] & tm[j]=bemult
+ bpx[j]=vu[nvu-3,ti[0]]+';'+STRTRIM(STRING(format='(E8.1)',bemult),2)+';'+vu[nvu-1,ti[0]]
+ ENDIF ELSE IF vpx[0,j] EQ 'g' THEN BEGIN ;check for VAR_UNIT EQ g (gram) for AVDC style TAV file
+ bemult=0.001d^ex[j] & tm[j]=bemult
+ bpx[j]='0;'+STRTRIM(STRING(format='(E8.1)',bemult),2)+';kg'
+ ENDIF ELSE BEGIN ;separate out vpx value into prefix and base-value and test
+ ;check for valid prefix - first check for 'deka' (da)
+ pref=STRMID(vpx[0,j],0,2) & bas=STRMID(vpx[0,j],2)
+ pi=WHERE(pref EQ up[0,*],pcnt)
+ IF pcnt EQ 0 THEN BEGIN ;test for the remaining prefixes
+ pref=STRMID(vpx[0,j],0,1) & bas=STRMID(vpx[0,j],1)
+ pi=WHERE(pref EQ up[0,*],pcnt)
+ ENDIF
+ IF pcnt NE 0 THEN BEGIN
+ pmult=DOUBLE(up[nup-1,pi[0]])
+ ;check for valid base
+ ti=WHERE(bas EQ vu[0,*],tcnt)
+ IF tcnt NE 0 THEN BEGIN
+ bpmult=(DOUBLE(vu[nvu-2,ti[0]])*pmult)^ex[j] & tm[j]=bpmult
+ bpx[j]=vu[nvu-3,ti[0]]+';'+STRTRIM(STRING(format='(E8.1)',bpmult),2)+';'+vu[nvu-1,ti[0]]
+ ENDIF ELSE IF bas EQ 'g' THEN BEGIN ;check for VAR_UNIT EQ g (gram) for AVDC style TAV file
+ bpmult=(pmult*0.001D)^ex[j] & tm[j]=bpmult
+ bpx[j]='0;'+STRTRIM(STRING(format='(E8.1)',bpmult),2)+';kg'
+ ENDIF ELSE errstate=errtxt[0]
+ ENDIF ELSE errstate=errtxt[0]
+ ENDELSE
+ j=j+1
+ENDWHILE
+
+IF errstate EQ '' THEN BEGIN ;No errors detected so continue
+ ;Create VAR_SI_CONVERSION value
+ tmult=1.0D
+ FOR j=0,vpn-1 DO tmult=tmult*tm[j]
+
+ ;reformat the multiplier e.g. 1.000E+003 becomes 1E3
+ ;convert to Exponential form if necessary
+ IF (tmult EQ 273.15D) OR (tmult MOD 60.D EQ 0.D) OR ((tmult GE 0.01D) AND (tmult LT 1.D2) $
+ AND (tmult*1.D4 MOD 1.D2 EQ 0.D)) THEN tmults=STRTRIM(STRUPCASE(tmult),2) $ ;keep the same format
+ ELSE tmults=STRTRIM(STRING(format='(E14.6)',tmult),2)
+
+ epos=STRPOS(tmults,'E') & ppos=STRPOS(tmults,'.')
+ IF epos NE -1 THEN BEGIN ;remove unnecessary characters after 'E'
+ ep=FIX(STRMID(tmults,epos+1)) & epx=STRTRIM(ep,2)
+ tmults=STRMID(tmults,0,epos+1)+epx
+ ENDIF
+ IF ppos NE -1 THEN BEGIN ;remove any trailing zeroes after the decimal place
+ IF epos NE -1 THEN ep=STRMID(tmults,ppos+1,epos-(ppos+1)) ELSE ep=STRMID(tmults,ppos+1)
+ WHILE STRMID(ep,STRLEN(ep)-1,1) EQ '0' DO ep=STRMID(ep,0,STRLEN(ep)-1)
+ IF ep NE '' THEN tmults=STRMID(tmults,0,ppos+1)+ep ELSE tmults=STRMID(tmults,0,ppos)
+ IF epos NE -1 THEN tmults=tmults+'E'+epx
+ ENDIF
+
+ ;Scale the units by the power e.g. W2 = (m2 kg s-3)^2
+ FOR j=0,vpn-1 DO BEGIN
+ vsc=STRSPLIT(bpx[j],';',/EXTRACT)
+ vspl=STRSPLIT(vsc[2],' ',/EXTRACT,COUNT=vscnt)
+ sichk=STRARR(vscnt) & pwchk=sichk
+ IF (vpx[1,j] NE '') AND (vpx[1,j] NE '1') AND (vsc[2] NE '1') THEN BEGIN
+ bpx[j]=vsc[0]+';'+vsc[1]+';'
+ FOR k=0,vscnt-1 DO BEGIN
+ ;separate out SI units and power values
+ sires=STRSPLIT(vspl[k],'-0123456789',/Extract)
+ sichk[k]=sires[0] ;SI Unit
+ IF STRLEN(sichk[k]) NE STRLEN(vspl[k]) THEN $
+ pwchk[k]=STRMID(vspl[k],STRLEN(sichk[k])) $
+ ELSE pwchk[k]='1' ;Power value
+ pwm=FIX(pwchk[k])*FIX(vpx[1,j])
+ IF k EQ 0 THEN sp='' ELSE sp=' '
+ bpx[j]=bpx[j]+sp+sichk[k]+STRTRIM(pwm,2)
+ ENDFOR
+ ENDIF
+ ENDFOR
+
+ ;Put together VAR_SI_CONVERSION
+ vsc=STRSPLIT(bpx[0],';',/EXTRACT)
+ ;IF vsc[2] EQ '1' THEN vpx[1,0]=''
+ var_si_conv[0]=vsc[0]+';'+tmults+';'+vsc[2]
+ ;add remaining base units to VAR_SI_CONVERSION
+ IF vpn GT 1 THEN BEGIN
+ FOR j=1,vpn-1 DO BEGIN
+ vsc=STRSPLIT(bpx[j],';',/Extract)
+ var_si_conv[0]=var_si_conv[0]+' '+vsc[2]
+ ENDFOR
+ ENDIF
+ var_si_conv[1]=var_si_conv[0] ;Default is that the 2nd entry is the same as the first
+
+ ;check VAR_SI_CONVERSION for repeated SI units e.g. m m-3 becomes m-2
+ schk=STRSPLIT(var_si_conv[0],' ;',/Extract) & scnt=N_ELEMENTS(schk)
+ IF scnt GT 3 THEN BEGIN ;more than one SI unit in VAR_SI_CONVERSION
+ sichk=STRARR(scnt-2) & pwchk=sichk
+ FOR j=0,scnt-3 DO BEGIN ;separate out SI units and power values
+ sires=STRSPLIT(schk[j+2],'-0123456789',/Extract)
+ sichk[j]=sires[0] ;SI Unit
+ IF STRLEN(sichk[j]) NE STRLEN(schk[j+2]) THEN $
+ pwchk[j]=STRMID(schk[j+2],STRLEN(sichk[j])) $
+ ELSE pwchk[j]='1' ;Power value
+ ENDFOR
+ j=0 & pwval=0
+ WHILE j LT scnt-3 DO BEGIN
+ si=WHERE((sichk[j] NE '') AND (sichk[j] EQ sichk[j+1:scnt-3]),sicnt)
+ IF sicnt NE 0 THEN BEGIN
+ FOR k=0,sicnt-1 DO BEGIN
+ si[k]=si[k]+j+1
+ pwval=FIX(pwchk[j])+FIX(pwchk[si[k]])
+ IF (pwval[0] EQ 0) AND (k EQ sicnt-1) THEN BEGIN
+ sichk[j]='' & pwchk[j]=''
+ ENDIF ELSE IF (pwval[0] EQ 1) and (k EQ sicnt-1) THEN pwchk[j]='' $
+ ELSE pwchk[j]=STRTRIM(pwval[0],2)
+ ;make null all the other SI values
+ sichk[si[k]]='' & pwchk[si[k]]=''
+ ENDFOR
+ ENDIF ELSE IF pwchk[j] EQ '1' THEN pwchk[j]=''
+ j=j+1
+ ENDWHILE
+ ;Also do check on the power value of the last SI Unit
+ IF pwchk[scnt-3] EQ '1' THEN pwchk[scnt-3]=''
+
+ ;Put units in power value order
+ ;Create two possible versions, one using the MAX command (primary) and another using SORT
+ ;Note: When using the SORT command, identical elements are sorted arbitrary and may vary between operating systems
+ FOR k=0,1 DO BEGIN
+ pwhold=pwchk
+ oi=WHERE(pwhold EQ '',ocnt)
+ IF ocnt NE 0 THEN pwhold[oi]='1'
+ IF k EQ 0 THEN BEGIN ;Do MAX order method
+ pws=INTARR(scnt-2)
+ ;determine minimum power value
+ mnp=MIN(FIX(pwhold))
+ FOR j=0,scnt-3 DO BEGIN
+ mxp=MAX(FIX(pwhold),mxs)
+ pws[j]=mxs
+ pwhold[mxs]=mnp - 1 ;make it the lowest value
+ ENDFOR
+ sichk=sichk[pws] & pwchk=pwchk[pws]
+ ENDIF ELSE BEGIN ;Do SORT method (results are operating system dependent)
+ pws=SORT(FIX(pwhold))
+ sichk=sichk[REVERSE(pws)] & pwchk=pwchk[REVERSE(pws)]
+ ENDELSE
+ var_si_conv[k]=schk[0]+';'+schk[1]+';'+sichk[0]+pwchk[0]
+ si=WHERE(sichk NE '',sicnt)
+ IF sicnt EQ 0 THEN var_si_conv[k]=var_si_conv[k]+'1' $ ;i.e. values cancelled out
+ ELSE BEGIN
+ FOR j=1,scnt-3 DO BEGIN
+ si=WHERE(sichk[0:j-1] NE '',sicnt)
+ IF (sichk[j] EQ '') OR (sicnt EQ 0) THEN var_si_conv[k]=var_si_conv[k]+sichk[j]+pwchk[j] $
+ ELSE var_si_conv[k]=var_si_conv[k]+' '+sichk[j]+pwchk[j]
+ ENDFOR
+ ENDELSE
+ ENDFOR
+ ENDIF
+
+ IF rd LT 0 THEN BEGIN
+ ;VAR_DATA_TYPE is Real or Double so make VAR_SI_CONVERSION values floating point
+ FOR k=0,1 DO BEGIN
+ tchkh=STRSPLIT(var_si_conv[k],';',/Extract) & tup=STRUPCASE(tchkh)
+ FOR j=0,1 DO BEGIN
+ IF STRPOS(tup[j],'.') EQ -1 THEN BEGIN
+ IF STRPOS(tup[j],'E') EQ -1 THEN tchkh[j]=tchkh[j]+'.0' $
+ ELSE tchkh[j]=STRMID(tup[j],0,STRPOS(tup[j],'E'))+'.0'+STRMID(tup[j],STRPOS(tup[j],'E'))
+ ENDIF
+ ENDFOR
+ var_si_conv[k]=tchkh[0]+';'+tchkh[1]+';'+tchkh[2]
+ ENDFOR
+ ENDIF
+
+ ;Check for invalid VAR_UNITS - third VAR_SI_CONVERSION is 1 plus extra values
+ FOR k=0,1 DO BEGIN
+ vsc=STRSPLIT(var_si_conv[k],';',/EXTRACT)
+ vsc[2]=STRTRIM(vsc[2],2)
+ IF (errstate EQ '') AND (STRMID(vsc[2],0,1) EQ '1') AND (STRLEN(vsc[2]) GT 1) THEN BEGIN
+ errstate=errtxt[1] & var_si_conv[k]=''
+ ENDIF
+ ENDFOR
+ENDIF
+
+END ;procedure Var_Units_Test
+
+
+
+PRO read_tablefile, tablefile
+;Procedure to identify the version number of the TAV file, read the contents of the
+;LABELS/FIELDS (tab_arr), and determine the FILE_META_VERSION in the global attributes (tab_ver).
+;This routine also creates a flag (tab_type) to determine whether the input file is an original
+;table.dat file used by Envisat or the GEOMS TAV file, with the HDF/netCDF file generated
+;accordingly.
+; ----------
+;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
+;
+;History:
+; 20050802: Original IDLCR8HDF Routine - Version 1.0
+; 20061012: Bug-fix to separate semi-colons with a space when one immediately follows the other,
+; so that the number of subvalues is correctly determined by the program. Common
+; variable definition WIDGET_WIN added - Version 2.0
+; 20080302: Added code to differentiate between the AVDC TAV file and original Envisat table.dat.
+; The tab_type flag is used to differentiate between the two formats. Note that some of the
+; table.dat labels are renamed to match the equivalent TAV file labels for compatibility
+; when testing Metadata entries - refer to EnviName/AVDCName arrays; Ensure that the TAV
+; Version value is correctly formatted and is version 03 or greater - Version 3.0
+; 20100205: Add RETURN command after all STOP_WITH_ERROR calls, which allows program to return to the
+; calling program if the reterr argument is included in the idlcr8hdf call - Version 3.09
+; 20110401: Change AVDC references to GEOMS; Test AVDC TAV file version is version 04 or greater;
+; Remove check on format of TAV file version; Conform to new INFOTXT_OUTPUT reporting
+; - Version 4.0b1
+;
+;Input: Tablefile - Name of the file containing the Table Attributes.
+;
+;Outputs: tab_ver - String containing FILE_META_VERSION value.
+; tab_arr - 2-D string array of size nf*mcnt+2, where nf=Number of Fields, and mcnt=maximum
+; number of values detected in any one field. tab_arr(*,0) is the name of each field,
+; and tab_arr(*,1) is the number of values in each field.
+; tab_type - 0/1 where 0 identifies an AVDC format TAV file and 1 identifies an original
+; Envisat format table.dat file.
+;
+;Called by: IDLCR8HDF
+;
+;Subroutines Called: STOP_WITH_ERROR (if error state detected); INFOTXT_OUTPUT
+; Possible Conditions for STOP_WITH_ERROR call (plus [line number] where called):
+; 1. Table Attribute Values file version not identified
+; 2. Envisat table.dat ASC2HDF program version not found
+; 3. First Envisat table.dat field value not found
+; 4. Table Attribute Values file version is not in a valid format
+;
+; Information Conditions (when the program reports issues and continues):
+; 1. [Original EVDC]/[GEOMS] Reporting Guidelines Apply (depending on type of Table Attribute
+; Values file read in)
+; 2. Old version of the TAV file in use. Update from http://avdc.gsfc.nasa.gov/Tools
+
+COMMON TABLEDATA
+COMMON WIDGET_WIN
+
+;Possible error messages for this procedure
+proname='Read_TableFile procedure: '
+errtxt=STRARR(4)
+errtxt[0]='Table Attribute Values file version not identified'
+errtxt[1]='Envisat table.dat ASC2HDF program version not found'
+errtxt[2]='First Envisat table.dat field value not found'
+errtxt[3]='Table Attribute Values file version is invalid (should be ddRddd or ddRdddd): '
+FOR i=0,2 do errtxt[i]=errtxt[i]+' with the search criteria used by this program'
+
+;Array of Envisat Field names to be changed to equivalent AVDC Field names
+enviname=['_NAME','_AFFILIATION','DATA_VARIABLES_00_00','BASE_UNIT']
+avdcname=['ORIGINATOR','AFFILIATION','DATA_VAR_ALL_00_00','VAR_UNITS']
+
+ON_IOERROR,TypeConversionError
+dum='' & tab_ver=''
+min_fmv=4 ;TAV Version must be GE this value e.g. 04R001, 06R002 but not 03R004
+OPENR,lu,tablefile,/GET_LUN
+;determine TAV file version
+REPEAT BEGIN
+ READF,lu,dum
+ dumup=STRUPCASE(dum)
+ envitest=STRPOS(STRCOMPRESS(dumup,/Remove_all),'!TABLE.DATVERSION') NE -1
+ avdctest=STRPOS(STRCOMPRESS(dumup,/Remove_all),'!VERSION') NE -1
+ENDREP UNTIL (envitest) OR (avdctest) OR (EOF(lu))
+IF EOF(lu) THEN BEGIN
+ STOP_WITH_ERROR,o3[3]+proname,errtxt[0],lu & RETURN
+ENDIF
+res=STRSPLIT(dumup,' ',/Extract) & vi=WHERE(res EQ 'VERSION')
+IF N_ELEMENTS(res) LE vi[0]+1 THEN BEGIN
+ STOP_WITH_ERROR,o3[3]+proname,errtxt[0]+': '+dum,lu & RETURN
+ENDIF
+IF envitest THEN BEGIN
+ tab_type=1 & infotxt='0 EVDC original style table.dat'
+ n_title=4
+ENDIF ELSE BEGIN
+ tab_type=0 & infotxt='0 GEOMS compliant Table Attribute Values'
+ n_title=5
+ENDELSE
+infotxt=infotxt+' file input. '+STRMID(infotxt,2,n_title)+' Reporting Guidelines apply'
+INFOTXT_OUTPUT,infotxt
+
+;Ensure Meta Version has a valid format
+;i. Check third character is an 'R'
+;ii. Check number of characters is 6 or 7
+;iii. Check dd and ddd(d) are numeric
+;iv. Check that version number is 03 or greater for AVDC file type
+;v. Issue warning and convert to ddRddd if format is ddRdddd
+fmv=res[vi[0]+1] ;File_Meta_Version
+valid=0 ;set to test for valid string to number conversion
+FOR i=0,STRLEN(fmv)-1 DO IF i NE 2 THEN fmvtest=FIX(STRMID(fmv,i,1))
+fmvtest=FIX(STRMID(fmv,0,2)) ;To test for TAV version 'min_fmv' or greater
+valid=1 ;FILE_META_VERSION characters are numeric (except for the 'R') if program gets to here
+TypeConversionError:
+IF (STRMID(fmv,2,1) NE 'R') OR (STRLEN(fmv) lt 6) OR (STRLEN(fmv) gt 7) OR $
+ (valid EQ 0) THEN BEGIN
+ STOP_WITH_ERROR,o3[3]+proname,errtxt[3]+fmv,lu & RETURN
+ENDIF
+
+IF (tab_type EQ 0) AND (fmvtest LT min_fmv) THEN BEGIN
+ infotxt=STRARR(2)
+ infotxt[0]='3 Old version of the Table Attribute Values file used as input: '
+ infotxt[0]=infotxt[0]+' TAV Version '+fmv
+ infotxt[1]=' Please update from http://avdc.gsfc.nasa.gov/Tools'
+ INFOTXT_OUTPUT,infotxt
+ENDIF
+
+IF envitest THEN BEGIN ;identify asc2hdf version in table.dat and read past the CHECK_ATTRIBUTE line
+ WHILE (STRPOS(STRUPCASE(dum),'ASC2HDF') EQ -1) AND (NOT EOF(lu)) DO READF,lu,dum
+ IF EOF(lu) THEN BEGIN
+ STOP_WITH_ERROR,o3[3]+proname,errtxt[1],lu & RETURN
+ ENDIF
+ cr8_hdf_ver=';IDLCR8HDF'
+ dum=STRTRIM(dum,2)
+ WHILE ((STRMID(dum,0,1) EQ '!') OR (STRMID(dum,0,1) EQ '#') OR (dum EQ '')) AND $
+ (NOT EOF(lu)) DO BEGIN
+ READF,lu,dum & dum=STRTRIM(dum,2)
+ ENDWHILE
+ IF EOF(lu) THEN BEGIN
+ STOP_WITH_ERROR,o3[3]+proname,errtxt[2],lu & RETURN
+ ENDIF
+ READF,lu,dum ;to get to the line after 'CHECK_ATTRIBUTE'
+ENDIF ELSE cr8_hdf_ver=';IDLCR8HDF'
+tab_ver=fmv+cr8_hdf_ver ;= the FILE_META_VERSION input value in the global attributes
+
+;determine no. of FIELDS/LABELS as well as the maximum number of elements
+nf=0 & mcnt=0 & firstfield=''
+dum=STRTRIM(dum,2)
+WHILE NOT EOF(lu) DO BEGIN
+ ncnt=-1
+ IF (STRMID(dum,0,1) NE '!') AND (STRMID(dum,0,1) NE '#') AND $
+ (STRMID(dum,0,1) NE '=') AND (dum NE '') THEN BEGIN
+
+ FOR i=0,N_ELEMENTS(enviname)-1 DO BEGIN
+ epos=STRPOS(STRUPCASE(dum),enviname[i])
+ IF (epos NE -1) AND (epos LE 3) THEN ncnt=i
+ ENDFOR
+ IF (ncnt EQ 0) OR (ncnt EQ 1) THEN BEGIN
+ IF STRMID(STRUPCASE(dum),0,3) NE 'PI_' THEN nf=nf-1 ELSE ecnt=0
+ ;This puts all PI_,DO_, and DS_NAME or AFFILIATION values into either the ORIGINATOR or AFFILIATION fields
+ ENDIF ELSE ecnt=0
+
+ nf=nf+1
+ IF firstfield EQ '' THEN firstfield=dum
+ IF ncnt NE -1 THEN dum=avdcname[ncnt] ;change the name of the Envisat field to AVDC equivalent
+ REPEAT BEGIN
+ READF,lu,dum & dum=STRTRIM(dum,2)
+ IF STRMID(dum,0,1) EQ '=' THEN ecnt=ecnt+1
+ ENDREP UNTIL (STRMID(dum,0,1) NE '=') OR (EOF(lu))
+ IF ecnt GT mcnt THEN mcnt=ecnt
+ ENDIF ELSE IF NOT EOF(lu) THEN BEGIN
+ READF,lu,dum & dum=STRTRIM(dum,2)
+ ENDIF
+ENDWHILE
+FREE_LUN,lu
+
+;read in the contents of the file
+tab_arr=STRARR(nf,mcnt+2) ;note tab_arr(*,0) EQ FIELD/LABEL name and
+;tab_arr(*,1) EQ N_ELEMENTS under each FIELD/LABEL
+tab_hold=STRARR(mcnt)
+
+OPENR,lu,tablefile,/GET_LUN
+READF,lu,dum & dum=STRTRIM(dum,2)
+WHILE dum NE firstfield DO BEGIN
+ READF,lu,dum & dum=STRTRIM(dum,2)
+ENDWHILE
+
+i=0
+WHILE i LE nf-1 DO BEGIN
+ ncnt=-1
+ FOR j=0,N_ELEMENTS(enviname)-1 DO BEGIN
+ epos=STRPOS(STRUPCASE(dum),enviname[j])
+ IF (epos NE -1) AND (epos LE 3) THEN ncnt=j
+ ENDFOR
+ IF (ncnt EQ 0) OR (ncnt EQ 1) THEN BEGIN
+ IF STRMID(STRUPCASE(dum),0,3) NE 'PI_' THEN i=i-1 ELSE ecnt=0
+ ENDIF ELSE ecnt=0
+ IF ncnt NE -1 THEN dum=avdcname[ncnt] ;change the name of the Envisat field to AVDC equivalent
+ tab_arr[i,0]=dum
+ REPEAT BEGIN
+ READF,lu,dum & dum=STRTRIM(dum,2)
+ IF STRMID(dum,0,1) EQ '=' THEN BEGIN
+ tab_hold[ecnt]=STRMID(dum,1) ;strip the '=' sign
+ ;add space between adjacent semi-colons so StrSplit finds correct number of sub-values
+ REPEAT BEGIN
+ cpos=STRPOS(tab_hold[ecnt],';;')
+ IF cpos NE -1 THEN tab_hold[ecnt]=STRMID(tab_hold[ecnt],0,cpos+1)+ $
+ ' '+STRMID(tab_hold[ecnt],cpos+1)
+ ENDREP UNTIL cpos EQ -1
+ ecnt=ecnt+1
+ ENDIF
+ ENDREP UNTIL (STRMID(dum,0,1) NE '=') OR (EOF(lu))
+ tab_arr[i,1]=STRTRIM(ecnt,2)
+ tab_arr[i,2:ecnt+1]=tab_hold[0:ecnt-1]
+ IF i NE nf-1 THEN BEGIN
+ WHILE (STRMID(dum,0,1) EQ '!') OR (STRMID(dum,0,1) EQ '#') OR (dum EQ '') DO BEGIN
+ READF,lu,dum & dum=STRTRIM(dum,2)
+ ENDWHILE
+ ENDIF
+ i=i+1
+ENDWHILE
+FREE_LUN,lu
+
+END ;Procedure Read_TableFile
+
+
+
+PRO test_file_input,aname,fentry
+;Procedure to test that a file name given as an entry to a free text attribute is valid, based on Envisat
+;Metadata Guidelines (not currently used by AVDC). Note: No longer permitted under GEOMS guidelines.
+; ----------
+;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
+;
+; History:
+; 20050802: Original IDLCR8HDF Routine - Version 1.0
+; 20061012: Common variable definition WIDGET_WIN added - Version 2.0
+; 20100205: Add RETURN command after all STOP_WITH_ERROR calls, which allows program to return to the
+; calling program if the reterr argument is included in the idlcr8hdf call - Version 3.09
+;
+; Inputs: aname - Global or Variable Attribute Label
+; fentry - Filename entry used as the Global or Variable Attribute value
+;
+; Outputs: None
+;
+; Called by: READ_METADATA
+;
+; Subroutines Called: STOP_WITH_ERROR (if error state detected)
+; Possible Conditions for STOP_WITH_ERROR call (plus [line number] where called):
+; 1. Syntax of Filename entry incorrect
+; 2. Filename entry not found or not usable
+; 3. File size is too large
+
+COMMON WIDGET_WIN
+
+sfile=4096 ;maximum permitted file size
+
+;possible error message for this procedure
+proname='Test_File_Input procedure: '
+errtxt2=STRARR(3) & lu=-1
+errtxt2[0]='Syntax of Filename entry incorrect: '
+errtxt2[1]='Filename entry not found or not usable: '
+errtxt2[2]='File size is too large (maximum permitted: '+STRTRIM(sfile,2)+' bytes)'
+
+si=STRPOS(fentry,'"')+1 & ei=STRPOS(fentry,'"',/Reverse_Search)-si
+IF ei EQ si THEN BEGIN
+ STOP_WITH_ERROR,o3[3]+proname+aname+': ',errtxt2[0]+fentry,lu & RETURN
+ENDIF
+faname=STRMID(fentry,si,ei)
+ftest=FILE_TEST(faname,/Read,/Regular)
+IF ftest EQ 0 THEN BEGIN
+ STOP_WITH_ERROR,o3[3]+proname+aname+': ',errtxt2[1]+faname,lu & RETURN
+ENDIF
+OPENR,fu,faname,/GET_LUN
+ftest=FSTAT(fu) & FREE_LUN,fu
+IF ftest.size GT sfile THEN BEGIN
+ STOP_WITH_ERROR,o3[3]+proname+AName+'='+fentry+': ',errtxt2[2],lu & RETURN
+ENDIF
+
+END ;procedure Test_File_Input
+
+
+
+PRO geoms_rule_changes, code, in1, in2, in3, in4
+;Procedure to check Metadata for old/redundant rules, labels and/or values and update/report
+;as required
+; ----------
+;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
+;
+; History:
+; 20110401: Introduced. Incorporates GEOMS rules changes from v3.0 to v4.0 - Version 4.01
+; 20111220: Add ISO646-US ASCII character set check - Version 4.0b7
+; 20120313: Add UVVIS.DOAS plus additional gases to Code 6 checks - Version 4.0b8
+; 20120703: Bug Fix when checking for non-ASCII characters (rule 10) - Version 4.0b11
+; 20131023: Add GEOMS rule change for MIXING.RATIO[.VOLUME][.MASS], and UNCERTAINTY (rule 5);
+; Fixed bug that caused incorrect rep_list_2 values to be written to file (rule 5);
+; Stopped checks for illegal characters if ORIGINATOR values are present in the
+; TAV file (rule 10) - Version 4.0b16
+; 20131029: Add array of invalid ASCII characters that can be replaced with valid ISO646 US
+; ASCII characters (rule 10) - Version 4.0b17
+; 20131029: Modify UNCERTAINTY rules to allow more exceptions (rule 5); Allow for renaming of
+; LIDAR.DIAL DATA_SOURCE to LIDAR.WATERVAPORDIAL if conditions met (rule 6) -
+; Version 4.0b18
+; 20140226: Add 'DU' to obsolete descriptor value and fix bug causing no new variable name to
+; be generated if more than 3 sub-values in the name (rule 5); Add 'RO.SAC.C' to
+; the obsolete DATA_SOURCE list (rule 6) - Version 4.0b19
+; 20140521: Account for extra parameter in the UVVIS-DOAS data source when trying to determine
+; if file is based on a DATA_TEMPLATE (rule 9) - Version 4.0b21
+; 20140806: Modify UNCERTAINTY rules to allow more exceptions including additional instrument
+; types and check on molec cm-2 to allow for prefix values (rule 5) - Version 4.0b22
+; 20140909: Add RO.F3C.FM[1][2][3] to the obsolete list of DATA_SOURCES (replaced by
+; F3C.FM[1][2][3]) (rule 6) - Version 4.0b23
+; 20141110: Numerous changes while testing harmonization of EVDC files - additional
+; DATA_SOURCEs and units added to identify correct new UNCERTAINTY values, including
+; identification of covariance uncertainties; Bug fix when doing checks on
+; AIR.CONCENTRATION; LIDAR.BACKSCATTER conversion and units definition added; List
+; of non-ASCII characters to be checked expanded; Bug fix when correcting
+; for non-ASCII characters when the replacement value has more than one character
+; e.g. ' deg. ' - Version 4.0b24
+; 20150409: Strip trailing and leading spaces from DATA_TEMPLATE value when doing checks
+; (rule 9) - Version 4.0b28
+; 20160213: Add extra satellite instruments to the obsolete list for DATA_SOURCE (rule 6);
+; Fix issue with DATA_TEMPLATE identification (rule 9) - Version 4.0b37
+; 20171121: Fix bug that selected the incorrect DATA_TEMPLATE in some situations (rule 9)
+; - Version 4.0b43
+; 20190506: Add FILE_META_VERSION information to INFORMATION/ERROR comment when
+; DATA_TEMPLATE value is not as expected; Fix bug that caused a crash if the
+; file does not use a DATA_TEMPLATE (rule 9) - Version 4.0b50
+; 20191205: Add rule 11 which does checks on optional VERSION_NAME sub-field of
+; DATA_SOURCE - Version 4.0b53
+; 20200709: Fix in option 5 when looking for obsolete variable names. Change from doing a
+; STRPOS search to looking for an exact change for DATA_VARIABLES_01 names due to
+; introduction of DRY.AIR.... names to the TAV - Version 4.0b55
+; 20201020: Change infotxt status from 1 to 2 in option 5; Change to LIDAR.H2O data source
+; if needing to update out-of-date H2O LIDAR DATA_SOURCE values in option 6
+; - Version 4.0b56
+; 20220805: Change INFOTXT message in rule 10 (non-ASCII character check) so it includes
+; the entire Metadata entry - Version 4.0b60
+;
+; Inputs: code - Integer value identifying type of check to carry out
+; in1 - First set of inputs required for checks (optional, dependent on code value)
+; in2 - Second set of inputs required for checks (optional, dependent on code value)
+; in3 - Third set of inputs required for checks (optional, dependent on code value)
+; in4 - Fourth set of inputs required for checks (optional, dependent on code value)
+;
+; Outputs: Any or all of the inputs can be changed dependent on the code value
+;
+; Called by: READ_METADATA; CHECK_METADATA; SET_UP_STRUCTURE; FIND_HDF_FILENAME
+;
+; Subroutines Called: INFOTXT_OUTPUT
+; Information Conditions (when the program is able to make changes):
+; 1. Attribute entry is not GEOMS compliant, and removed from the metadata saved to the output file
+; 2. Dataset name renamed according to new DATETIME reporting conventions
+; 3. Double underscore replaced with a single underscore in the variable name
+; 4. VAR_DEPEND value made self-referencing for axis variable
+; 5. Axis variable cannot be dependent on another variable
+; 6. VAR_UNITS=MJD2000 not GEOMS compliant, changed to VAR_UNITS=MJD2K
+; 7. DATA_FILE_VERSION must be in the form 'nnn'
+; 8. Obsolete Dataset name renamed
+; 9. Obsolete DATA_SOURCE value renamed
+; 10. Obsolete FILE_ACCESS values renamed
+; 11. Rename obsolete VAR_UNITS=DIMENSIONLESS/NONE values
+; 12. DATA_TEMPLATE field is not present in the TAV file
+; 13. DATA_TEMPLATE value renamed based on DATA_SOURCE value
+; 14. Entry must only include valid characters from the ISO646-US ASCII character set
+;
+; Code 0: Check for Attributes that have become redundant between v3.0 and v4.0.
+; Inputs: in1=mh, in2=mhgood
+; Code 1: Check for redundant _START/STOP/INTEGRATION.TIME variable names and replace with
+; DATETIME.START/STOP/INTEGRATION.TIME (for single occurences only). Also check
+; for double underscores in the variable names and replace with single underscore
+; Inputs: in1=vncnt, in2=dv
+; Code 2: Check for Axis Variables and, if found, make VAR_DEPEND=INDEPENDENT self-referencing
+; Inputs: in1=vardeptest, in2=vn[vc], in3=resvd[1], in4=holdvd
+; Code 3: Change VAR_UNIT value MJD2000 to MJD2K
+; Inputs: in1=meta_arr[i], in2=res[1], in3=writeonce
+; Code 4: Do DATA_FILE_VERSION checks
+; Inputs: in1=dfvv (DATA_FILE_VERSION value)
+; Code 5: Look for obsolete DATA_VARIABLES (based on list) and, if possible, modify values to
+; GEOMS compliance
+; Inputs: in1=vncnt, in2=dv, in3=vuv, in4=data_source
+; Code 6: Look for obsolete DATA_SOURCE (based on list) and, if possible, modify values to
+; GEOMS compliance
+; Inputs: in1=vncnt, in2=dv, in3=data_source
+; Code 7: Looks for and replaces obsolete CALVAL and NDSC FILE_ACCESS values
+; Inputs: in1=facnt, in2=fav
+; Code 8: Looks for VAR_UNITS=DIMENSIONLESS or NONE and replace with '' or '1'
+; Inputs: in1=vucnt, in2=vuv, in3=vdv
+; Code 9: Do DATA_TEMPLATE/DATA_QUALITY checks
+; Inputs: in1=m_v0, in2=m_v1, in3=ga_chk
+; Code 10: Do checks on ISO646-US ASCII character set
+; Inputs: in1=meta_arr or dtest, in2=vn[vc] (Variable Name for datasets only)
+; Code 11: Do checks on DATA_VERSION_NAME part of DATA_SOURCE value
+; Inputs: in1=DATA_VERSION_NAME, in2=meta_arr
+
+COMMON TABLEDATA
+COMMON METADATA
+COMMON DATA
+COMMON WIDGET_WIN
+
+CASE 1 OF
+ code EQ 0: BEGIN
+ ;Check for redundant attributes
+ redundant=['DATA_TYPE','DATA_LEVEL','VAR_MONOTONE','VAR_DIMENSION',$
+ 'VAR_AVG_TYPE','VIS_LABEL','VIS_FORMAT','VIS_PLOT_TYPE',$
+ 'VIS_SCALE_TYPE','VIS_SCALE_MIN','VIS_SCALE_MAX']
+ nrd=N_ELEMENTS(redundant)
+
+ FOR i=0,nrd-1 DO BEGIN
+ rdl=STRLEN(redundant[i])
+ fai=WHERE(STRMID(STRUPCASE(in1),0,rdl) EQ redundant[i],facnt)
+ IF facnt NE 0 THEN BEGIN
+ in2[fai]=0 ;Metadata lines not wanted
+ test1=(STRMID(redundant[i],0,3) EQ 'VAR') OR (STRMID(redundant[i],0,3) EQ 'VIS')
+ IF test1 THEN att_type='Variable' ELSE att_type='Global'
+ infotxt='2 '+redundant[i]+' not a GEOMS '+att_type+' attribute|.'
+ infotxt=infotxt+' Removed from the metadata saved to the output file'
+ INFOTXT_OUTPUT, infotxt
+ ENDIF
+ ENDFOR
+ END ;Case 0
+
+ code EQ 1: BEGIN
+ ;Check for redundant _START/STOP sub-values and replace with
+ ;DATETIME.START/STOP/INTEGRATION.TIME (for single occurences only)
+ time_attr=['START','STOP','INTEGRATION','RESOLUTION']
+ n_tim=N_ELEMENTS(time_attr)
+ iscnt=0 ;Used for RESOLUTION/INTEGRATION.TIME checks
+ FOR i=0,n_tim-1 DO BEGIN
+ si=WHERE(STRPOS(in2,'_'+time_attr[i]+'.TIME') NE -1,scnt)
+ IF (scnt EQ 1) AND (iscnt EQ 0) THEN BEGIN ;change name and also relevant VAR_NAME if present
+ IF i EQ 2 THEN iscnt=1 ;used in the event both INTEGRATION and RESOLUTION.TIME are present
+ vnchange[si[0]]=in2[si[0]]
+ IF i GE 2 THEN in2[si[0]]=time_attr[2]+'.TIME' $
+ ELSE in2[si[0]]='DATETIME.'+time_attr[i]
+ IF qa_yes THEN itxt=' expected to be ' ELSE itxt=' renamed '
+ infotxt='2 Dataset name '+vnchange[si[0]]+itxt+in2[si[0]]
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ ENDFOR
+ ;Check for double underscore and replace with single underscore
+ FOR i=0,in1-1 DO BEGIN
+ IF STRPOS(in2[i],'__') NE -1 THEN BEGIN
+ vnchange[i]=in2[i]
+ infotxt=STRARR(2)
+ infotxt[0]='2 Double underscore present in Variable Name '+in2[i]+'|'
+ infotxt[1]=' replaced with a single underscore'
+ INFOTXT_OUTPUT, infotxt
+ vns=STRSPLIT(in2[i],'_',/Extract,COUNT=n_vns)
+ FOR j=0,n_vns-1 DO IF j EQ 0 THEN in2[i]=vns[j] ELSE in2[i]=in2[i]+'_'+vns[j]
+ ENDIF
+ ENDFOR
+ END ;Case 1
+
+ code EQ 2: BEGIN ;Check for and Apply GEOMS rule changes for INDEPENDENT and axis variables
+ ;RULE: Any variable that is mentioned in VAR_DEPEND is by definition an axis variable
+ vdtest=STRUPCASE(in1) & vnv=STRUPCASE(in2) & vdv=STRUPCASE(in3) ;vdtest is an array of VAR_DEPEND variables
+ avi=WHERE(vnv EQ vdtest,avcnt) ;i.e. is the VAR_NAME an axis variable
+ IF avcnt NE 0 THEN BEGIN ;possible axis variable so do axis variable checks
+ test1=(vdv EQ 'INDEPENDENT') OR (vdv EQ vnv) ;value is independent or self-referencing
+ test2=(vdv NE 'INDEPENDENT') AND (vdv NE vnv) AND (vdv NE 'CONSTANT') ;neither independent, self-referencing, nor constant
+ test6=(vdv EQ 'CONSTANT') AND (vnv EQ 'DATETIME') ;i.e. VAR_NAME=DATETIME and VAR_DEPEND=CONSTANT
+ IF (test1) OR (test6) THEN BEGIN
+ IF vdv NE vnv THEN BEGIN
+ infotxt='2 '+in4+' should be self-referencing for axis variable VAR_NAME='+vnv+'|.'
+ infotxt=infotxt+' VAR_DEPEND value changed in metadata'
+ INFOTXT_OUTPUT,infotxt
+ in3=vnv ;self-referencing VAR_DEPEND value
+ ENDIF
+ ENDIF ELSE IF test2 THEN BEGIN
+ in4[0]='E1' ;i.e. axis variable cannot be dependent on another variable so return error
+ ENDIF
+ ENDIF ELSE BEGIN ;not an axis variable so ensure VAR_DEPEND value is not self-referencing
+ IF vdv EQ vnv THEN in4[0]='E3' ;i.e. VAR_DEPEND value cannot be self-referencing
+ ENDELSE
+ END ;Case 2
+
+ code EQ 3: BEGIN ;Change MJD2000 to MJD2K
+ in1='VAR_UNITS=MJD2K' & in2='MJD2K'
+ IF in3 EQ 0 THEN BEGIN ;First instance of MJD2000 so include comment
+ infotxt='2 VAR_UNITS=MJD2000 not GEOMS compliant|. Changed to VAR_UNITS=MJD2K'
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ END
+
+ code EQ 4: BEGIN ;checks on DATA_FILE_VERSION
+ in1=STRTRIM(in1,2)
+ vp=0 & errval=0 & in1h=in1
+ ON_IOERROR,TypeConversionError
+ IF STRMID(STRUPCASE(in1),0,1) EQ 'V' THEN BEGIN
+ vp=1 & in1=STRMID(in1,1)
+ ENDIF
+ dpos=STRPOS(in1,'.')
+ valid=0 ;to check for conversion error
+ IF dpos NE -1 THEN BEGIN
+ IF LONG(STRMID(in1,dpos+1)) EQ 0L THEN atxt='' ELSE atxt=STRMID(in1,dpos+1)
+ in1=STRMID(in1,0,dpos)+atxt
+ IF FIX(in1) GE 100 THEN in1=STRMID(in1,0,dpos) ;just use version up to the decimal point
+ ENDIF
+ IF (FIX(in1) GT 999) OR (FIX(in1) LT 1) THEN errval=1
+ valid=1 ;OK if got to here
+
+ TypeConversionError: ;Check for invalid DATA_FILE_VERSION value
+ IF (valid EQ 0) OR (errval EQ 1) THEN BEGIN
+ infotxt='3 DATA_FILE_VERSION='+in1h+' must be in the form ''nnn'''
+ INFOTXT_OUTPUT, infotxt
+ in1=in1h
+ ENDIF ELSE BEGIN ;format value correctly (nnn)
+ IF FIX(in1) LT 10 THEN in1='00'+STRTRIM(FIX(in1),2) $
+ ELSE IF FIX(in1) LT 100 THEN in1='0'+STRTRIM(FIX(in1),2) $
+ ELSE in1=STRTRIM(in1,2)
+ IF (vp EQ 1) OR (in1h NE in1) THEN BEGIN
+ infotxt='2 DATA_FILE_VERSION must be in the form ''nnn''|. Renamed from '
+ infotxt=infotxt+in1h+' to '+in1
+ INFOTXT_OUTPUT, infotxt
+ ENDIF
+ ENDELSE
+ END
+
+ code EQ 5: BEGIN ;Look for obsolete DATA_VARIABLES
+ ;Notes = what to do about ALTITUDE.LAYER.INDEX, CHL.A.CONCENTRATION,
+ ; PRESSURE.LEVEL.INDEX (still in 04R001), TSM.CONCENTRATION
+ ;List of obsolete DATA_VARIABLE_01 values
+ obs_list_1=['AIR.TEMPERATURE','TEMPERATURE.AIR','COLUMN.VERTICAL','H2O.RELATIVE.HUMIDITY',$
+ 'PRESSURE.SURFACE','TEMPERATURE.SURFACE','TEMPERATURE.INTERNAL.BOX',$
+ 'TEMPERATURE.INTERNAL.INSTRUMENT','TEMPERATURE.SEA.SURFACE','AIR.NUMBER.DENSITY',$
+ 'SEA.SURFACE.TEMPERATURE','OPACITY.ATMOSPHERIC.VERTICAL']
+ ;Corresponding list of replacement DATA_VARIABLE_1 values (direct replacement)
+ rep_list_1=['TEMPERATURE','TEMPERATURE','COLUMN','HUMIDITY.RELATIVE',$
+ 'SURFACE.PRESSURE','SURFACE.TEMPERATURE','INTERNAL.BOX.TEMPERATURE',$
+ 'INTERNAL.INSTRUMENT.TEMPERATURE','SEA.SURFACE.TEMPERATURE','NUMBER.DENSITY',$
+ 'SEA.SURFACE.TEMPERATURE.SKIN','OPACITY.ATMOSPHERIC']
+ ;Obsolete DATA_VARIABLE_01 which need special handling
+ obs_list_spec_1=['.AMF','.AVK','.CONCENTRATION','.MIXING.RATIO']
+ ;Obsolete DATA_VARIABLE_02 values
+ obs_list_2=['SOLAR','VERTICAL.SOLAR',$
+ 'EMISSION.VERTICAL','VERTICAL.EMISSION',$
+ 'VERTICAL','SLANT','SOLAR.OCCULTATION','VERTICAL.ZENITH','VERTICAL.NADIR',$
+ 'VERTICAL.LUNAR','VERTICAL.SOLAR.FOCUS']
+ ;[0,1] for FTIR/UVVIS (except Brewer and Dobson) only, [2,3] for MWR only
+ ;Corresponding list of replacement DATA_VARIABLE_2 values (direct replacement)
+ rep_list_2=['ABSORPTION.SOLAR|SCATTER.SOLAR','ABSORPTION.SOLAR|SCATTER.SOLAR',$
+ 'EMISSION','EMISSION',$
+ '','','OCCULTATION.SOLAR','ZENITH','NADIR',$
+ 'ABSORPTION.LUNAR','ABSORPTION.SOLAR.FOCUS']
+ ;Replacements for .CONCENTRATION - dependent on VAR_UNITS
+ rep_list_conc=['.MIXING.RATIO.VOLUME','.MIXING.RATIO.MASS','.NUMBER.DENSITY','.PARTIAL.PRESSURE']
+ ;Conditional Miscellaneous changes
+ misc_list_1=['TEMPERATURE_SKIN','AIR.MASS.FACTOR']
+ ;Corresponding change to miscellaneous variables
+ misc_list_2=['SEA.SURFACE.TEMPERATURE.SKIN','O3.COLUMN_AMF']
+ ;Obsolete DATA_VARIABLE_03 descriptors (may or may not be able to fix these)
+ obs_list_3=['DU','UNCERTAINTY.STDEV','UNCERTAINTY.RMS','UNCERTAINTY.RELATIVE','UNCERTAINTY.TOTAL', $
+ 'UNCERTAINTY.RANDOM','UNCERTAINTY.SYSTEMATIC']
+ ;Corresponding list of replacement DATA_VARIABLE_03 descriptors
+ rep_list_3=['','UNCERTAINTY.COMBINED.STANDARD','UNCERTAINTY.COMBINED.STANDARD', $
+ 'UNCERTAINTY.COMBINED.STANDARD.RELATIVE','UNCERTAINTY.COMBINED.STANDARD', $
+ 'UNCERTAINTY.RANDOM.STANDARD','UNCERTAINTY.SYSTEMATIC.STANDARD']
+ rep_list_4=['','','','','UNCERTAINTY.COMBINED.COVARIANCE','UNCERTAINTY.RANDOM.COVARIANCE', $
+ 'UNCERTAINTY.SYSTEMATIC.COVARIANCE']
+ vn_v=STRARR(in1,3) & vn_new=STRARR(in1)
+ ;Separate out Variable Name/[Mode or Descriptor]/[Descriptor]
+ FOR i=0,in1-1 DO BEGIN
+ res=STRSPLIT(in2[i],'_',/EXTRACT,COUNT=nres)
+ IF nres GT 3 THEN nres=3
+ FOR j=0,nres-1 DO vn_v[i,j]=res[j]
+ ENDFOR
+ ;Test for values in obs_list_1
+ FOR i=0,N_ELEMENTS(obs_list_1)-1 DO BEGIN
+ ;ri=WHERE(STRPOS(STRUPCASE(vn_v[*,0]),obs_list_1[i]) NE -1,rcnt)
+ ;Change to exact match from v4.0b55 (20200709) - due to introduction of DRY.AIR.... variable names
+ ri=WHERE(STRUPCASE(vn_v[*,0]) EQ obs_list_1[i],rcnt)
+ IF rcnt NE 0 THEN BEGIN ;obsolete values found so replace
+ vlen=STRLEN(obs_list_1[i])
+ FOR j=0,rcnt-1 DO BEGIN
+ change_val=1
+ ;Special case for SEA.SURFACE.TEMPERATURE (only change if mode is _SKIN)
+ IF obs_list_1[i] EQ 'SEA.SURFACE.TEMPERATURE' THEN BEGIN
+ IF STRUPCASE(vn_v[ri[j],1]) EQ 'SKIN' THEN vn_v[ri[j],1]='' ELSE change_val=0
+ ENDIF
+ IF change_val EQ 1 THEN BEGIN
+ vpos=STRPOS(STRUPCASE(vn_v[ri[j],0]),obs_list_1[i])
+ vn_v[ri[j],0]=STRMID(vn_v[ri[j],0],0,vpos)+rep_list_1[i]+STRMID(vn_v[ri[j],0],vpos+vlen)
+ ENDIF
+ ENDFOR
+ ENDIF
+ ENDFOR
+ ;Special cases for DATA_VARIABLE_01 values .AMF, .AVK, .CONCENTRATION, and .MIXING.RATIO
+ nols1=N_ELEMENTS(obs_list_spec_1)
+ FOR i=0,nols1-1 DO BEGIN
+ ri=WHERE(STRPOS(STRUPCASE(vn_v[*,0]),obs_list_spec_1[i]) NE -1,rcnt)
+ IF (i EQ nols1-1) AND (rcnt NE 0) THEN BEGIN
+ ;do check for .MIXING.RATIO. in which case no correction required
+ rxi=WHERE(STRPOS(STRUPCASE(vn_v[ri,0]),'.MIXING.RATIO.') EQ -1,rxcnt)
+ IF rxcnt NE 0 THEN BEGIN ;some values are just .MIXING.RATIO so correct these only
+ rcnt=rxcnt & ri=ri[rxi]
+ ENDIF ELSE rcnt=0
+ ENDIF
+ IF rcnt NE 0 THEN BEGIN ;obsolete values found
+ vlen=STRLEN(obs_list_spec_1[i])
+ IF i LE 1 THEN BEGIN ;move .AMF and .AVK to DESCRIPTOR section (2 or 3)
+ FOR j=0,rcnt-1 DO BEGIN
+ vpos=STRPOS(STRUPCASE(vn_v[ri[j],0]),obs_list_spec_1[i])
+ ai=WHERE(vn_v[ri[j],*] EQ '',acnt)
+ IF acnt NE 0 THEN BEGIN
+ ;the DESCRIPTOR field is empty, so can convert value OK (otherwise no change made)
+ vn_v[ri[j],0]=STRMID(vn_v[ri[j],0],0,vpos)
+ vn_v[ri[j],ai[0]]=STRMID(obs_list_spec_1[i],1) ;write to first empty field
+ ENDIF
+ ENDFOR
+ ENDIF ELSE BEGIN
+ ;.CONCENTRATION or .MIXING.RATIO so attempt to identify actual type of measurement
+ ;(mixing ratio etc) by looking at corresponding VAR_UNIT values
+ mrvi=WHERE((STRPOS(STRLOWCASE(in3[ri]),'pp') NE -1) OR $
+ (STRPOS(STRLOWCASE(in3[ri]),'m-1 sr-1') NE -1),mrvcnt) ;testing for volume mixing ratio
+ mrmi=WHERE(STRPOS(STRLOWCASE(in3[ri]),' g ') NE -1,mrmcnt) ;test for mass mixing ratio
+ ndi=WHERE(STRPOS(STRLOWCASE(in3[ri]),'mol') NE -1,ndcnt) ;test for number density
+ ppi=WHERE(STRPOS(STRLOWCASE(in3[ri]),'pa') NE -1,ppcnt) ;test for partial pressure
+ IF i EQ nols1-1 THEN cnti=[mrvcnt,mrmcnt,0,0] ELSE cnti=[mrvcnt,mrmcnt,ndcnt,ppcnt]
+ ci=WHERE(cnti GT 0,ccnt)
+ IF ccnt EQ 1 THEN BEGIN ;type of measurement successfully found
+ ;Note: not set up to handle if more than one type of measurement found in the file
+ vlen=STRLEN(obs_list_spec_1[i])
+ FOR j=0,rcnt-1 DO BEGIN
+ IF (STRUPCASE(vn_v[ri[j],0]) EQ 'AIR.CONCENTRATION') AND (ci[0] EQ 2) THEN $
+ ;Do special case check for AIR.CONCENTRATION - can only change to NUMBER.DENSITY
+ vn_v[ri[j],0]='NUMBER.DENSITY' $
+ ELSE BEGIN
+ vpos=STRPOS(STRUPCASE(vn_v[ri[j],0]),obs_list_spec_1[i])
+ vn_v[ri[j],0]=STRMID(vn_v[ri[j],0],0,vpos)+rep_list_conc[ci[0]]+STRMID(vn_v[ri[j],0],vpos+vlen)
+ ENDELSE
+ ENDFOR
+ ENDIF
+ ENDELSE
+ ENDIF
+ ENDFOR
+ FOR i=0,N_ELEMENTS(obs_list_2)-1 DO BEGIN ;Check for obsolete mode values
+ ri=WHERE(STRUPCASE(vn_v[*,1]) EQ obs_list_2[i],rcnt)
+ IF rcnt NE 0 THEN BEGIN
+ test1=(i LE 1) AND (STRPOS(STRUPCASE(in4),'FTIR') NE -1)
+ test2=(i LE 1) AND (STRPOS(STRUPCASE(in4),'UVVIS') NE -1) AND $
+ (STRPOS(STRUPCASE(in4),'DOBSON') EQ -1) AND (STRPOS(STRUPCASE(in4),'BREWER') EQ -1)
+ test3=((i EQ 2) OR (i EQ 3)) AND ((STRPOS(STRUPCASE(in4),'MWR') NE -1) OR $
+ (STRPOS(STRUPCASE(in4),'MICROWAVE') NE -1))
+ IF i LE 1 THEN mi=STRSPLIT(rep_list_2[i],'|',/EXTRACT)
+ ;test1, test2, and test3 are special case obsolete values
+ IF test1 THEN vn_v[ri,1]=mi[0] $
+ ELSE IF test2 THEN vn_v[ri,1]=mi[1] $
+ ELSE IF test3 THEN vn_v[ri,1]=rep_list_2[i] $ ;direct replacement
+ ELSE BEGIN ;all other obsolete mode values
+ IF i LE 1 THEN vn_v[ri,1]=mi[0] ELSE vn_v[ri,1]=rep_list_2[i] ;direct replacement
+ IF i EQ 5 THEN BEGIN ;SLANT (if column measurement then add to main variable name)
+ FOR j=0,rcnt-1 DO BEGIN
+ IF STRLEN(vn_v[ri[j],0])-STRPOS(vn_v[ri[j],0],'COLUMN') EQ 6 THEN $
+ vn_v[ri[j],0]=vn_v[ri[j],0]+'.'+obs_list_2[i]
+ ENDFOR
+ ENDIF
+ ENDELSE
+ ENDIF
+ ENDFOR
+ ;Check conditional miscellaneous changes
+ FOR i=0,N_ELEMENTS(misc_list_1)-1 DO BEGIN
+ IF i EQ 0 THEN ri=WHERE(STRPOS(STRUPCASE(in2),misc_list_1[i]) NE -1,rcnt) $
+ ELSE ri=WHERE(STRUPCASE(in2) EQ misc_list_1[i],rcnt)
+ IF rcnt NE 0 THEN BEGIN
+ IF i EQ 0 THEN BEGIN ;ensure DATA_SOURCE is BUOY before changing
+ IF STRPOS(STRUPCASE(in4),'BUOY') NE -1 THEN BEGIN
+ FOR j=0,rcnt-1 DO BEGIN
+ vn_v[ri[j],0]=misc_list_2[i] & vn_v[ri[j],1]=''
+ ENDFOR
+ ENDIF
+ ENDIF ELSE BEGIN ;ensure O3.COLUMN is also present as a DATA_VARIABLE before changing
+ mi=WHERE(STRUPCASE(vn_v[*,0]) EQ 'O3.COLUMN',mcnt)
+ IF mcnt NE 0 THEN BEGIN
+ vn_v[ri[0],0]='O3.COLUMN' & vn_v[ri[0],1]='AMF'
+ ENDIF
+ ENDELSE
+ ENDIF
+ ENDFOR
+
+ FOR i=0,N_ELEMENTS(obs_list_3)-1 DO BEGIN ;Check for obsolete descriptor values (UNCERTAINTY)
+ ;acceptable units for identifying correct replacement name (test1 instruments only)
+ vuok=['ppv','ppmv','ppbv','pptv','k','du','molec cm-2','molec cm-3','degc','m-1 sr-1','dimensionless']
+ vuok2=['ppv2','ppmv2','ppbv2','pptv2'] ;covariance
+ test1=(STRPOS(STRUPCASE(in4),'MWR') NE -1) OR (STRPOS(STRUPCASE(in4),'MICROWAVE') NE -1) OR $
+ (STRPOS(STRUPCASE(in4),'HAGAR') NE -1) OR (STRPOS(STRUPCASE(in4),'MIPAS.STR') NE -1) OR $
+ (STRPOS(STRUPCASE(in4),'SIOUX') NE -1) OR (STRPOS(STRUPCASE(in4),'UVVIS.AMAXDOAS') NE -1) OR $
+ (STRPOS(STRUPCASE(in4),'UVVIS.SAOZ') NE -1) OR (STRPOS(STRUPCASE(in4),'ATMOINSPECTOR') NE -1) OR $
+ (STRPOS(STRUPCASE(in4),'FISH') NE -1) OR (STRPOS(STRUPCASE(in4),'AMON') NE -1) OR $
+ (STRPOS(STRUPCASE(in4),'LPMA') NE -1) OR (STRPOS(STRUPCASE(in4),'SALOMON') NE -1) OR $
+ (STRPOS(STRUPCASE(in4),'RADIOMETER.IR.CIMEL') NE -1) OR (STRPOS(STRUPCASE(in4),'UVVIS') NE -1) OR $
+ (STRPOS(STRUPCASE(in4),'SPECTROMETER') NE -1) OR (STRPOS(STRUPCASE(in4),'LIDAR.WATERVAPORRAMAN_UNIVAQ') NE -1)
+ ;check for DATA_SOURCE where CoVariance as well as Standard uncertainties are reported
+ test2=(STRPOS(STRUPCASE(in4),'FTIR.CO_KIT') NE -1) OR (STRPOS(STRUPCASE(in4),'FTIR.CO_ULG') NE -1) OR $
+ (STRPOS(STRUPCASE(in4),'FTIR.HNO3_KIT') NE -1) OR (STRPOS(STRUPCASE(in4),'FTIR.HNO3_ULG') NE -1) OR $
+ (STRPOS(STRUPCASE(in4),'FTIR.N2O_KIT') NE -1) OR (STRPOS(STRUPCASE(in4),'FTIR.N2O_ULG') NE -1) OR $
+ (STRPOS(STRUPCASE(in4),'FTIR.NO2_KIT') NE -1) OR (STRPOS(STRUPCASE(in4),'FTIR.NO2_ULG') NE -1)
+ vnvi=1 ;determines the location index of the descriptor variable
+ ri=WHERE(STRUPCASE(vn_v[*,1]) EQ obs_list_3[i],rcnt)
+ IF rcnt EQ 0 THEN BEGIN
+ vnvi=2 & ri=WHERE(STRUPCASE(vn_v[*,2]) EQ obs_list_3[i],rcnt)
+ ENDIF
+ IF rcnt NE 0 THEN BEGIN
+ IF i LE 3 THEN vn_v[ri,vnvi]=rep_list_3[i] $ ;direct replacement
+ ELSE BEGIN ;need to identify whether the UNITS are % or fit vuok criteria
+ FOR j=0,rcnt-1 DO BEGIN
+ IF STRTRIM(in3[ri[j]],2) EQ '%' THEN vn_v[ri[j],vnvi]=rep_list_3[i]+'.RELATIVE' $
+ ELSE BEGIN
+ ;test for Standard Uncertainties
+ vi=WHERE(STRTRIM(STRLOWCASE(in3[ri[j]]),2) EQ vuok,vcnt)
+ IF vcnt EQ 0 THEN $ ;do test for [Prefix]molec cm-2/molec cm-3
+ IF (STRPOS(STRLOWCASE(in3[ri[j]]),'molec cm-2') NE -1) OR $
+ (STRPOS(STRLOWCASE(in3[ri[j]]),'molec cm-3') NE -1) THEN vcnt=1
+ IF ((test1) OR (test2)) AND (vcnt NE 0) THEN $ can assume that the given error values are Standard Deviation
+ vn_v[ri[j],vnvi]=rep_list_3[i]
+ ;test for Covariance Uncertainties
+ vi=WHERE(STRTRIM(STRLOWCASE(in3[ri[j]]),2) EQ vuok2,vcnt)
+ IF (test2) AND (vcnt NE 0) THEN vn_v[ri[j],vnvi]=rep_list_4[i]
+ ENDELSE
+ ENDFOR
+ ENDELSE
+ ENDIF
+ ENDFOR
+ ;create any comments for INFOTXT_OUTPUT
+ FOR i=0,in1-1 DO BEGIN
+ ;put full DATA_VARIABLE names back together
+ vn_new[i]=vn_v[i,0]
+ FOR j=1,2 DO IF vn_v[i,j] NE '' THEN vn_new[i]=vn_new[i]+'_'+vn_v[i,j]
+ ;Check to see if it is different to the original DATA_VARIABLE
+ IF vn_new[i] NE in2[i] THEN BEGIN
+ vnchange[i]=in2[i] & in2[i]=vn_new[i]
+ IF qa_yes THEN itxt=' expected to be ' ELSE itxt=' renamed '
+ infotxt='2 Dataset name '+vnchange[i]+itxt+in2[i]
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ ENDFOR
+ END
+
+ code EQ 6: BEGIN ;Look for obsolete DATA_SOURCE
+ ;List of obsolete DATA_SOURCE_01 values
+ obs_list=['RO.CHAMP','RO.SAC.C','RO.F3C.FM','RO.CNOFS','RO.GRACE.A','FTIR_', $
+ 'UVVIS.DOAS_','MICROWAVE.RADIOMETER','LIDAR.BACKSCATTER',$
+ 'LIDAR.DIAL','LIDAR.OLEX','LIDAR.RIEGL','LIDAR.RMR']
+ ;Corresponding replacement list
+ rep_list=['CHAMP','SAC.C','F3C.FM','CNOFS','GRACE.A','FTIR.','UVVIS.DOAS.','MWR.','LIDAR.']
+ ;Possible list of species for the above instruments (note: gases will take precedence
+ ;over Aerosol and Temperature, o/w the first species found will be considered the
+ ;primary species measured)
+ spc_list=['BrO.','C2H2.','C2H6.','CCl2F2.','CCl3F.','CH4.','CHF2Cl.','CHOCHO.','ClO.','ClONO2.','CO.',$
+ 'CO2.','COF2.','H2CO.','H2O.','HCFC22.','HCl.','HCN.','HCOOH.','HF.','HNO3.','HONO.','IO.',$
+ 'N2O.','NO.','NO2.','O3.','OClO.','OCS.','SF6.','SO2.','AEROSOL','TEMPERATURE']
+
+ oli=-1 ;will change to obs_list index value if obsolete DATA_SOURCE_01 found
+ n_obsl=N_ELEMENTS(obs_list)
+ lid_i=n_obsl-5 ;needs to be start index of the lidar measurements in obs_list
+ spc_i=n_obsl-8 ;needs to be start index of the instruments requiring species information in obs_list
+ uv_i=n_obsl-7 ;needs to be UVVIS.DOAS index in obs_list
+ FOR i=0,N_ELEMENTS(obs_list)-1 DO IF STRPOS(STRUPCASE(in3),obs_list[i]) NE -1 THEN oli=i
+ IF oli NE -1 THEN BEGIN ;obsolete DATA_SOURCE value found
+ IF oli GE lid_i THEN oli=lid_i ;to correspond with rep_list index for lidar
+ pri=-1 ;will change if species found
+ IF oli GE spc_i THEN BEGIN ;if not an RO DATA_SOURCE then need to determine species
+ n_sl=N_ELEMENTS(spc_list)
+ ;Look through VAR_NAME values to try and identify new DATA_SOURCE from species list
+ spc_found=INTARR(in1)-1 ;identifies index of found species names in the Datasets
+ FOR i=0,n_sl-1 DO BEGIN
+ fi=WHERE(STRPOS(STRMID(in2,0,STRLEN(spc_list[i])),spc_list[i]) NE -1,fcnt)
+ IF fcnt NE 0 THEN spc_found[fi]=i
+ ENDFOR
+ ;Check list of found species - don't include Aerosol and Temperature
+ fi=WHERE((spc_found NE -1) AND (spc_found LT n_sl-2),fcnt)
+ IF fcnt NE 0 THEN pri=spc_found[fi[0]] $ ;primary species index determined
+ ELSE IF (oli EQ uv_i) OR (oli EQ lid_i) THEN BEGIN ;Test for Temperature (Lidar Only) and Aerosol
+ fi=WHERE(spc_found NE -1,fcnt)
+ IF fcnt NE 0 THEN BEGIN
+ pri=MIN(spc_found[fi]) ;i.e. Aerosol takes precedence over Temperature
+ IF (oli EQ uv_i) AND (pri EQ n_sl-1) THEN pri=-1 ;UVVIS.DOAS can't be Temperature
+ ENDIF
+ ENDIF
+ ENDIF ELSE BEGIN ;cases with straight replacement
+ old_ds=STRTRIM(in3,2) & in3=rep_list[oli]+STRMID(old_ds,STRLEN(obs_list[oli])) & pri=-2
+ ENDELSE
+ IF pri NE -1 THEN BEGIN ;update DATA_SOURCE
+ IF pri GE 0 THEN BEGIN
+ old_ds=in3
+ ;check special cases for LIDAR water vapor measurements
+ IF (spc_list[pri] EQ 'H2O.') AND (STRPOS(STRUPCASE(in3),'LIDAR.DIAL') NE -1) THEN $
+ in3='H2O' $
+ ELSE IF (spc_list[pri] EQ 'H2O.') AND (STRPOS(STRUPCASE(in3),'LIDAR.BACKSCATTER') NE -1) THEN $
+ in3='H2O' $
+ ELSE in3=spc_list[pri]
+ IF STRPOS(in3,'.') NE -1 THEN in3=STRMID(in3,0,STRLEN(in3)-1)
+ in3=rep_list[oli]+in3+STRMID(old_ds,STRPOS(old_ds,'_'))
+ ENDIF
+ IF qa_yes THEN itxt=' expected to be ' ELSE itxt=' renamed '
+ infotxt='2 DATA_SOURCE value '+old_ds+itxt+in3
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ ENDIF
+ END
+
+ code EQ 7: BEGIN ;Look for obsolete CALVAL and NDSC FILE_ACCESS values
+ obs_list=['CALVAL','NDSC']
+ rep_list=['EVDC','NDACC']
+
+ FOR i=0,in1-1 DO BEGIN
+ oi=WHERE(in2[i] EQ obs_list,ocnt)
+ IF ocnt NE 0 THEN BEGIN
+ IF qa_yes THEN itxt=' expected to be ' ELSE itxt=' renamed '
+ infotxt='2 FILE_ACCESS value '+in2[i]+itxt+rep_list[oi[0]]
+ INFOTXT_OUTPUT,infotxt
+ in2[i]=rep_list[oi[0]]
+ ENDIF
+ ENDFOR
+ END
+
+ code EQ 8: BEGIN ;Look for VAR_UNITS=DIMENSIONLESS or NONE and replace with '' or '1'
+ vuv=STRUPCASE(in2)
+ FOR i=0,in1-1 DO BEGIN
+ IF (vuv[i] EQ 'DIMENSIONLESS') OR (vuv[i] EQ 'NONE') THEN BEGIN
+ IF in3[i] EQ 'STRING' THEN newv='' ELSE newv='1'
+ IF qa_yes THEN itxt=' expected to be ' ELSE itxt=' renamed '
+ infotxt='2 VAR_UNITS='+vuv[i]+itxt+'VAR_UNITS='+newv+' based on VAR_DATA_TYPE='+in3[i]
+ INFOTXT_OUTPUT,infotxt
+ in2[i]=newv
+ ENDIF
+ ENDFOR
+ END
+
+ code EQ 9: BEGIN ;DATA_QUALITY/DATA_TEMPLATE checks
+ attr_arr_glob_prov=['DATA_TEMPLATE','DATA_QUALITY']
+ geoms_te='' & meta_te='' & std_txt=' Required Metadata Template: '
+ ;Check if DATA_TEMPLATE field is present in the TAV file
+ dti=WHERE(tab_arr[*,0] EQ attr_arr_glob_prov[0],dtcnt)
+ ;Check if DATA_TEMPLATE label is present in the Metadata
+ gi=WHERE(in1 EQ attr_arr_glob_prov[0],gcnt)
+ IF gcnt EQ 1 THEN IF in2[gi[0]] NE '' THEN meta_te=STRUPCASE(STRTRIM(in2[gi[0]],2))
+ IF dtcnt NE 0 THEN tab_arr_sub=tab_arr[dti[0],2:FIX(tab_arr[dti[0],1]+1)] ;Valid Template values from TAV file
+ IF dtcnt EQ 0 THEN BEGIN
+ infotxt='3'+std_txt+'DATA_TEMPLATE field is not present in the TAV file. Checks cannot be performed'
+ INFOTXT_OUTPUT,infotxt
+ std_txt=''
+ ENDIF ELSE IF meta_te NE '' THEN BEGIN
+ ;check if value is present in the TAV file
+ ci=WHERE(STRUPCASE(tab_arr_sub) EQ meta_te,ccnt)
+ IF ccnt NE 0 THEN BEGIN
+ geoms_te=tab_arr_sub[ci[0]] & in2[gi[0]]=geoms_te
+ infotxt='0'+std_txt+geoms_te
+ INFOTXT_OUTPUT,infotxt
+ std_txt=''
+ ENDIF
+ ENDIF
+ IF std_txt NE '' THEN BEGIN ;template value not present in the metadata or value not in the TAV file
+ ;Determine DATA_TEMPLATE value based on DATA_SOURCE value (if applicable)
+ di=WHERE(in1 EQ 'DATA_SOURCE',dcnt)
+ IF dcnt EQ 1 THEN BEGIN ;DATA_SOURCE found
+ ;Extract DATA_SOURCE value
+ IF in2[di[0]] NE '' THEN BEGIN
+ res=STRSPLIT(STRTRIM(in2[di[0]],2),'_',/EXTRACT)
+ ;First check - First part of DATA_SOURCE matches text in TAV entry
+ res1=STRUPCASE(STRSPLIT(res[0],'.',/EXTRACT,COUNT=res1cnt))
+ ci=WHERE(STRPOS(STRUPCASE(tab_arr_sub),res1[0]) NE -1,ccnt)
+ IF ccnt EQ 1 THEN geoms_te=tab_arr_sub[ci[0]] $ ;Template found
+ ELSE IF (ccnt GT 1) AND (N_ELEMENTS(res1) GT 1) THEN BEGIN
+ ;2nd Check - More than one option so try and match 2nd part of DATA_SOURCE with 4th part in TAV entry
+ tab_arr_sub=tab_arr_sub[ci]
+ tab_arr_part_sub=STRARR(ccnt)
+ numi=-1
+ FOR i=0,ccnt-1 DO BEGIN
+ rest=STRSPLIT(tab_arr_sub[i],'-',/EXTRACT)
+ rest=[rest,'','',''] ;ensure there are at least 4 sub-values
+ tab_arr_part_sub[i]=rest[3]
+ IF IS_A_NUMBER_HDF(rest[3]) THEN numi=i
+ ENDFOR
+ ci=WHERE(STRUPCASE(tab_arr_part_sub) EQ res1[1],ccnt)
+ IF (ccnt EQ 0) AND (numi NE -1) THEN BEGIN
+ ;2nd part is likely a gas e.g. FTIR.CO, so use template where 4th part of TAV entry is a number
+ geoms_te=tab_arr_sub[numi]
+ ENDIF ELSE IF ccnt EQ 1 THEN geoms_te=tab_arr_sub[ci[0]] $ ;unique template found
+ ELSE IF ccnt GT 1 THEN BEGIN ;ccnt GT 1, Try alternatives for H2O and O3, or test for fourth part
+ tasx=tab_arr_sub[ci]
+ CASE 1 OF
+ (res1[0] EQ 'UVVIS') AND (res1[1] EQ 'DOAS'): BEGIN
+ IF res1cnt GE 4 THEN BEGIN
+ IF res1[3] EQ 'AEROSOL' THEN res1x=STRUPCASE(res1[2])+'-AEROSOL' $
+ ELSE res1x=STRUPCASE(res1[2])+'-GAS'
+ ENDIF ELSE res1x=res1[1]
+ END
+ res1[1] EQ 'H2O': res1x='WATERVAPOR'
+ res1[1] EQ 'O3': res1x='OZONE'
+ ELSE: res1x=res1[1]
+ ENDCASE
+ ci=WHERE(STRPOS(STRUPCASE(tasx),res1x) NE -1,ccnt)
+ IF ccnt EQ 1 THEN geoms_te=tasx[ci[0]]
+ ENDIF
+ ENDIF
+ IF geoms_te NE '' THEN BEGIN ;DATA_TEMPLATE value found based on DATA_SOURCE
+ IF gcnt EQ 1 THEN BEGIN ;DATA_TEMPLATE label is present
+ fmvi=WHERE(in1 EQ 'FILE_META_VERSION',fmvcnt)
+ fmvtxt=''
+ IF fmvcnt EQ 1 THEN BEGIN
+ resfmv=STRSPLIT(in2[fmvi[0]],' ;',/EXTRACT,COUNT=fmvcnt)
+ IF fmvcnt GE 1 THEN BEGIN
+ fmv_ver=resfmv[0]
+ fmvtxt=' and '+fmv_ver+' metadata definitions'
+ ENDIF
+ ENDIF
+ in2[gi[0]]=geoms_te ;Add value to DATA_TEMPLATE label
+ IF qa_yes THEN itxt=' expected to be ' ELSE itxt=' renamed '
+ infotxt='2 DATA_TEMPLATE value'+itxt+geoms_te+' based on DATA_SOURCE value'+fmvtxt
+ INFOTXT_OUTPUT,infotxt
+ std_txt=''
+ ENDIF ELSE IF gcnt EQ 0 THEN in3[0]=geoms_te ;Need to add new Global Attribute Information
+ ENDIF
+ ENDIF
+ ENDIF
+ ENDIF
+ IF geoms_te NE '' THEN BEGIN ;Check if DATA_QUALITY label is present
+ gi=WHERE(in1 EQ attr_arr_glob_prov[1],gcnt)
+ IF gcnt EQ 0 THEN in3[1]='DATA_QUALITY' ;Need to add new Global Attribute Information
+ ENDIF
+ IF std_txt NE '' THEN BEGIN ;No template file required based on selection criteria
+ infotxt='0'+std_txt+'DATA_TEMPLATE value not required based on selection criteria'
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ END
+
+ code EQ 10: BEGIN ;check printable characters in Metadata and Datasets (of type String)
+ ;Note: Also allowed are Control Characters Null Character (0B), Horizontal Tab (9B),
+ ;Line Feed (10B), and Carriage Return (13B) - Note: 10B and 13B for Metadata only
+ exempt=['PI_NAME','DO_NAME','DS_NAME','PI_AFFILIATION','DO_AFFILIATION','DS_AFFILIATION',$
+ 'PI_ADDRESS','DO_ADDRESS','DS_ADDRESS','PI_EMAIL','DO_EMAIL','DS_EMAIL']
+ non_char=[27B, 128B, 147B, 176B, 186B, 197B, 216B, 226B, 232B, 233B, 248B, 252B] ;128B = Euro
+ rep_char=['',' ','"',' deg. ',' deg. ','A','O','a','e','e','o','u'] ;replace above non-US ASCII chars with these values
+
+ ;Do check for the Full TAV which includes ORIGINATOR attributes
+ ochk=STRPOS(tab_arr(*,0),'ORIGINATOR') NE -1
+ oi=WHERE(ochk NE 0,ocnt)
+ mentry=0 ;Default assumes String Dataset Entry
+ n_nchar=N_ELEMENTS(non_char)
+ IF N_PARAMS() EQ 3 THEN in2='Dataset '+STRTRIM(in2,2) $ ;Dataset Entry
+ ELSE mentry=1 ;Metadata entry
+ FOR i=0L,N_ELEMENTS(in1)-1L DO BEGIN
+ exempt_att=0 ;default assumes attribute is not exempt from checks
+ IF mentry EQ 1 THEN BEGIN
+ res=STRSPLIT(in1[i],'=',/EXTRACT)
+ in2='Metadata entry '+in1[i] ;res[0]
+ IF ocnt NE 0 THEN BEGIN
+ ;Do not need to check 'exempt' attributes if using the full version of the TAV file
+ ei=WHERE(STRUPCASE(res[0]) EQ exempt,ecnt)
+ IF ecnt NE 0 THEN exempt_att=1 ;invalid entries can be corrected by the code
+ ENDIF
+ ENDIF
+
+ IF (~qa_yes) AND (exempt_att EQ 0) THEN BEGIN
+ ;If not doing QA then check for characters that can be replaced
+ non_char_i=[-1]
+ FOR j=0,n_nchar-1 DO BEGIN
+ testb=BYTE(in1[i])
+ ni=WHERE(testb EQ non_char[j],ncnt)
+ IF ncnt NE 0 THEN BEGIN
+ IF non_char_i[0] EQ -1 THEN non_char_i=[j] $
+ ELSE non_char_i=[non_char_i,j]
+ inval=in1[i]
+ FOR k=0,ncnt-1 DO BEGIN
+ IF k NE 0 THEN ni[k]=ni[k]+((STRLEN(rep_char[j])-1)*k)
+ inval=STRMID(inval,0,ni[k])+rep_char[j]+STRMID(inval,ni[k]+1)
+ ENDFOR
+ in1[i]=inval
+ ENDIF
+ ENDFOR
+ IF non_char_i[0] NE -1 THEN BEGIN
+ infotxt='2 Illegal character(s) '
+ FOR j=0,N_ELEMENTS(non_char_i)-1 DO BEGIN
+ IF j EQ 0 THEN infotxt=infotxt+STRTRIM(non_char[non_char_i[j]],2) $
+ ELSE infotxt=infotxt+', '+STRTRIM(non_char[non_char_i[j]],2)
+ ENDFOR
+ infotxt=infotxt+' in '+in2+' replaced with valid ISO646-US ASCII character(s)'
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ ENDIF
+ testb=BYTE(in1[i])
+
+ IF exempt_att EQ 0 THEN BEGIN
+ IF mentry EQ 1 THEN $
+ test=(testb NE 0B) AND (testb NE 9B) AND (testb NE 10B) AND (testb NE 13B) $
+ ELSE test=(testb NE 0B) AND (testb NE 9B) ;Check for Dataset Control Characters
+ ;check for non-char value and replace
+ bi=WHERE(((testb LT 32B) OR (testb GT 126B)) AND (test),bcnt)
+ IF bcnt NE 0 THEN BEGIN ;Illegal character present
+ IF bcnt GT 1L THEN BEGIN ;check for repeat illegal characters and only display once
+ bix=[bi[0]]
+ FOR j=1L,bcnt-1L DO BEGIN
+ ri=WHERE(testb[bi[j]] EQ testb[bi[0L:j-1L]],rcnt)
+ IF rcnt EQ 0 THEN bix=[bix,bi[j]]
+ ENDFOR
+ bi=bix & bcnt=N_ELEMENTS(bi)
+ ENDIF
+ infotxt=STRARR(2)
+ infotxt[0]='3 Entry must only include valid characters from the ISO646-US ASCII character set.'
+ infotxt[1]=' Illegal character(s) '
+ FOR j=0L,bcnt-1L DO BEGIN
+ cval=STRTRIM(testb[bi[j]],2)
+ ;print,testb[bi[j]] ;to output actual byte value
+ IF (cval EQ STRTRIM(8B,2)) OR (cval EQ '') THEN BEGIN
+ ;Illegal non-printable character so print in Decimal Byte form
+ cval=STRING(format='(I4)',testb[bi[j]]) & cval=STRTRIM(cval,2)+'B'
+ ENDIF
+ IF j NE bcnt-1L THEN itxt=', ' ELSE itxt=''
+ infotxt[1]=infotxt[1]+cval+itxt
+ ENDFOR
+ infotxt[1]=infotxt[1]+' in '+in2+' (may not display correctly)'
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ ENDIF
+ ENDFOR
+ END
+ code EQ 11: BEGIN ;Check DATA_VERSION_NAME part of the DATA_SOURCE
+ ;Check that entry is made up of dot-separated alpha_numeric words
+ n_char=STRLEN(in1)
+ strok=1B ;default is that entry is OK
+ test2=0B & test3=0B ;will change to 1 if test conditions are true
+ charchkm1=0B ;to check test character against the previous character
+ FOR i=0,n_char-1 DO BEGIN
+ charchk=BYTE(STRMID(in1,i,1))
+ test1=(charchk EQ 46B) OR ((charchk GE 65B) AND (charchk LE 90B)) OR $ ;'.' or A-Z
+ ((charchk GE 97B) AND (charchk LE 122B)) OR $ ;a-z
+ ((charchk GE 48B) AND (charchk LE 57B)) ;0-9
+ IF (i EQ 0) OR (i EQ n_char-1) THEN test2=charchk EQ 46B ;i.e. first or last character is a '.'
+ test3=(charchk EQ 46B) AND (charchkm1 EQ 46B) ;i.e. two consecutive dots
+ IF (~test1) OR (test2) OR (test3) THEN strok=0B
+ charchkm1=charchk
+ ENDFOR
+ IF ~strok THEN BEGIN
+ ;write error message
+ infotxt='3 VERSION_NAME sub-value of the DATA_SOURCE must consist of dot-separated alpha-numeric words: '
+ infotxt=infotxt+in1
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ END
+ENDCASE
+
+END ;Procedure GEOMS_Rule_Changes
+
+
+
+PRO pre_defined_att_checks, ct, nav, vav, vnx, vdtv,verror
+;Procedure to perform checks on the HDF4 pre-defined attributes, as well as check that
+;the numeric variable attribute values are of the same data type as that given by the
+;corresponding VAR_DATA_TYPE value (if input is via session memory)
+; ----------
+;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
+;
+; History:
+; 20101120: Introduced - Version 4.0
+;
+; Inputs: ct - Identifies the type of check to be performed on the inputs
+; nav - The HDF pre-defined attribute to check
+; vav - The Variable Attribute corresponding to the pre-defined attribute
+; vnx - The Dataset from which the attributes are derived
+; vdtv - The Data Type of the Dataset
+; verror - Boolean to indicate whether the Variable Attribute data type error has
+; already been written
+;
+; Outputs: None - Any Pre-defined attributes written to an HDF file are recreated by the program
+;
+; Called by: IDLCR8HDF, READ_METADATA
+;
+; Subroutines Called: INFOTXT_OUTPUT
+; Information Conditions (when the program is able to make changes):
+; 1. VAR_VALID_MIN/MAX or VAR_FILL_VALUE data types do not match VAR_DATA_TYPE
+; 2. HDF4 pre-defined attribute data type does not match VAR_DATA_TYPE
+; 3. HDF4 pre-defined attribute units value does not match VAR_UNITS
+; 4. HDF4 pre-defined attribute value does not equal equivalent variable
+; attribute value (valid_range, _FillValue)
+; 5. HDF4 pre-defined attribute [long_name/format/coordsys] present in the HDF file
+; 6. Datasets that have been scaled or had an offset applied are not permitted
+
+COMMON DATA
+COMMON WIDGET_WIN
+
+IF ct NE -1 THEN BEGIN ;check units, _FillValue, and valid_range attributes only
+ ncsa=['units','_Fillvalue','valid_range','valid_range']
+ var_atts=['VAR_UNITS','VAR_FILL_VALUE','VAR_VALID_MIN','VAR_VALID_MAX']
+
+ ;Change VAR_DATA_TYPE value to IDL Type code
+ CASE 1 OF
+ vdtv EQ 'BYTE': idltc=1
+ vdtv EQ 'SHORT': idltc=2
+ (vdtv EQ 'INTEGER') OR (vdtv EQ 'LONG'): idltc=3
+ (vdtv EQ 'REAL') OR (vdtv EQ 'FLOAT'): idltc=4
+ vdtv EQ 'DOUBLE': idltc=5
+ vdtv EQ 'STRING': idltc=7
+ ELSE: idltc=0
+ ENDCASE
+ test1=(SIZE(vav,/TYPE) EQ 7) AND (STRTRIM(vav,2) NE '')
+ test2=SIZE(vav,/TYPE) NE 7
+ test3=(SIZE(nav,/TYPE) EQ 7) AND (STRTRIM(nav,2) NE '')
+ test4=SIZE(nav,/TYPE) NE 7
+ IF ((test1) OR (test2)) AND (ct NE 0) THEN BEGIN
+ IF (SIZE(vav,/TYPE) NE idltc) AND (idltc NE 0) AND (verror[0] EQ 0) THEN BEGIN
+ ;Check data type of Variable Attribute
+ infotxt=STRARR(2)
+ itxt='VAR_VALID_MIN, VAR_VALID_MAX or VAR_FILL_VALUE'
+ infotxt[0]='2 '+itxt+' data types do not match VAR_DATA_TYPE='+vdtv
+ IF vnx NE '' THEN infotxt[1]=' for Dataset '+vnx+'|.' $
+ ELSE BEGIN
+ infotxt[0]=infotxt[0]+'|.' & infotxt[1]=' '
+ ENDELSE
+ infotxt[1]=infotxt[1]+' Data type will be changed to match VAR_DATA_TYPE value in HDF file'
+ IF ~qa_yes THEN INFOTXT_OUTPUT,infotxt ;error for QA output separately
+ verror[0]=1
+ ENDIF
+ ENDIF
+ IF ((test3) OR (test4)) AND (ct NE 0) THEN BEGIN
+ IF (SIZE(nav,/TYPE) NE idltc) AND (idltc NE 0) AND (o3[0] EQ 'H4') AND (verror[1] EQ 0) THEN BEGIN
+ ;Check data type of pre-defined attribute
+ infotxt=STRARR(2)
+ infotxt[0]='2 HDF4 pre-defined attribute '+ncsa[ct]+' data type does not match VAR_DATA_TYPE='+vdtv
+ IF vnx NE '' THEN infotxt[0]=infotxt[0]+' for Dataset '+vnx+'|.' $
+ ELSE infotxt[0]=infotxt[0]+'|.'
+ infotxt[1]=' Data type will be changed to match VAR_DATA_TYPE value in HDF file'
+ INFOTXT_OUTPUT,infotxt
+ IF ncsa[ct] EQ 'valid_range' THEN verror[1]=1 ;to only report 1 valid_range error
+ ENDIF
+ ENDIF
+ IF ((test1) OR (test2)) AND ((test3) OR (test4)) THEN BEGIN
+ ;Compare Variable Attributes against pre-defined attributes
+ IF ct EQ 0 THEN BEGIN ;units check
+ IF (STRTRIM(nav,2) NE STRTRIM(vav,2)) AND (o3[0] EQ 'H4') THEN BEGIN
+ infotxt=STRARR(2)
+ infotxt[0]='2 HDF4 pre-defined attribute units='+nav+' does not match VAR_UNITS='+vav
+ IF vnx NE '' THEN infotxt[0]=infotxt[0]+' for Dataset '+vnx+'|.' $
+ ELSE infotxt[0]=infotxt[0]+'|.'
+ infotxt[1]=' HDF4 units will be changed to match VAR_UNITS value in HDF file'
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ ENDIF ELSE BEGIN
+ IF (nav NE vav) AND (idltc NE 7) AND (idltc NE 0) AND (o3[0] EQ 'H4') THEN BEGIN
+ infotxt=STRARR(2)
+ infotxt[0]='2 HDF4 pre-defined attribute '+ncsa[ct]+' value does not equal '+var_atts[ct]+' value'
+ IF vnx NE '' THEN infotxt[0]=infotxt[0]+' for Dataset '+vnx+'|.' $
+ ELSE infotxt[0]=infotxt[0]+'|.'
+ infotxt[1]=' HDF4 '+ncsa[ct]+' will be changed to match '+var_atts[ct]+' value in HDF file'
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ ENDELSE
+ ENDIF
+ENDIF ELSE BEGIN ;Do checks on remaining pre-defined attributes
+ ncsa_str=['long_name','format','coordsys']
+ ncsa_cal=['scale_factor','scale_factor_err','add_offset','add_offset_err','calibrated_nt']
+ FOR i=0,N_ELEMENTS(ncsa_str)-1 DO BEGIN
+ ni=WHERE(STRLOWCASE(nav) EQ ncsa_str[i],ncnt)
+ IF ncnt NE 0 THEN BEGIN
+ IF qa_yes THEN itxt=' present in' ELSE itxt=' will not be written to'
+ infotxt='0 HDF4 pre-defined attribute '+ncsa_str[i]+itxt+' the HDF file'
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ ENDFOR
+ calfound=0
+ FOR i=0,N_ELEMENTS(ncsa_cal)-1 DO BEGIN
+ ni=WHERE((STRLOWCASE(nav) EQ ncsa_cal[i]) AND (calfound EQ 0),ncnt)
+ IF ncnt NE 0 THEN BEGIN
+ calfound=1
+ infotxt='3 Datasets that have been scaled or had an offset applied are not permitted'
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ ENDFOR
+ENDELSE
+
+END ;Procedure Pre_Defined_Att_Checks
+
+
+
+PRO read_metadata, metafile, inf
+;Procedure to check Metadata contents and return as meta_arr - also:
+;1. Ensures uniform input (making ATTRIBUTE_NAME uppercase, deleting unwanted spaces etc).
+;2. Checks that attribute length falls within allowable limits
+;3. Checks for repeated, missing, new, obsolete attributes
+;4. Update FILE_META_VERSION with TAV and software version (tab_ver)
+
+;Please also note the following:
+;1. Any extra valid Global Attributes will be included (and added to the attr_arr_glob array).
+;2. If redundant attributes are detected they will be omitted from the Metadata written
+; to the HDF file.
+; ----------
+;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
+;
+; History:
+; 20050802: Original IDLCR8HDF Routine - Version 1.0
+; 20061012: Error messages modified; Allow VAR_MONOTONE as a Variable Attribute if it is present in the
+; Metadata; If DATA_TYPE is present in the Metadata then change it to DATA_LEVEL; Allow
+; additional Global Attributes; Puts Variable Attribute names in the same order as
+; attr_arr_data; Common variable definition WIDGET_WIN added - Version 2.0
+; 20080302: VAR_MONOTONE not included in the output Metadata if it is present in the input Metadata
+; and an AVDC style TAV file is read in; if DATA_TYPE is present in the Metadata it is changed
+; to DATA_LEVEL if an AVDC style TAV file is read in, and vice versa if an original style
+; Envisat table.dat file read in; Warnings added if VAR_MONOTONE attribute is omitted, when
+; changing DATA_TYPE to DATA_LEVEL (or vice versa), and if extra Global Attributes are
+; included in the Metadata; Checks for missing or repeated Global Attributes; Global
+; Attributes can now be listed in any order in the Metadata (will be written to the HDF file
+; in a standard order) - Version 3.0
+; 20091203: Stop using DATA_TYPE for EVDC style input (if found change to DATA_LEVEL); If VAR_MONOTONE
+; is present in the Metadata it is removed - Version 3.08
+; 20100205: Add RETURN command after all STOP_WITH_ERROR calls, which allows program to return to the
+; calling program if the reterr argument is included in the idlcr8hdf call - Version 3.09
+; 20101120: Add DATA_STOP_DATE and remove DATA_LEVEL from Global Attributes; Remove VAR_AVG_TYPE,
+; VAR_DIMENSION, and VIS attributes from Variable Attributes; Add call to GEOMS_RULE_CHANGES
+; to account for changes in Metadata setup between v3.0 and v4.0; Remove option to have a
+; filename as a value for free text attributes; Increase checks between DATA_VARIABLES and
+; VAR_NAME values - Version 4.0b0
+; 20111220: Call additional GEOMS_RULE_CHANGES; Perform checks on HDF4 pre-defined attributes
+; - Version 4.0b7
+; 20150127: Stop removing leading and trailing spaces from variable attribute values (when there is
+; only 1 value) to allow for possibility of VAR_FILL_VALUEs being an empty string
+; - Version 4.0b25
+; 20150811: Add checks for invalid FILE_META_VERSION entries - Version 4.0b29
+; 20150924: Change to v4.0b25 so that leading and trailing spaces are not removed from VAR_FILL_VALUE
+; entries only (see attr_arr_str) - Version 4.0b30
+; 20151012: Improve/fix checks for invalid whitespace in Metadata labels or values and add log output when
+; changes are made. Checks previously done in CHECK_METADATA moved to this routine
+; - Version 4.0b31
+; 20151021: Do not do checks on invalid whitespace for attributes that have numeric values - Version 4.0b32
+; 20151109: Add log message when adding/checking for missing mandatory variable attributes - Version 4.0b34
+; 20151116: Fix bug when doing whitespace checks and there is no metadata entry - Version 4.0b35
+; 20180528: Fix bug when adding non-standard global attributes to meta_arr - Version 4.0b47
+; 20191205: Do checks for sub-value separators (. or ; or _) being at the start or end of attribute
+; values, and for consecutive sub-value separators (e.g. ;;). This can occur as the STRSPLIT
+; command can remove these while testing sub-values, so they are not detected as part of
+; any tests - Version 4.0b53
+; 20200311: Do not include '.' separator when doing checks for sub-value separators at the start or end of
+; the attribute values e.g. allow DO_NAME=Boyd;Ian S. - Version 4.0b54
+; 20201020: Generate message for invalid metadata attributes, previously removed them 'quietly'
+; - Version 4.0b56
+; 20201026: Fix bug from 4.0b56 which caused an error message if _FillValue was present in the GEOMS file
+; - Version 4.0b57
+;
+; Inputs: metafile - Either, the filename of the input file containing the Metadata or, if program
+; input is via string array and Structure, a string array containing the Global
+; and Variable Attributes.
+; inf - flag, where 0=Metafile is an array; 1=Metafile is a file.
+;
+; Output: meta_arr - a string array containing the Global and Variable Attributes, which have been
+; checked and uniformally formatted.
+;
+; Called by: IDLCR8HDF
+;
+; Subroutines Called: STOP_WITH_ERROR (if error state detected);
+; INFOTXT_OUTPUT, GEOMS_RULE_CHANGES
+; Possible Conditions for STOP_WITH_ERROR call (plus [line number] where called):
+; 1. Metadata value contains too many characters
+; 2. No or invalid DATA_VARIABLES and VAR_NAME attributes present in the metadata
+; 3. DATA_VARIABLES or VAR_NAME value appears more than once
+; 4. No valid Metadata in command line parameter file or arrays
+; 5. VAR_NAME does not match corresponding DATA_VARIABLES value
+; 6. Number of DATA_VARIABLES values does not match the number of VAR_NAME occurrences
+; 7. A required Global Attribute Label is not present in the Metadata
+; 8. A required Global Attribute Label is repeated in the Metadata
+; 9. Number of Attribute occurrences does not match the number of datasets
+; 10. DATA_QUALITY is not present but is a mandatory attribute when DATA_TEMPLATE is reported
+;
+; Information Conditions (when the program is able to make changes):
+; 1. Invalid Attribute Label ignored
+; 2. Missing Global Attribute added
+; 3. DATA_VARIABLES attribute added to metadata based on VAR_NAME values
+; 4. Values for DATA_VARIABLES appended to the metadata based on VAR_NAME values
+; 5. VAR_NAME value appended to label based on DATA_VARIABLES value
+; 6. New Global Attribute added
+
+COMMON TABLEDATA
+COMMON METADATA
+COMMON DATA
+COMMON WIDGET_WIN
+
+attr_arr_glob=['PI_NAME','PI_AFFILIATION','PI_ADDRESS','PI_EMAIL',$
+ 'DO_NAME','DO_AFFILIATION','DO_ADDRESS','DO_EMAIL',$
+ 'DS_NAME','DS_AFFILIATION','DS_ADDRESS','DS_EMAIL',$
+ 'DATA_DESCRIPTION','DATA_DISCIPLINE','DATA_GROUP','DATA_LOCATION',$
+ 'DATA_SOURCE','DATA_VARIABLES','DATA_START_DATE','DATA_STOP_DATE',$
+ 'DATA_FILE_VERSION','DATA_MODIFICATIONS','DATA_CAVEATS','DATA_RULES_OF_USE',$
+ 'DATA_ACKNOWLEDGEMENT','DATA_QUALITY','DATA_TEMPLATE','DATA_PROCESSOR',$
+ 'FILE_NAME','FILE_GENERATION_DATE','FILE_ACCESS','FILE_PROJECT_ID','FILE_DOI', $
+ 'FILE_ASSOCIATION','FILE_META_VERSION']
+
+attr_arr_glob_opt=['DATA_DESCRIPTION','DATA_MODIFICATIONS','DATA_CAVEATS','DATA_RULES_OF_USE',$
+ 'DATA_ACKNOWLEDGEMENT','DATA_QUALITY','DATA_TEMPLATE','DATA_PROCESSOR',$
+ 'FILE_PROJECT_ID','FILE_ASSOCIATION']
+
+attr_arr_glob_ins=['DATA_START_DATE','DATA_STOP_DATE','FILE_GENERATION_DATE', $
+ 'FILE_DOI','FILE_META_VERSION']
+
+attr_arr_glob_prov=['DATA_TEMPLATE','DATA_QUALITY']
+
+attr_arr_data=['VAR_NAME','VAR_DESCRIPTION','VAR_NOTES','VAR_SIZE','VAR_DEPEND',$
+ 'VAR_DATA_TYPE','VAR_UNITS','VAR_SI_CONVERSION','VAR_VALID_MIN',$
+ 'VAR_VALID_MAX','VAR_FILL_VALUE']
+
+attr_arr_data_opt=['VAR_NOTES']
+
+;attr_arr_str=['VAR_FILL_VALUE','_fillvalue'] ;do not take STRTRIM of these values
+
+;Attributes that have user defined values
+attr_free=['DATA_DESCRIPTION','DATA_MODIFICATIONS','DATA_CAVEATS',$
+ 'DATA_RULES_OF_USE','DATA_ACKNOWLEDGEMENT','DATA_QUALITY', $
+ 'DATA_PROCESSOR','FILE_PROJECT_ID','FILE_ASSOCIATION', $
+ 'VAR_DESCRIPTION','VAR_NOTES']
+
+;Atrributes that have numeric values (except for string datatype)
+num_atts=['VAR_VALID_MIN','VAR_VALID_MAX','VAR_FILL_VALUE','_fillvalue','valid_range']
+
+;To check for pre-defined attributes
+ncsa=['long_name','units','format','coordsys','valid_range','_fillvalue','scale_factor', $
+ 'scale_factor_err','add_offset','add_offset_err','calibrated_nt']
+
+;Possible sub-value separators
+sv_sep=['.','_',';']
+
+nchar=65535L ;limit on number of characters
+
+;possible error messages for this procedure
+errtxt=STRARR(9)
+proname='Read_Metadata procedure: '
+errtxt[0]=' value contains too many characters (maximum permitted: '+STRTRIM(nchar,2)+')'
+errtxt[1]='No or invalid DATA_VARIABLES and VAR_NAME attributes present in the metadata'
+errtxt[2]=' value appears more than once: '
+errtxt[3]='No valid Metadata in input'
+errtxt[4]='VAR_NAME does not match corresponding DATA_VARIABLES value: '
+errtxt[5]='Number of DATA_VARIABLES values does not match the number of VAR_NAME occurrences '
+errtxt[6]='Global Attribute Label not present in the Metadata: ' ;now INFOTXT
+errtxt[7]='Global Attribute Label repeated in the Metadata: '
+errtxt[8]=' occurrences does not match the number of datasets (should be '
+
+dum='' & nrline=0L & dvfound=-1L
+nd=N_ELEMENTS(attr_arr_data)
+ng=N_ELEMENTS(attr_arr_glob)
+nna=N_ELEMENTS(num_atts)
+dsei=WHERE(attr_arr_glob EQ 'DS_EMAIL') ;any new global attributes should be after this attribute
+vni=WHERE(attr_arr_data EQ 'VAR_NAME') ;used in VAR_NAME checks
+ndv=0 ;will reset to the number of datasets
+
+IF inf EQ 1 THEN BEGIN ;count number of lines in the file
+ OPENR,lu,metafile,/GET_LUN
+ WHILE NOT EOF(lu) DO BEGIN
+ READF,lu,dum & nrline=nrline+1L
+ ENDWHILE
+ FREE_LUN,lu
+ mh=STRARR(nrline) & mv_lng=LON64ARR(nrline) & mv_dbl=DBLARR(nrline)
+ OPENR,lu,metafile,/GET_LUN
+ READF,lu,mh
+ FREE_LUN,lu
+ENDIF ELSE mh=metafile ;metadata is already in an array
+mhhold=STRTRIM(mh,1) ;create temporary array and remove leading blanks from all the entries
+mhgood=INTARR(N_ELEMENTS(mh))+1 ;will be assigned zero for unwanted metadata lines
+
+;Check for Redundant Attribute Labels
+GEOMS_RULE_CHANGES,0,mhhold,mhgood
+
+;test1 - check first character is A-Z or a-z; test2 - check for '=' and not a redundant label
+test1=(((BYTE(STRMID(mhhold,0,1)) GE 65) AND (BYTE(STRMID(mhhold,0,1)) LE 90)) OR $
+ ((BYTE(STRMID(mhhold,0,1)) GE 97) AND (BYTE(STRMID(mhhold,0,1)) LE 122)) OR $
+ (STRMID(STRUPCASE(mhhold),0,10) EQ '_FILLVALUE'))
+test2=(STRPOS(mhhold,'=') GE 1) AND (mhgood EQ 1)
+
+nri=WHERE((test1) AND (test2),nrline)
+IF nrline EQ 0 THEN BEGIN
+ STOP_WITH_ERROR,o3[3]+proname,errtxt[3],lu & RETURN
+ENDIF
+
+;Generate an error for invalid metadata attributes
+bnri=WHERE(~test1,bnrline)
+IF bnrline NE 0 THEN BEGIN
+ FOR i=0L,bnrline-1L DO BEGIN
+ infotxt='2 Invalid Metadata Attribute: '+mhhold[bnri[i]]+'|'
+ infotxt=infotxt+' removed from the metadata saved to the output file'
+ INFOTXT_OUTPUT,infotxt
+ ENDFOR
+ENDIF
+
+;Set holding arrays for metadata
+mh=mh[nri] & mv_lng=mv_lng[nri] & mv_dbl=mv_dbl[nri]
+lu=-1
+m_v0=STRARR(nrline) & m_v1=m_v0 ;to hold metadata labels and values
+m_v2=INTARR(nrline) ;global/variable attribute switch
+
+;Separate out Metadata entries
+FOR i=0L,nrline-1L DO BEGIN
+ ;separate out the Meta entry into two components
+ eqpos=STRPOS(mh[i],'=')
+ m_v0[i]=STRMID(mh[i],0,eqpos)
+ IF eqpos EQ STRLEN(mh[i])-1 THEN m_v1[i]='' $ ;ie. the last character is the '='
+ ELSE m_v1[i]=STRMID(mh[i],eqpos+1)
+ENDFOR
+
+;Check for any attribute label that is not uppercase (and not an HDF pre-defined label) or has leading or trailing spaces
+ui=WHERE(m_v0 NE STRUPCASE(m_v0),ucnt)
+lti=WHERE(m_v0 NE STRTRIM(m_v0,2),ltcnt)
+
+IF ltcnt NE 0L THEN BEGIN ;some attribute labels have leading or trailing spaces
+ m_v0=STRTRIM(m_v0,2)
+ IF qa_yes THEN BEGIN ;only report if doing QA, otherwise fix quietly
+ infotxt='2 Leading or trailing spaces in Metadata Attribute Labels not permitted'
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ENDIF
+
+IF ucnt NE 0L THEN BEGIN ;some attribute labels are not fully uppercase
+ writeonce=0 ;write out information text once only
+ FOR i=0L,ucnt-1L DO BEGIN
+ ;check for HDF predefined labels which should be lowercase
+ pi=WHERE(STRLOWCASE(m_v0[ui[i]]) EQ ncsa,pcnt)
+ IF pcnt EQ 0 THEN BEGIN ;label expected to be in uppercase
+ m_v0[ui[i]]=STRUPCASE(m_v0[ui[i]])
+ IF writeonce EQ 0 THEN BEGIN ;write INFORMATION text to log
+ writeonce=1
+ IF qa_yes THEN itxt='must be' ELSE itxt='made'
+ infotxt='2 Metadata Attribute Labels '+itxt+' UPPERCASE'
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ ENDIF
+ ENDFOR
+ENDIF
+
+;Check for and remove leading and trailing spaces in Metadata (sub-)values (except for free-text attributes
+;and VAR_FILL_VALUEs). Also for non-free-text attributes check if the first and last characters are
+;sub-value separators, or if there are consecutive sub-value separators in the attribute value.
+;Also check for values with excessive length.
+IF qa_yes THEN wstxt='are not permitted' ELSE wstxt='removed' ;used when reporting white space in sub-values
+
+FOR i=0L,nrline-1L DO BEGIN
+ ;si=WHERE(STRUPCASE(m_v0[i]) EQ STRUPCASE(num_atts),scnt)
+ ;IF scnt EQ 0 THEN si=WHERE(m_v0[i] EQ attr_free,scnt)
+ si=WHERE(m_v0[i] EQ attr_free,scnt)
+ IF scnt EQ 0 THEN BEGIN ;not a free text attribute
+ nmi=WHERE(STRUPCASE(m_v0[i]) EQ STRUPCASE(num_atts),nmcnt) ;is it an attribute that should have a numeric value
+ res=STRSPLIT(m_v1[i],';',/Extract,COUNT=svcnt)
+ spi=WHERE(STRTRIM(res,2) NE res,spcnt)
+ IF spcnt NE 0 THEN BEGIN
+ IF (svcnt EQ 1) AND (qa_yes) THEN BEGIN
+ IF (nmcnt NE 0) AND (STRTRIM(res[0],2) NE '') THEN infotxt='' $ value should be a number not a string so do not check for spaces
+ ELSE IF res[0] EQ ' ' THEN infotxt='' $ there could be a single space in the HDF file if there is no entry
+ ELSE infotxt='2 Leading or trailing spaces in '+m_v0[i]+'='+m_v1[i]+' value '+wstxt
+ ENDIF ELSE IF svcnt GT 1 THEN BEGIN
+ infotxt='2 Leading or trailing spaces between '+m_v0[i]+' sub-values '+wstxt
+ ENDIF ELSE infotxt='' ;Quietly fix if only single value and not doing QA
+ IF infotxt NE '' THEN INFOTXT_OUTPUT,infotxt
+ res=STRTRIM(res,2)
+ m_v1[i]=res[0]
+ IF svcnt GT 1 THEN FOR j=1,svcnt-1 DO m_v1[i]=m_v1[i]+';'+res[j]
+ ENDIF
+
+ IF nmcnt EQ 0 THEN BEGIN
+ ;Check that first and last characters of the metadata value aren't sub-value separators
+ ;and that there are no consecutive sub-value separators
+ mv1len=STRLEN(m_v1[i]) ;string length of attribute value
+ mv1start=STRMID(m_v1[i],0,1) ;first character
+ mv1end=STRMID(m_v1[i],mv1len-1L,1) ;last character
+ FOR k=0,1 DO BEGIN
+ writeonce=1B ;so that error message is only written once for each attribute
+ IF k EQ 0 THEN $
+ infotxt='3 Sub-value separators at the start or end of the attribute value are not permitted' $
+ ELSE infotxt='3 Consecutive sub-value separators are not permitted'
+ FOR j=0,N_ELEMENTS(sv_sep)-1 DO BEGIN
+ spchk=sv_sep[j]+sv_sep[j]
+ ;Note: in some cases a '.' may be present at the end of a sub-value e.g. DO_NAME=Boyd;Ian S.
+ IF (k EQ 0) AND (sv_sep[j] NE '.') THEN test=(mv1start EQ sv_sep[j]) OR (mv1end EQ sv_sep[j]) $
+ ELSE IF k EQ 1 THEN test=STRPOS(m_v1[i],spchk) NE -1 $
+ ELSE test=0B
+ IF (test) AND (writeonce) THEN BEGIN
+ IF mv1len GT 50 THEN infotxt=infotxt+' in '+m_v0[i]+' attribute' $
+ ELSE infotxt=infotxt+': '+m_v0[i]+'='+m_v1[i]
+ INFOTXT_OUTPUT,infotxt
+ writeonce=0B
+ ENDIF
+ ENDFOR
+ ENDFOR
+ ENDIF
+ ENDIF
+ ;check for string length of attribute value
+ IF STRLEN(m_v1[i]) GT nchar THEN BEGIN
+ infotxt='3 '+m_v0[i]+errtxt[0]
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ENDFOR
+
+FOR i=0L,nrline-1L DO BEGIN
+ ;Do check for DATA_VARIABLES and separate out the values
+ IF m_v0[i] EQ 'DATA_VARIABLES' THEN BEGIN
+ dv=STRSPLIT(m_v1[i],' ;',/Extract,COUNT=ndv) & dvfound=i
+ ENDIF
+
+ ;Check for FILE_META_VERSION and add value derived from READ_TABLEFILE if not doing QA
+ IF m_v0[i] EQ 'FILE_META_VERSION' THEN BEGIN
+ IF (~qa_yes) AND (m_v1[i] NE tab_ver) THEN BEGIN
+ m_v1[i]=tab_ver
+ infotxt='0 FILE_META_VERSION updated with corrected TAV file version and tool name'
+ INFOTXT_OUTPUT,infotxt
+ ENDIF ELSE IF qa_yes THEN BEGIN
+ ;Check Revision number is less than or equal to the TAV revision number, must be '04''R''xxx'
+ revno=STRSPLIT(m_v1[i],';',/EXTRACT,count=fmvcnt) & revno=STRTRIM(revno,2)
+ tavno=STRSPLIT(tab_ver,';',/EXTRACT) & tavno=STRTRIM(tavno,2)
+ IF fmvcnt NE 2 THEN BEGIN
+ infotxt='2 FILE_META_VERSION values must include the TAV file version and the HDF file '
+ infotxt=infotxt+'creation tool name e.g. FILE_META_VERSION='+tavno[0]+';CUSTOM'
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ revok=0 ;default is to report a TAV file version error unless revok eq 3
+ IF STRLEN(revno[0]) EQ STRLEN(tavno[0]) THEN BEGIN
+ vrev=STRMID(revno[0],0,2) & vtav=STRMID(tavno[0],0,2)
+ rrev=STRMID(revno[0],2,1) & rtav=STRMID(tavno[0],2,1)
+ srev=STRMID(revno[0],3) & stav=STRMID(tavno[0],3)
+ IF IS_A_NUMBER_HDF(vrev) THEN BEGIN
+ IF (FIX(vrev) GE 4) AND (FIX(vrev) LE FIX(vtav)) THEN revok++ ;TAV version OK
+ ENDIF
+ IF rrev EQ rtav THEN revok++ ;'R' is present as the third character
+ IF (IS_A_NUMBER_HDF(srev)) AND (revok eq 2) THEN BEGIN
+ IF (FIX(srev) LE FIX(stav)) OR ((FIX(srev) GT FIX(stav)) AND (FIX(vrev) LT FIX(vtav))) THEN revok++ ;TAV sub-version OK
+ ENDIF
+ ENDIF
+ IF revok NE 3 THEN BEGIN
+ infotxt='2 FILE_META_VERSION sub-value '+revno[0]+' is not valid. It must be the TAV file version e.g. '+tavno[0]
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ ENDIF
+ ENDIF
+
+ ;Check whether a Global Attribute (1), GEOMS Variable Attribute (2), or pre-defined Variable Attribute (3)
+ gi=WHERE(m_v0[i] EQ attr_arr_glob,gcnt)
+ vi=WHERE(m_v0[i] EQ attr_arr_data,vcnt)
+ pi=WHERE(STRLOWCASE(m_v0[i]) EQ ncsa,pcnt)
+ IF gcnt NE 0 THEN m_v2[i]=1 $
+ ELSE IF vcnt NE 0 THEN m_v2[i]=2 $
+ ELSE IF pcnt NE 0 THEN m_v2[i]=3 $
+ ELSE BEGIN ;check for new Global Attribute
+ IF i GT 0 THEN test1=m_v2[i-1] EQ 1 ELSE test1=1 ;i.e. the previous attribute was also a global attribute
+ test2=(STRMID(m_v0[i],0,3) NE 'VAR') AND (STRMID(m_v0[i],0,3) NE 'VIS') ;does not start with 'VAR' or 'VIS'
+ test3=STRPOS(m_v0[i],' ') EQ -1 ;No spaces in the label
+ ;Check for possible mispellings of an actual attribute by removing '_' and comparing
+ gv_chk=STRUPCASE(STRJOIN(STRSPLIT(m_v0[i],' _',/EXTRACT),/SINGLE))
+ test4=1
+ FOR j=0,ng-1 DO BEGIN
+ ga_j=STRUPCASE(STRJOIN(STRSPLIT(attr_arr_glob[j],' _',/EXTRACT),/SINGLE))
+ IF gv_chk EQ ga_j THEN test4=0 ;Match with actual Global Attribute so do not keep
+ ENDFOR
+ IF (test1) AND (test2) AND (test3) AND (test4) THEN m_v2[i]=1 ;it is considered a new global attribute
+ ENDELSE
+ENDFOR
+
+;Check for any invalid attribute labels and remove
+bi=WHERE(m_v2 EQ 0,bcnt)
+IF bcnt NE 0 THEN BEGIN
+ FOR i=0,bcnt-1 DO BEGIN
+ infotxt='2 Unknown Attribute Label '+m_v0[bi[i]]+'| ignored'
+ INFOTXT_OUTPUT, infotxt
+ ENDFOR
+ENDIF
+;resize arrays, or stop the program if no valid data
+gi=WHERE(m_v2 NE 0,nrline)
+IF nrline NE 0 THEN BEGIN
+ m_v0=m_v0[gi] & m_v1=m_v1[gi] & m_v2=m_v2[gi]
+ mv_lng=mv_lng[gi] & mv_dbl=mv_dbl[gi]
+ENDIF ELSE BEGIN
+ STOP_WITH_ERROR,o3[3]+proname,errtxt[3],lu & RETURN
+ENDELSE
+
+;Check to see that Global and Variable Attributes are present
+gi=WHERE(m_v2 EQ 1,gcnt)
+vi=WHERE(m_v2 EQ 2,vcnt)
+pi=WHERE(m_v2 EQ 3,pcnt) ;to do pre-defined attribute checks if necessary
+IF (gcnt EQ 0) OR (vcnt EQ 0) THEN BEGIN
+ STOP_WITH_ERROR,o3[3]+proname,errtxt[3],lu & RETURN
+ENDIF
+
+;Do QA on pre-defined attributes
+IF pcnt NE 0 THEN PRE_DEFINED_ATT_CHECKS,-1,m_v0[pi]
+
+;Do DATA_VARIABLES checks
+IF ndv EQ 0 THEN BEGIN
+ ;If no DATA_VARIABLES values found then try to determine values from VAR_NAME values
+ dvi=WHERE((m_v0 EQ 'VAR_NAME') AND (m_v1 NE ''),ndv)
+ IF ndv NE 0 THEN BEGIN
+ dv=m_v1[dvi]
+ ;Create or append to DATA_VARIABLES attribute
+ IF dvfound EQ -1L THEN BEGIN ;increase array sizes by 1 to accommodate DATA_VARIABLES
+ dvfound=nrline & nrline=nrline+1L
+ m_v0=[m_v0,'DATA_VARIABLES'] & m_v1=[m_v1,''] & m_v2=[m_v2,1]
+ mv_lng=[mv_lng,0LL] & mv_dbl=[mv_dbl,0.D]
+ infotxt='2 Missing DATA_VARIABLES Global Attribute| added to metadata based on VAR_NAME values'
+ ENDIF ELSE BEGIN
+ infotxt='2 Missing DATA_VARIABLES attribute values| appended to the metadata based on VAR_NAME values'
+ ENDELSE
+ INFOTXT_OUTPUT, infotxt
+ ENDIF ELSE BEGIN
+ STOP_WITH_ERROR,o3[3]+proname,errtxt[1],lu & RETURN
+ ;No or invalid DATA_VARIABLES and VAR_NAME attributes present in the metadata so cannot proceed
+ ENDELSE
+ENDIF
+;Check for repeated DATA_VARIABLES
+FOR i=0,ndv-1 DO BEGIN
+ ri=WHERE(STRUPCASE(dv[i]) EQ STRUPCASE(dv),rcnt)
+ IF (rcnt GT 1) AND (dv[i] NE '') THEN BEGIN
+ STOP_WITH_ERROR,o3[3]+proname,'DATA_VARIABLES'+errtxt[2]+dv[i],lu & RETURN
+ ENDIF
+ENDFOR
+
+;Do VAR_NAME checks
+vi=WHERE(m_v0 EQ attr_arr_data[vni[0]],vcnt) ;identify VAR_NAME indices
+IF vcnt NE 0 THEN BEGIN
+ ;Check if VAR_NAME values are present, if not add value from DATA_VARIABLES (if number of DATA_VARIABLES values match vcnt)
+ IF ndv EQ vcnt THEN BEGIN
+ FOR i=0,vcnt-1 DO BEGIN
+ IF (m_v1[vi[i]] NE '') AND (m_v1[vi[i]] NE dv[i]) THEN BEGIN
+ STOP_WITH_ERROR,o3[3]+proname,errtxt[4]+'('+m_v1[vi[i]]+'/'+dv[i]+')',lu & RETURN
+ ENDIF ELSE IF m_v1[vi[i]] EQ '' THEN BEGIN
+ m_v1[vi[i]]=dv[i]
+ infotxt='2 Missing VAR_NAME value for '+m_v0[vi[i]]+'|. '+dv[i]+'| appended to label based on DATA_VARIABLES value
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ ENDFOR
+ ENDIF ELSE BEGIN ;number of DATA_VARIABLES values does not match the number of VAR_NAME occurrences
+ STOP_WITH_ERROR,o3[3]+proname,errtxt[5]+'('+STRTRIM(ndv,2)+'/'+STRTRIM(vcnt,2)+')',lu & RETURN
+ ENDELSE
+ ;Check for repeated VAR_NAMES
+ FOR i=0,vcnt-1 DO BEGIN
+ ri=WHERE(STRUPCASE(m_v1[vi[i]]) EQ STRUPCASE(m_v1[vi]),rcnt)
+ IF (rcnt GT 1) AND (m_v1[vi[i]] NE '') THEN BEGIN
+ STOP_WITH_ERROR,o3[3]+proname,'VAR_NAME'+errtxt[2]+m_v1[vi[i]],lu & RETURN
+ ENDIF
+ ENDFOR
+ENDIF ELSE BEGIN ;No VAR_NAME labels so create from DATA_VARIABLES
+ FOR i=0,ndv-1 DO BEGIN
+ m_v0=[m_v0,'VAR_NAME'] & m_v1=[m_v1,dv[i]] & m_v2=[m_v2,2]
+ mv_lng=[mv_lng,0LL] & mv_dbl=[mv_dbl,0.D]
+ IF i EQ 0 THEN vi=[nrline] ELSE vi=[vi,nrline+1L]
+ ENDFOR
+ infotxt='1 Missing VAR_NAME attribute labels and values| added to metadata based on DATA_VARIABLES values'
+ INFOTXT_OUTPUT,infotxt
+ENDELSE
+
+;Check for obsolete FTIR, UVVIS.DOAS, LIDAR, and MWR DATA_SOURCE_01 values and modify of possible
+dsi=WHERE(m_v0 EQ 'DATA_SOURCE',dsicnt)
+vni=WHERE(m_v0 EQ 'VAR_NAME',vncnt)
+;extract VAR_NAME values
+dv=STRARR(vncnt) & data_source=''
+FOR i=0,vncnt-1 DO dv[i]=m_v1[vni[i]]
+IF dsicnt EQ 1 THEN BEGIN
+ IF m_v1[dsi[0]] NE '' THEN BEGIN
+ data_source=m_v1[dsi[0]] & dschange=data_source
+ GEOMS_RULE_CHANGES,6,vncnt,dv,data_source
+ IF data_source NE dschange THEN m_v1[dsi[0]]=data_source
+ ENDIF
+ENDIF
+
+;Check for missing Global Attributes which have values which can be calculated by the program
+FOR i=0,N_ELEMENTS(attr_arr_glob_ins)-1 DO BEGIN
+ gaii=WHERE(m_v0 EQ attr_arr_glob_ins[i],gaicnt)
+ IF gaicnt EQ 0 THEN BEGIN ;missing mandatory global attribute so add to the array
+ IF attr_arr_glob_ins[i] EQ 'FILE_META_VERSION' THEN itxt=tab_ver ELSE itxt=''
+ nrline=nrline+1L
+ m_v0=[m_v0,attr_arr_glob_ins[i]] & m_v1=[m_v1,itxt] & m_v2=[m_v2,1]
+ mv_lng=[mv_lng,0LL] & mv_dbl=[mv_dbl,0.D]
+ infotxt='2 Missing mandatory Global Attribute '+attr_arr_glob_ins[i]+'| added'
+ INFOTXT_OUTPUT, infotxt
+ ENDIF
+ENDFOR
+;Do DATA_TEMPLATE/DATA_QUALITY checks
+ngp=N_ELEMENTS(attr_arr_glob_prov)
+ga_chk=STRARR(ngp) ;initialize DATA_TEMPLATE/QUALITY values
+GEOMS_RULE_CHANGES,9,m_v0,m_v1,ga_chk
+;Check to see if Global Attributes have to be added
+FOR i=0,ngp-1 DO BEGIN
+ IF ga_chk[i] NE '' THEN BEGIN
+ m_v0=[m_v0,attr_arr_glob_prov[i]]
+ IF i EQ 0 THEN m_v1=[m_v1,ga_chk[i]] ELSE m_v1=[m_v1,'']
+ m_v2=[m_v2,1] & nrline=nrline+1L
+ mv_lng=[mv_lng,0LL] & mv_dbl=[mv_dbl,0.D]
+ IF i EQ 0 THEN itxt='='+ga_chk[i] ELSE itxt=''
+ infotxt='2 Missing Global Attribute '+attr_arr_glob_prov[i]+itxt+'| added'
+ INFOTXT_OUTPUT, infotxt
+ ENDIF
+ENDFOR
+
+;Calculate expected size of meta_arr array
+gi=WHERE(m_v2 EQ 1,gcnt)
+IF gcnt GT ng THEN ngv=gcnt ELSE ngv=ng
+n_arr=ngv+(nd*ndv)
+meta_arr=STRARR(n_arr) & mlh=LON64ARR(n_arr) & mdh=DBLARR(n_arr)
+;mlh and mdh are holding arrays to hold numeric values in the metadata (will be renamed mv_lng and mv_dbl)
+
+;Check for missing or repeated standard Global Attributes
+i=0
+WHILE i LE ng-1 DO BEGIN
+ gai=WHERE(m_v0[gi] EQ attr_arr_glob[i],gacnt)
+ IF gacnt EQ 0 THEN BEGIN ;Check to see if Global Attribute Optional or Mandatory
+ gaoi=WHERE(attr_arr_glob[i] EQ attr_arr_glob_opt, gaocnt)
+ IF gaocnt EQ 0 THEN BEGIN ;it is mandatory, therefore missing
+ infotxt='3 Global Attribute Label not present in the Metadata: '+attr_arr_glob[i]
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ ENDIF ELSE IF gacnt GT 1 THEN BEGIN ;Global Attribute Repeated
+ STOP_WITH_ERROR,o3[3]+proname,errtxt[7]+attr_arr_glob[i],lu & RETURN
+ ENDIF
+ IF gacnt EQ 0 THEN BEGIN ;remove attribute from the attr_arr_glob array
+ attr_arr_glob=[attr_arr_glob[0:i-1],attr_arr_glob[i+1:ng-1]]
+ ng=ng-1
+ ENDIF ELSE i=i+1
+ENDWHILE
+
+;Check for all Standard Global Attributes, add any new ones, and put in expected order
+xg=0 & mi=0
+FOR i=0,gcnt-1 DO BEGIN
+ gai=WHERE(m_v0[gi[i]] EQ attr_arr_glob,gacnt)
+ CASE 1 OF
+ gacnt EQ 0: BEGIN ;New Global Attribute
+ IF i EQ 0 THEN BEGIN
+ attr_arr_glob=[m_v0[gi[i]],attr_arr_glob]
+ ENDIF ELSE IF i EQ ng THEN BEGIN
+ attr_arr_glob=[attr_arr_glob,m_v0[gi[i]]]
+ ENDIF ELSE BEGIN
+ attr_arr_glob=[attr_arr_glob[0:mi],m_v0[gi[i]],attr_arr_glob[mi+1:N_ELEMENTS(attr_arr_glob)-1]]
+ mi=mi+1
+ ENDELSE
+ ;IF (i GE ng-(1+xg)) OR (i LE dsei) THEN BEGIN
+ ; attr_arr_glob=[attr_arr_glob,m_v0[gi[i]]] & mi=ng
+ ; IF i LE dsei THEN xg=xg+1
+ ;ENDIF ELSE BEGIN
+ ; attr_arr_glob=[attr_arr_glob[0:mi],m_v0[gi[i]],attr_arr_glob[mi+1:ng-1]]
+ ; mi=mi+1
+ ;ENDELSE
+ ng=ng+1
+ infotxt='1 Non-standard Global Attribute '+m_v0[gi[i]]+'| added'
+ INFOTXT_OUTPUT, infotxt
+ END
+ ELSE: mi=i ;gai[0]
+ ENDCASE
+ENDFOR
+FOR i=0,gcnt-1 DO BEGIN
+ gai=WHERE(m_v0[gi[i]] EQ attr_arr_glob,gacnt)
+ meta_arr[gai[0]]=m_v0[gi[i]]+'='+m_v1[gi[i]]
+ENDFOR
+
+;Check for repeated new Global Attributes
+FOR i=0,ng-1 DO BEGIN
+ gai=WHERE(m_v0[gi] EQ attr_arr_glob[i],gacnt)
+ IF gacnt GT 1 THEN BEGIN ;Global Attribute Repeated
+ STOP_WITH_ERROR,o3[3]+proname,errtxt[7]+attr_arr_glob[i],lu & RETURN
+ ENDIF
+ENDFOR
+
+;Do checks for all the variable attributes and put in expected order
+vni=-1
+FOR i=0,nd-1 DO BEGIN
+ si=ng+i ;start index of meta_arr for variable attributes
+ di=WHERE(m_v0 EQ attr_arr_data[i],dcnt)
+ IF attr_arr_data[i] EQ 'VAR_NAME' THEN vni=di ;to enable checks on optional variable attributes
+ IF dcnt EQ ndv THEN BEGIN ;i.e. correct number of attribute labels found so add to meta_arr in correct order
+ FOR j=0,ndv-1 DO BEGIN
+ meta_arr[si+(nd*j)]=m_v0[di[j]]+'='+m_v1[di[j]]
+ mlh[si+(nd*j)]=mv_lng[di[j]] & mdh[si+(nd*j)]=mv_dbl[di[j]]
+ ENDFOR
+ ENDIF ELSE IF dcnt EQ 0 THEN BEGIN ;attribute not present - so add with no label
+ oi=WHERE(attr_arr_data[i] EQ attr_arr_data_opt,ocnt) ;check whether missing attribute is optional
+ IF ocnt EQ 0 THEN BEGIN
+ FOR j=0,ndv-1 DO meta_arr[si+(nd*j)]=attr_arr_data[i]+'='
+ IF qa_yes THEN itxt=' must be present in the metadata' ELSE itxt=' added to the metadata with no value'
+ infotxt='2 Mandatory variable attribute '+attr_arr_data[i]+itxt
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ ENDIF ELSE BEGIN ;incorrect number of attributes - report with error unless attribute is optional
+ oi=WHERE(attr_arr_data[i] EQ attr_arr_data_opt,ocnt)
+ IF (ocnt NE 0) AND (vni[0] GE 0) AND (di[0]-vni[0] GT 0) THEN BEGIN ;match VAR_NOTES attributes to the corresponding VAR_NAME index value
+ FOR j=0,dcnt-1 DO BEGIN
+ vnih=di[j]-vni[0] & jh=0
+ FOR k=0,N_ELEMENTS(vni)-1 DO BEGIN
+ dvn=di[j]-vni[k]
+ IF (dvn LT vnih) AND (dvn GT 0) THEN BEGIN
+ vnih=di[j]-vni[k] & jh=k
+ ENDIF
+ ENDFOR
+ meta_arr[si+(nd*jh)]=m_v0[di[j]]+'='+m_v1[di[j]]
+ mlh[si+(nd*jh)]=mv_lng[di[j]] & mdh[si+(nd*jh)]=mv_dbl[di[j]]
+ ENDFOR
+ ENDIF ELSE BEGIN ;Not optional or too much out of order, therefore report as an error
+ STOP_WITH_ERROR,o3[3]+proname,'Number of '+attr_arr_data[i]+errtxt[8]+STRTRIM(ndv,2)+')',lu & RETURN
+ ENDELSE
+ ENDELSE
+ENDFOR
+
+;remove any empty meta_arr cells (can happen due to optional attributes)
+;and save numeric metadata values in correct order
+gi=WHERE(STRTRIM(meta_arr,2) NE '')
+meta_arr=meta_arr[gi] & mv_lng=mlh[gi] & mv_dbl=mdh[gi]
+
+;Call GEOMS_RULE_CHANGES to check for obsolete Dataset names and update if possible
+;Identify VAR_NAME and VAR_UNITS indices
+vni=WHERE(STRMID(meta_arr,0,8) EQ 'VAR_NAME',vncnt)
+vui=WHERE(STRMID(meta_arr,0,9) EQ 'VAR_UNITS',vucnt)
+vnchange=STRARR(vncnt) ;array to hold original VAR_NAME values if they have been changed
+;extract VAR_NAME values
+dv=STRARR(vncnt)
+FOR i=0,vncnt-1 DO BEGIN
+ res=STRSPLIT(meta_arr[vni[i]],' =',/EXTRACT,COUNT=nres)
+ IF nres EQ 2 THEN dv[i]=res[1]
+ENDFOR
+;Check for obsolete _START/STOP/INTEGRATION sub-values and replace with DATETIME.START/STOP
+;or INTEGRATION.TIME (for single occurences only). Also check for double underscores and
+;replace with single underscores in the Variable Names
+GEOMS_RULE_CHANGES,1,vncnt,dv
+
+;Check for other obsolete variable names
+IF vncnt EQ vucnt THEN BEGIN
+ vuv=STRARR(vucnt)
+ FOR i=0,vucnt-1 DO BEGIN
+ res=STRSPLIT(meta_arr[vui[i]],'=',/EXTRACT,COUNT=nres)
+ IF nres EQ 2 THEN vuv[i]=STRTRIM(res[1],2)
+ ENDFOR
+ GEOMS_RULE_CHANGES,5,vncnt,dv,vuv,data_source
+ENDIF
+;Update meta_arr in the case of modified Dataset name values
+vci=WHERE(vnchange NE '',vcres)
+IF vcres NE 0 THEN BEGIN ;need to update VAR_NAME and DATA_VARIABLES attribute values
+ dvi=WHERE(STRMID(meta_arr,0,14) EQ 'DATA_VARIABLES')
+ meta_arr[dvi[0]]='DATA_VARIABLES='
+ FOR i=0,vncnt-1 DO BEGIN
+ ;Check for new VAR_NAME value and update
+ IF vnchange[i] NE '' THEN meta_arr[vni[i]]='VAR_NAME='+dv[i]
+ ;Append to new DATA_VARIABLES list
+ IF i EQ 0 THEN itxt='' ELSE itxt=';'
+ meta_arr[dvi[0]]=meta_arr[dvi[0]]+itxt+dv[i]
+ ENDFOR
+ENDIF
+
+;Check for obsolete CALVAL and EVDC FILE_ACCESS values and replace
+fai=WHERE(STRMID(meta_arr,0,11) EQ 'FILE_ACCESS')
+res=STRSPLIT(meta_arr[fai[0]],' =;',/EXTRACT,COUNT=facnt)
+IF facnt GE 2 THEN BEGIN
+ fav=STRUPCASE(res[1:facnt-1]) & fachange=fav
+ GEOMS_RULE_CHANGES,7,facnt-1,fav
+ IF ARRAY_EQUAL(fav,fachange) EQ 0 THEN BEGIN ;FILE_ACCESS values changed so update meta_arr
+ meta_arr[fai[0]]='FILE_ACCESS='
+ FOR i=0,facnt-2 DO BEGIN
+ IF i EQ 0 THEN itxt='' ELSE itxt=';'
+ meta_arr[fai[0]]=meta_arr[fai[0]]+itxt+fav[i]
+ ENDFOR
+ ENDIF
+ENDIF
+
+;Check for VAR_UNITS=DIMENSIONLESS or NONE and fix if possible.
+;Need to check VAR_DATA_TYPE. If string then VAR_UNITS value should be blank, o/w replace with '1'
+vdi=WHERE(STRMID(meta_arr,0,13) EQ 'VAR_DATA_TYPE',vdcnt)
+IF (vdcnt NE 0) AND (vdcnt EQ vucnt) THEN BEGIN
+ vdv=STRARR(vdcnt)
+ FOR i=0,vdcnt-1 DO BEGIN
+ res=STRSPLIT(meta_arr[vdi[i]],'=',/EXTRACT,COUNT=nres)
+ IF nres EQ 2 THEN vdv[i]=STRUPCASE(STRTRIM(res[1],2))
+ ENDFOR
+ vuchange=vuv
+ GEOMS_RULE_CHANGES,8,vucnt,vuv,vdv
+ FOR i=0,vucnt-1 DO BEGIN ;Check for changes and update meta_arr
+ IF STRUPCASE(vuchange[i]) NE vuv[i] THEN meta_arr[vui[i]]='VAR_UNITS='+vuv[i]
+ ENDFOR
+ENDIF
+
+;Check for non-permitted characters (must be ISO646-US character set)
+GEOMS_RULE_CHANGES,10,meta_arr
+
+END ;Procedure Read_Metadata
+
+
+
+PRO extract_and_test, dentry, cpos, nel, ta1, ta2, taname, aname, ti
+;Procedure called by Check_Metadata to extract the relevant parts of tab_arr and test against
+;a corresponding Metadata value.
+; ----------
+;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
+;
+; History:
+; 20050802: Original IDLCR8HDF Routine - Version 1.0
+; 20050909: Make VAR_SI_CONVERSION values match the VAR_DATA_TYPE, and add fix for VAR_SI_CONVERSION
+; calculation rules e.g. if VAR_UNITS=cm m-3 then VAR_SI_CONVERSION=0;0.01;m-2 - Version 1.1
+; 20061012: Procedure simplified, with all portions of the code related to the VAR_UNITS and
+; VAR_SI_CONVERSION calculations moved to the Var_Units_Test routine; Common variable
+; definition WIDGET_WIN added - Version 2.0
+; 20100205: Add RETURN command after all STOP_WITH_ERROR calls, which allows program to return to the
+; calling program if the reterr argument is included in the idlcr8hdf call - Version 3.09
+; 20120313: Add option to check for Metadata Entries that are only different to the TAV entry
+; because of case sensitivity and/or use of incorrect apostrophe ("`" instead of "'")
+; - Version 4.0b8
+; 20140226: For AFFILIATION, if there is no match, do additional check of the acronym only and,
+; if present, correct the value - Version 4.0b19
+; 20150811: Add infoerr array. For these Global Attributes the program will no longer stop, but
+; will generate an ERROR message and continue if there is a problem with any of the
+; entries - Version 4.0b29
+; 20161230: Add COUNTRY and AFFILIATION to infoerr array and allow for this when creating infotxt
+; message - Version 4.0b41
+; 20171121: Fix error caused by incorrectly referring to the variable infoerr as inforerr
+; - Version 4.0b43
+; 20191205: Account for DATA_SOURCE being able to have 2 or 3 sub-values, where the third sub-
+; value is a VERSION_NAME - Version 4.0b53
+; 20220805: Ensure the 3-digit instrument ID is numeric (each character is now checked in
+; IS_A_NUMBER_HDF) - Version 4.0b60
+;
+; Inputs: dentry - the Metadata entry to be checked (everything after the '=')
+; cpos - a pattern used by StrSplit on DEntry to split it into its component parts e.g. ';' or '_'
+; nel - the number of sub-values in the set of tab_arr entries being checked (ta1 only)
+; ta1 - the set of tab_arr values used to check against the Metadata contents
+; ta2 - a second set of tab_arr values used for DATA_SOURCE checks, o/w set to ['']
+; taname - a combination of the Metadata Attribute name and Table Field/Label used for the
+; error messages
+; aname - the Metadata attribute name to be used for the error message
+; ti - the index value of the ta2 values to be checked. Used for DATA_SOURCE checks only
+;
+; Output: ti - the index value(s) of tab_arr which match the Metadata value (used in Check_Metadata
+; to do further checks on the ORIGINATOR values)
+;
+; Called by: CHECK_METADATA
+;
+; Subroutines Called: STOP_WITH_ERROR (if error state detected)
+; Possible Conditions for STOP_WITH_ERROR call (plus [line number] where called):
+; 1. Incorrect number of (sub-)values for a particular Metadata attribute
+; 2. Metadata value does not match any values in the Table Attribute Values file
+
+COMMON TABLEDATA
+COMMON WIDGET_WIN
+
+proname='Check_Metadata/Extract_And_Test procedures: ' & lu=-1L
+errtxt2=STRARR(2)
+errtxt2[0]='Number of (sub-)values for this attribute should be '
+errtxt2[1]='Doesn''t match any Table Attribute Values under FIELD: '
+
+infoerr=['COUNTRY','AFFILIATION','FILE_ACCESS','FILE_META_VERSION','DATA_LOCATION', $
+ 'DATA_DISCIPLINE','DATA_GROUP'] ;do not stop the program for errors in these attributes
+
+;Check number of Metadata entry components is as expected
+achk=STRSPLIT(dentry,cpos,/Extract,COUNT=n_achk)
+oki=WHERE(n_achk EQ nel,okcnt) ;Note: DATA_SOURCE can have 2 or 3 sub-values (nel=[2,3] from 4.0b53)
+IF okcnt EQ 0 THEN BEGIN
+ STOP_WITH_ERROR,o3[3]+proname+taname+'='+dentry+': ',errtxt2[0]+STRTRIM(nel,2)+'.',lu
+ RETURN
+ENDIF
+
+;allow fix to be made to the affiliation if the acronym is correct but institute name is wrong
+n_ta1=N_ELEMENTS(ta1)
+IF (STRPOS(taname,'AFFILIATION') NE -1) AND (cpos NE '_') THEN BEGIN
+ affchk=1 & acroval=STRTRIM(achk[1],2) & t_acrovals=STRARR(n_ta1)
+ENDIF ELSE affchk=0
+
+;check Metadata entry with relevant TAV entry or entries
+IF cpos EQ '_' THEN BEGIN ;DATA_SOURCE entry so need to treat components separately
+ achk=STRCOMPRESS(achk,/Remove_All) & nel=1
+ achkh=' '+achk[1]
+ ;strip the last three characters (instrument ID) off achk(1)
+ achk[1]=STRMID(achk[1],0,STRLEN(achk[1])-3)
+ ;Check that the instrument ID is valid
+ instidchk=STRMID(achkh,STRLEN(achkh)-3,3)
+ IF ~IS_A_NUMBER_HDF(instidchk,2) THEN BEGIN
+ infotxt='3 DATA_SOURCE 3-digit instrument identifier missing or invalid'
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ENDIF ELSE achk=STRCOMPRESS(dentry,/Remove_All)
+FOR j=0,n_ta1-1 DO BEGIN
+ ;extract valid portion of the entries (depending on ATTRIBUTE name)
+ tchk=STRSPLIT(ta1[j],';',/Extract)
+ ta1[j]=STRCOMPRESS(tchk[0],/Remove_All)
+ IF nel GT 1 THEN $
+ FOR k=1,nel-1 DO ta1[j]=ta1[j]+';'+STRCOMPRESS(tchk[k],/Remove_All)
+ IF affchk EQ 1 THEN t_acrovals[j]=tchk[1]
+ENDFOR
+IF ta2[0] NE '' THEN BEGIN
+ FOR j=0,N_ELEMENTS(ta2)-1 DO BEGIN
+ ;extract valid portion of the entries (depending on ATTRIBUTE name)
+ tchk=STRSPLIT(ta2[j],';',/Extract)
+ ta2[j]=STRCOMPRESS(tchk[ti],/Remove_All)
+ ENDFOR
+ENDIF
+IF cpos EQ '_' THEN BEGIN
+ gi=WHERE(ta1 EQ achk[0],gcnt)
+ IF gcnt NE 0 THEN gi=WHERE(ta2 EQ achk[1],gcnt)
+ENDIF ELSE BEGIN ;Check for Case Sensitive or incorrect apostrophe problem
+ gi=WHERE(ta1 EQ achk,gcnt)
+ IF gcnt EQ 0 THEN BEGIN
+ ;Check for Case Sensitivity and/or incorrect apostrophe usage or correct affiliation acronym only
+ ;Will be fixed in CHECK_METADATA
+ gi=WHERE(STRLOWCASE(ta1) EQ STRLOWCASE(achk),gcnt)
+ IF gcnt NE 0 THEN dentry=ta1[gi[0]] ELSE BEGIN
+ ;Check for "`" instead of "'" in metadata entry
+ IF STRPOS(achk,'`') NE -1 THEN BEGIN
+ achkh=achk
+ WHILE STRPOS(achkh,'`') NE -1 DO STRPUT,achkh,'''',STRPOS(achkh,'`')
+ gi=WHERE(STRLOWCASE(ta1) EQ STRLOWCASE(achkh),gcnt)
+ IF gcnt NE 0 THEN dentry=ta1[gi[0]]
+ ENDIF
+ ;If AFFILIATION check to see if the acronym only is valid
+ IF affchk EQ 1 THEN BEGIN
+ gi=WHERE(STRLOWCASE(acroval) EQ STRLOWCASE(t_acrovals),gcnt)
+ IF gcnt NE 0 THEN dentry=ta1[gi[0]]
+ ENDIF
+ ENDELSE
+ ENDIF
+ENDELSE
+
+IF gcnt EQ 0 THEN BEGIN
+ ei=-1
+ FOR i=0,N_ELEMENTS(infoerr)-1 DO IF STRPOS(taname,infoerr[i]) NE -1 THEN ei=i
+
+ IF ei NE -1 THEN BEGIN ;Log message instead of STOP_WITH_ERROR
+ IF ei LE 1 THEN metalabel=STRMID(taname,STRPOS(taname,'/')+1) ELSE metalabel=infoerr[ei]
+ infotxt='3 '+metalabel+'='+aname+' doesn''t match any '
+ infotxt=infotxt+'Table Attribute Values under '+infoerr[ei]+' field'
+ INFOTXT_OUTPUT,infotxt
+ ENDIF ELSE BEGIN
+ STOP_WITH_ERROR,o3[3]+proname+aname+': ',errtxt2[1]+taname,lu & RETURN
+ ENDELSE
+ENDIF
+ti=gi ;ti returned to the Check_MetaData procedure to check AFFILIATION, ADDRESS and EMAIL values
+
+END ;procedure Extract_and_Test
+
+
+
+PRO check_metadata
+;Procedure to check the validity of Metadata values by checking them against the corresponding
+;entries in the Table Attribute Values File
+; ----------
+;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
+;
+; History:
+; 20050802: Original IDLCR8HDF Routine - Version 1.0
+; 20050909: 'rd' flag included to convert VAR_SI_CONVERSION values to floating point if the
+; corresponding VAR_DATA_TYPE is real or double; Section included to check for
+; repeated SI units in the calculated VAR_SI_CONVERSION - Version 1.1
+; 20061012: Section added to do simple checks on Metadata ORIGINATOR values if the information
+; is not otherwise available in the TAV file; Warning messages written to output
+; windows and/or log file depending on the options chosen by the user; Section relating
+; to checks on VAR_UNITS and VAR_SI_CONVERSION moved to the VAR_UNITS_TEST routine;
+; Common variable definition WIDGET_WIN added - Version 2.0
+; 20080302: Sections included to handle checks when an Envisat table.dat file is used, including;
+; ensuring metadata labels DATA_TYPE and VAR_DATA_TYPE are not both found when using the
+; Strpos procedure; Handling different table.dat format of ORIGINATOR attributes; using
+; DATA_SOURCE_02 instead of AFFILIATION for the DATA_SOURCE checks. Section which puts
+; TAV file VAR_UNITS and UNIT_PREFIX values into arrays for the VAR_UNITS_TEST call is
+; no longer needed and is removed; Change made so that Free Attribute Metadata values
+; which include semi-colons do not have white space removed (i.e. the semi-colons are
+; not defined as separating sub-values) - Version 3.0
+; 20091203: Account for the situation when a VAR_NAME does not include a Variable Mode, but still
+; includes a Variable Descriptor ('__' will be present in the name) - Version 3.08
+; 20100205: Add RETURN command after all STOP_WITH_ERROR calls, which allows program to return to the
+; calling program if the reterr argument is included in the idlcr8hdf call - Version 3.09
+; 20101120: Do check for MJD2000 VAR_UNIT, and change to MJD2K; Check for LONG datatype (now refers to
+; LONG64) and change to INTEGER (now means LONG32) for TAV checks - Version 4.0b0
+; 20121015: Change VAR_SI_CONVERSION information/error message so will also report if the
+; VAR_SI_CONVERSION label does not have an accompanying value - Version 4.0b13
+; 20141110: Add check for DATETIME where VAR_SI_CONVERSION=0.0;86400.0;s, but VAR_UNITS is not MJD2K.
+; Will change VAR_UNITS to MJD2K - Version 4.0b24
+; 20150127: Stop removing leading and trailing spaces from variable attribute values (when there is
+; only 1 value) to allow for possibility of VAR_FILL_VALUEs being an empty string
+; - Version 4.0b25
+; 20150811: Remove STOP_WITH_ERROR message if FILE_META_VERSION doesn't have 2 values. This check is
+; now done in the READ_METADATA routine - Version 4.0b29
+; 20151012: Remove checks on white space in Metadata values. This check is now done in the
+; READ_METADATA routine - Version 4.0b31
+; 20161130: Allow the program to continue when an error is generated after doing VAR_UNITS and
+; VAR_SI_CONVERSION checks (previously called STOP_WITH_ERROR) - Version 4.0b40
+; 20161230: Modify checks on ORIGINATOR values as this section removed from the TAV file from
+; 04R025. Add tests for COUNTRY in XX_ADDRESS, and check for '@' in XX_EMAIL -
+; Version 4.0b41
+; 20170331: Allow for COUNTRY value in XX_ADDRESS to be all uppercase or have initial capitalization
+; (allow for special cases e.g. Cote d'Ivoire; words such as the, and, of also not
+; capitalized) - Version 4.0b42
+; 20191205: Add check for optional VERSION_NAME sub-value of DATA_SOURCE. If present then call
+; GEOMS_RULE_CHANGES to do checks on the value - Version 4.0b53
+; 20240912: Allow for VAR_SI_CONVERSION to have two possible values depending on the ordering of units
+; that have the same power value e.g. m-1 sr-1 or sr-1 m-1 - Version 4.0b64
+;
+; Inputs: meta_arr - a string array containing the Global and Variable Attributes
+; tab_arr - a string array containing the TAV or table.dat file attributes
+
+; Output: meta_arr - some meta_arr values may be corrected or calculated in this routine. White space
+; between metadata sub-values is also removed.
+;
+; Called by: IDLCR8HDF
+;
+; Subroutines Called: EXTRACT_AND_TEST
+; VAR_UNITS_TEST
+; GEOMS_RULE_CHANGES
+; STOP_WITH_ERROR (if error state detected); INFOTXT_OUTPUT
+; Possible Conditions for STOP_WITH_ERROR call (plus [line number] where called):
+; 1. An attribute value is expected but is absent or invalid
+; 2. Incorrect number of (sub-)values for a particular Metadata attribute
+; 3. NAME and AFFILIATION values don't match (only applies if reading in the full AVDC
+; TAV file)
+; 4. AFFILIATION (AVDC) or DATA_SOURCE_02 (Envisat) field not found in the TAV file
+; 5. There is no match between a given VAR_UNIT, and the list of valid units and prefixes
+; present in the TAV file
+; 6. VAR_UNITS value is not valid
+;
+; Information Conditions (when the program is able to make changes):
+; 1. ORIGINATOR values for ADDRESS or EMAIL replaced with those from the TAV file
+; 2. VAR_SI_CONVERSION values replaced with those calculated by the program
+
+COMMON TABLEDATA
+COMMON METADATA
+COMMON DATA
+COMMON WIDGET_WIN
+
+;Possible error messages for this procedure
+proname='Check_Metadata procedure: ' & lu=-1L
+errtxt=STRARR(4)
+errtxt[0]='No or Invalid value for this Attribute: '
+errtxt[1]='Number of (sub-)values for this Attribute should be ' ;changed to information message
+errtxt[2]='AFFILIATION value doesn''t match NAME value: '
+errtxt[3]=' field not found in the input Table Attribute Values file'
+
+;originator values to be checked
+orig_attr=['PI_','DO_','DS_','NAME','AFFILIATION','ADDRESS','EMAIL']
+countrysp1=['Congo, The Democratic Republic of the','Cote d''Ivoire'] ;special cases for initial capitalization of the COUNTRY value
+countrysp2=['the','and','of'] ;special cases for initial capitalization of the COUNTRY value (individual words)
+
+;initialize variables for VAR_UNITS/VAR_SI_CONVERSION checks
+firstvu=0 & tchk=[''] & rd=1 & writeonce=0
+xval=[2,2,3,1] ;expected number of sub-values for the four originator attributes
+new_name=[''] ;Entry which will contain any ORIGINATOR name values not present in the TAV file
+
+FOR i=0,N_ELEMENTS(meta_arr)-1 DO BEGIN
+ res=STRSPLIT(meta_arr[i],'=',/Extract,COUNT=nres)
+ IF res[0] EQ 'VAR_NAME' THEN vnval=meta_arr[i]
+ FOR j=0,2 DO BEGIN
+ FOR k=3,6 DO BEGIN
+ ;need to change ORIGINATOR NAME ATTRIBUTE names to match TAV File
+ IF res[0] EQ orig_attr[j]+orig_attr[k] THEN BEGIN
+ res[0]='ORIGINATOR' & jh=j & kh=k
+ ENDIF
+ ENDFOR
+ ENDFOR
+ ;Test whether metadata label is also a TAV field
+ ;First test checks for exact match between metadata label and TAV field
+ ;(This avoids possible problem using STRPOS (e.g. DATA_TYPE won't also pick up VAR_DATA_TYPE))
+ mi=WHERE(tab_arr[*,0] EQ res[0],mcnt)
+ ;Back-up test in the event that label has more than one TAV field (e.g. DATA_VARIABLES)
+ IF mcnt EQ 0 THEN mi=WHERE(STRPOS(tab_arr[*,0],res[0]) EQ 0,mcnt) ;was NE -1
+
+ IF res[0] EQ 'ORIGINATOR' THEN BEGIN
+ ;Do checks on _NAME, _AFFILIATION, _ADDRESS, and _EMAIL sub-values
+ resx=STRSPLIT(meta_arr[i],'=;',/Extract)
+ nel=N_ELEMENTS(resx)-1
+
+ IF (kh EQ 4) AND (nel EQ xval[1]) THEN BEGIN ;check on AFFILIATION value
+ resy=STRSPLIT(meta_arr[i],'=',/Extract)
+ mci=WHERE(tab_arr[*,0] EQ 'AFFILIATION',mccnt)
+ IF mccnt NE 0 THEN BEGIN
+ ta1=tab_arr[mci[0],2:tab_arr[mci[0],1]+1] ;subset of relevant FIELD ENTRIES
+ taname=tab_arr[mci[0],0]+'/'+resy[0] ;name combination for error message
+ dentry=resy[1] & ti=-1
+ EXTRACT_AND_TEST,dentry,';',xval[1],ta1,[''],taname,resy[1],ti
+ IF STRLEN(rerr[0]) GT 2 THEN RETURN
+ IF dentry NE resy[1] THEN BEGIN
+ ;Case and/or apostrophe differences between Metadata entry and TAV entry
+ achk2=STRSPLIT(tab_arr[mci[0],ti[0]+2],';',/Extract)
+ achk=STRTRIM(achk2[0],2)
+ FOR m=1,xval[1]-1 DO achk=achk+';'+STRTRIM(achk2[m],2)
+ IF qa_yes THEN itxt=' should be ' ELSE itxt=' replaced with '
+ infotxt='2 '+meta_arr[i]+itxt+resy[0]+'='+achk+' based on the TAV entry'
+ INFOTXT_OUTPUT,infotxt
+ meta_arr[i]=resy[0]+'='+achk
+ ENDIF
+ ENDIF
+ ENDIF
+ IF (kh EQ 5) AND (nel EQ xval[2]) THEN BEGIN ;check on COUNTRY value
+ mci=WHERE(tab_arr[*,0] EQ 'COUNTRY',mccnt)
+ IF mccnt NE 0 THEN BEGIN
+ ;check the COUNTRY value against the TAV values (note: can be all uppercase or initial capitalization)
+ aname=res[1] & ti=-1
+ ta1=tab_arr[mci[0],2:tab_arr[mci[0],1]+1] ;subset of relevant FIELD ENTRIES
+ taname=tab_arr[mci[0],0]+'/'+orig_attr[jh]+'ADDRESS' ;name combination for error message
+ EXTRACT_AND_TEST,STRUPCASE(resx[nel]),';',1,ta1,[''],taname,aname,ti
+ IF STRLEN(rerr[0]) GT 2 THEN RETURN
+ ;Test for upper case or initial capitalization
+ IF resx[nel] NE STRUPCASE(resx[nel]) THEN BEGIN
+ counterr=0 ;error code to detect problem with initial capitalization
+ sci=WHERE(STRUPCASE(resx[nel]) EQ STRUPCASE(countrysp1),spcnt)
+ IF spcnt NE 0 THEN BEGIN ;Test special case countries e.g. Cote d'Ivoire
+ IF resx[nel] NE countrysp1[sci[0]] THEN counterr=1
+ ENDIF ELSE BEGIN ;Test individual words in the country value
+ sepcount=STRSPLIT(resx[nel],' ,',COUNT=spcnt,/EXTRACT)
+ FOR j=0,spcnt-1 DO BEGIN
+ slen=STRLEN(sepcount[j])
+ sci=WHERE(STRUPCASE(sepcount[j]) EQ STRUPCASE(countrysp2),sp2cnt)
+ IF sp2cnt NE 0 THEN test=countrysp2[sci[0]] $ ;special case lowercase word e.g. the, and, of
+ ELSE IF slen EQ 1 THEN test=STRUPCASE(sepcount[j]) $ ;only one letter in the word so make uppercase
+ ELSE test=STRUPCASE(STRMID(sepcount[j],0,1))+STRLOWCASE(STRMID(sepcount[j],1)) ;capitalize the first letter
+ IF sepcount[j] NE test THEN counterr=1
+ ENDFOR
+ ENDELSE
+ IF counterr EQ 1 THEN BEGIN
+ uccountry=STRUPCASE(resx[nel])
+ IF qa_yes THEN itxt=' should be ' ELSE itxt=' replaced with '
+ infotxt='2 COUNTRY value for '+meta_arr[i]+itxt+uccountry
+ INFOTXT_OUTPUT,infotxt
+ meta_arr[i]=STRMID(meta_arr[i],0,STRPOS(meta_arr[i],';',/REVERSE_SEARCH)+1)+uccountry
+ ENDIF
+ ENDIF
+ ENDIF
+ ENDIF
+ IF (kh EQ 6) AND (nel EQ xval[3]) THEN BEGIN ;check @ present in the e-mail value
+ amppos=STRPOS(res[1],'@')
+ IF (amppos LE 0) OR (amppos EQ STRLEN(res[1])-1) THEN BEGIN
+ infotxt='3 Invalid format for '+orig_attr[jh]+'EMAIL value'
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ ENDIF
+ IF (nel NE xval[kh-3]) OR (STRMID(meta_arr[i],STRLEN(meta_arr[i])-1,1) EQ ';') THEN BEGIN
+ IF xval[kh-3] GT 1 THEN itxt='sub-' ELSE itxt=''
+ infotxt='3 Number of '+itxt+'values for attribute '+meta_arr[i]+' should be '+STRTRIM(xval[kh-3],2)
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ mcnt=0
+ ENDIF
+
+ IF mcnt NE 0 THEN BEGIN ;check Metadata value(s) against TAV file entries
+ IF (nres NE 2) AND (rd NE 0) THEN BEGIN
+ STOP_WITH_ERROR,o3[3]+proname,errtxt[0]+meta_arr[i],lu & RETURN
+ ENDIF ELSE IF res[0] EQ 'DATA_SOURCE' THEN BEGIN
+ ;need to test against 'DATA_SOURCE' and 'AFFILIATION' (AVDC) or 'DATA_SOURCE_02'
+ ;(original Envisat) entries
+ ta1=tab_arr[mi[0],2:tab_arr[mi[0],1]+1] ;DATA_SOURCE ENTRIES
+ IF tab_type EQ 0 THEN BEGIN
+ atxt='AFFILIATION' & atx='' & ti=1
+ ENDIF ELSE BEGIN
+ atxt='DATA_SOURCE_02' & atx='_01' & ti=0
+ ENDELSE
+ ai=WHERE(STRPOS(tab_arr[*,0],atxt) NE -1,acnt)
+ IF acnt EQ 0 THEN BEGIN
+ STOP_WITH_ERROR,o3[3]+proname,atxt+errtxt[3],lu & RETURN
+ ENDIF
+ ta2=tab_arr[ai[0],2:tab_arr[ai[0],1]+1] ;AFFILIATION/DATA_SOURCE_02 ENTRIES
+ taname=res[0]+atx+'/'+tab_arr[ai[0],0] ;name combination for error message
+ n_sv=[2,3] ;possible number of DATA_SOURCE sub-values
+ EXTRACT_AND_TEST,res[1],'_',n_sv,ta1,ta2,taname,res[1],ti
+ IF STRLEN(rerr[0]) GT 2 THEN RETURN
+
+ achk=STRSPLIT(res[1],'_',/EXTRACT,COUNT=n_achk)
+ IF n_achk EQ 3 THEN BEGIN
+ ;Check DATA_VERSION_NAME if there are 3 sub-values in DATA_SOURCE (introduced v4.0b53)
+ GEOMS_RULE_CHANGES,11,achk[2],meta_arr
+ ENDIF
+
+ ENDIF ELSE IF res[0] EQ 'DATA_VARIABLES' THEN BEGIN
+ ;make into 2-D array and separate out the components of the VAR_NAMES
+ achk=STRSPLIT(res[1],';',/Extract,COUNT=nel)
+ achk2=STRARR(mcnt,nel)
+ FOR j=0,nel-1 DO BEGIN
+ dv=STRSPLIT(achk[j],'_',/Extract,Count=n_dv)
+ IF n_dv GT mcnt THEN BEGIN ;invalid VAR_NAME composition
+ STOP_WITH_ERROR,o3[3]+proname,errtxt[0]+res[0]+'='+achk[j],lu & RETURN
+ ENDIF
+ achk2[0:N_ELEMENTS(dv)-1,j]=dv
+ ENDFOR
+ FOR j=0,nel-1 DO BEGIN
+ dv=WHERE(achk2[*,j] NE '',dvcnt)
+ FOR k=0,mcnt-1 DO BEGIN
+ IF achk2[k,j] NE '' THEN BEGIN
+ taname=tab_arr[mi[k],0]
+ ta1=REFORM(tab_arr[mi[k],2:tab_arr[mi[k],1]+1]) ;subset of TAV DATA_VARIABLES entries
+ IF (k NE 0) AND (dvcnt EQ 2) THEN BEGIN
+ ;need to check both variable mode and descriptor options
+ FOR m=k+1,mcnt-1 DO BEGIN
+ taname=taname+'/'+tab_arr[mi[m],0]
+ ta1=[ta1,REFORM(tab_arr[mi[m],2:tab_arr[mi[m],1]+1])]
+ ENDFOR
+ ENDIF
+ EXTRACT_AND_TEST,achk2[k,j],';',1,ta1,[''],taname,achk[j],0
+ IF STRLEN(rerr[0]) GT 2 THEN RETURN
+ ENDIF
+ ENDFOR
+ ENDFOR
+ ENDIF ELSE IF res[0] EQ 'FILE_ACCESS' THEN BEGIN
+ ;need to check each Metadata entry individually
+ achk=STRSPLIT(res[1],';',/Extract)
+ nel=N_ELEMENTS(achk)
+ ta1=tab_arr[mi[0],2:tab_arr[mi[0],1]+1] ;FILE_ACCESS entries
+ FOR j=0,nel-1 DO BEGIN
+ EXTRACT_AND_TEST,achk[j],';',1,ta1,[''],res[0],achk[j],0
+ IF STRLEN(rerr[0]) GT 2 THEN RETURN
+ ENDFOR
+ ENDIF ELSE IF res[0] EQ 'FILE_META_VERSION' THEN BEGIN
+ ;only need to look at the second value
+ achk=STRSPLIT(res[1],';',/Extract)
+ IF N_ELEMENTS(achk) NE 2 THEN achk=[achk,'x','x'] ;ensure there are at least 2 values
+ achk=[STRTRIM(achk[1],2)]
+ ta1=tab_arr[mi[0],2:tab_arr[mi[0],1]+1] ;FILE_META_VERSION entries
+ EXTRACT_AND_TEST,achk[0],';',1,ta1,[''],res[0],achk[0],0
+ IF STRLEN(rerr[0]) GT 2 THEN RETURN
+ ENDIF ELSE IF res[0] EQ 'VAR_UNITS' THEN BEGIN
+ ;VAR_UNITS and VAR_SI_CONVERSION entries (not needed for STRING data type)
+ IF rd NE 0 THEN BEGIN
+ IF STRUPCASE(res[1]) EQ 'MJD2000' THEN BEGIN ;change to MJD2K
+ in1=meta_arr[i] & in2=res[1]
+ GEOMS_RULE_CHANGES,3,in1,in2,writeonce
+ meta_arr[i]=in1 & res[1]=in2
+ writeonce=1
+ ENDIF
+ errstate='' & datetimechk=STRPOS(STRUPCASE(vnval),'DATETIME') NE -1
+ VAR_UNITS_TEST,res[1],rd,tab_type,tchk,errstate
+ IF errstate NE '' THEN BEGIN ;VAR_UNIT value not valid
+ ;STOP_WITH_ERROR,o3[3]+proname+STRMID(errstate,0,STRPOS(errstate,':')+2), $
+ ; STRMID(errstate,STRPOS(errstate,':')+2),lu
+ ;RETURN
+ IF ~datetimechk THEN itxt='. VAR_SI_CONVERSION value not checked' ELSE itxt=''
+ infotxt='3 '+errstate+itxt
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ ;Find VAR_SI_CONVERSION attribute
+ k=1
+ WHILE STRMID(meta_arr[i+k],0,17) NE 'VAR_SI_CONVERSION' DO k++
+ achk=STRSPLIT(meta_arr[i+k],'=',/Extract) ;VAR_SI_CONVERSION values
+ IF N_ELEMENTS(achk) EQ 1 THEN achk=[achk,' ']
+ IF datetimechk THEN BEGIN
+ ;Check if VAR_SI_CONVERSION values are 0.0;86400.0;s regardless of VAR_UNITS. If so, then don't change
+ achk1=STRSPLIT(achk[1],' ;',/EXTRACT)
+ achk1=[achk1,'','',''] ;Ensure there are at least 3 values extracted
+ IF (is_a_number_hdf(achk1[0])) AND (is_a_number_hdf(achk1[1])) THEN BEGIN
+ IF (FIX(achk1[0]) EQ 0) AND (LONG(achk1[1]) EQ 86400L) AND $
+ (STRLOWCASE(achk1[2]) EQ 's') THEN tchk[*]='0.0;86400.0;s'
+ ENDIF ELSE IF errstate NE '' THEN tchk[*]='0.0;86400.0;s'
+ ENDIF
+ ;check calculated VAR_SI_CONVERSION value(s) against existing value in the metadata
+ IF (errstate EQ '') OR (datetimechk) THEN BEGIN
+ vscgi=WHERE(achk[1] EQ tchk,vscgcnt)
+ IF vscgcnt EQ 0 THEN BEGIN
+ IF qa_yes THEN itxt=' should be ' ELSE itxt=' replaced with '
+ infotxt='2 '+meta_arr[i+k]+itxt+'VAR_SI_CONVERSION='+tchk[0]+' for '+vnval
+ INFOTXT_OUTPUT, infotxt
+ meta_arr[i+k]=achk[0]+'='+tchk[0]
+ ENDIF ELSE meta_arr[i+k]=achk[0]+'='+tchk[vscgi[0]]
+ ENDIF
+ ENDIF
+ ENDIF ELSE BEGIN ;perform checks on the remaining ATTRIBUTE_NAMES
+ achk=STRSPLIT(res[1],';',/Extract)
+ achk=STRTRIM(achk,2)
+ nel=N_ELEMENTS(achk) & ta2=['']
+ IF res[0] EQ 'VAR_DATA_TYPE' THEN BEGIN
+ ;to convert VAR_SI_CONVERSION values to floating point if necessary
+ CASE 1 OF
+ (STRUPCASE(res[1]) EQ 'REAL') OR (STRUPCASE(res[1]) EQ 'DOUBLE'): rd=-1
+ STRUPCASE(res[1]) EQ 'STRING': rd=0
+ ELSE: rd=1
+ ENDCASE
+ IF STRUPCASE(achk[0]) EQ 'LONG' THEN achk[0]='INTEGER'
+ ;Changed to INTEGER for EXTRACT_AND_TEST only - actual change occurs in SET_UP_STRUCTURE
+ ENDIF
+ IF (nel NE mcnt) OR (STRMID(meta_arr[i],STRLEN(meta_arr[i])-1,1) EQ ';') THEN BEGIN
+ IF mcnt GT 1 THEN itxt='sub-' ELSE itxt=''
+ infotxt='3 Number of '+itxt+'values for attribute '+meta_arr[i]+' should be '+STRTRIM(mcnt,2)
+ INFOTXT_OUTPUT,infotxt
+ ;STOP_WITH_ERROR,o3[3]+proname+meta_arr[i]+': ',errtxt[1]+STRTRIM(mcnt,2)+'.',lu
+ ;RETURN
+ ENDIF
+ nel=1
+ FOR j=0,mcnt-1 DO BEGIN
+ ta1=tab_arr[mi[j],2:tab_arr[mi[j],1]+1] ;subset of relevant ENTRIES
+ dentry=achk[j] & ti=mi[j]
+ EXTRACT_AND_TEST,dentry,';',nel,ta1,ta2,tab_arr[mi[j],0],achk[j],ti
+ IF STRLEN(rerr[0]) GT 2 THEN RETURN
+ IF (mcnt EQ 1) AND (dentry NE achk[j]) THEN BEGIN
+ ;Case and/or apostrophe differences between Metadata entry and TAV entry
+ achk2=STRSPLIT(tab_arr[mi[j],ti[0]+2],';',/Extract)
+ achk=STRTRIM(achk2[0],2)
+ IF qa_yes THEN itxt=' should be ' ELSE itxt=' replaced with '
+ infotxt='2 '+meta_arr[i]+itxt+res[0]+'='+achk+' based on the TAV entry'
+ INFOTXT_OUTPUT,infotxt
+ meta_arr[i]=res[0]+'='+achk
+ ENDIF
+ IF j EQ 0 THEN ta2[0]=''
+ ENDFOR
+ ENDELSE
+ ENDIF
+ENDFOR
+
+END ;Procedure Check_Metadata
+
+
+
+PRO extract_data, sds, inf, vc, dtest, ndl, infowrite
+;Procedure to extract the set of data corresponding to a particular Variable Attribute
+; ----------
+;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
+;
+; History:
+; 20050802: Introduced to IDLCR8HDF, originally part of the READ_DATA procedure - Version 1.0
+; 20050909: After reading in a data segment according to the VAR_SIZE values, check the next line to
+; ensure it is not a further data point (i.e. there are more data points than specified by
+; VAR_SIZE); If sds is a structure, and there is only one data point then return dtest as
+; an array rather than a scalar - Version 1.1
+; 20061012: The portion of READ_DATA dealing with extracting a specific data segment renamed
+; EXTRACT_DATA - Version 2.0
+; 20080302: The sets of data values in a file can now be in any order, rather than the same order as
+; that given by the DATA_VARIABLES attribute; dtest used as an input flag to indicate
+; whether the procedure call is from SET_UP_STRUCTURE [-1] or READ_DATA [0]. If from
+; SET_UP_STRUCTURE then only the number of data lines (ndl) is determined by the routine;
+; an error is generated in the case where the same VAR_NAME is repeated in the data file;
+; Error messages clarified when an incorrect number of data values are found in the
+; input data file - Version 3.0
+; 20100205: Add RETURN command after all STOP_WITH_ERROR calls, which allows program to return to the
+; calling program if the reterr argument is included in the idlcr8hdf call - Version 3.09
+; 20130116: Account for empty string data values in a dataset - previously '' values ignored in an
+; input data file - Version 4.0b15
+; 20150227: Do not automatically run STRTRIM on the dataset when it is read in to allow checks on
+; string datasets - Version 4.0b26
+; 20150811: Add infowrite parameter so that if DATETIME type dataset is in ISO8601 format then the
+; information/error message will only be written once instead of for each value
+; - Version 4.0b29
+; 20171130: Remove ds from STOP_WITH_ERROR parameters - Version 4.0b44
+;
+; Inputs: sds - Either a structure containing the Variable Attributes and Data, or the input
+; data file
+; inf - flag identifying sds type where, 0=structure; 1=data file
+; vc - the index value of the vn array holding the VAR_NAME being searched for in the data
+; file, or the index value of the sds structure which holds the data
+; dtest - a single string array to show which procedure called this routine, either
+; SET_UP_STRUCTURE [-1], or READ_DATA [0]
+; infowrite - optional input to stop ISO8601 to MJD2K information message being written multiple
+; times (when called by READ_DATA only)
+;
+; Outputs: dtest - a string array of size ndl that holds the data values
+; ndl - The total number of data lines i.e. the product of the VAR_SIZE values
+;
+; Called by: SET_UP_STRUCTURE; READ_DATA
+;
+; Subroutines Called: STOP_WITH_ERROR (if error state detected)
+; Possible Conditions for STOP_WITH_ERROR call (plus [line number] where called):
+; 1. EOF encountered when trying to find the VAR_NAME or while attempting to read in data
+; [2146,2220,2221]
+; 2. Invalid data point found in the dataset [2193]
+; 3. Two instances of the same VAR_NAME found in the input data file [2204]
+; 4. Too many data values present in the dataset or the next line in the file is invalid [2211]
+; 5. Number of data values does not match product of VAR_SIZE [2237]
+; 6. ISO8601 datetime format not valid [2253]
+
+COMMON DATA
+COMMON WIDGET_WIN
+
+;possible error messages for this procedure
+errtxt=STRARR(10)
+errtxt[0]='EOF encountered but not expected'
+errtxt[1]=' when trying to find VAR_NAME in '
+errtxt[2]=' when trying to determine VAR_SIZE for VAR_NAME='
+errtxt[3]=' - Number of data lines should be '
+errtxt[4]='Invalid data point found in the dataset at line '
+errtxt[5]='Two instances of this VAR_NAME present in the Data File'
+errtxt[6]='Too many data values present in the dataset or invalid data line entry following the dataset'
+errtxt[7]='Number of data values does not match product of VAR_SIZE: '
+errtxt[8]='ISO8601 format not valid: '
+errtxt[9]='Unable to determine correct number of data values due to VAR_SIZE calculation error'
+proname='Extract_Data procedure: '
+
+ndl=1L ;initialize data line count
+vsi=WHERE(vserror EQ vn[vc],vcnt) ;check whether there is a VAR_SIZE calculation error for this dataset
+sdata=ndm(vc,8) EQ 6 ;Check for String data type
+
+IF inf EQ 1 THEN BEGIN ;Find the first data value for the requested VAR_NAME
+ ON_IOERROR,TypeConversionError
+ dum='' & vntest=vn[vc]
+ ;Add check in case VAR_NAME has been changed from original input
+ IF vnchange[vc] NE '' THEN vntest=vnchange[vc]
+ OPENR,lu,sds,/GET_LUN
+ IF NOT EOF(lu) THEN BEGIN ;read through data file until the correct Variable Name is found
+ REPEAT READF,lu,dum UNTIL (EOF(lu)) OR (STRTRIM(STRUPCASE(dum),2) EQ STRUPCASE(vntest)) OR $
+ (STRTRIM(STRUPCASE(dum),2) EQ STRUPCASE(vn[vc]))
+ ENDIF
+ IF STRTRIM(STRUPCASE(dum),2) EQ STRUPCASE(vn[vc]) THEN vntest=vn[vc]
+ IF EOF(lu) THEN BEGIN
+ STOP_WITH_ERROR,o3[3]+proname+vn[vc],errtxt[0]+errtxt[1]+sds,lu & RETURN
+ ENDIF
+
+ ;Find first data point
+ dum='!'
+ valid=-1 ;set to test for successful line read
+ WHILE (STRMID(dum,0,1) EQ '!') OR (STRMID(dum,0,1) EQ '#') OR ((~sdata) AND (strtrim(dum,2) EQ '')) DO BEGIN
+ READF,lu,dum ;& dum=STRTRIM(dum,2)
+ ENDWHILE
+ valid=1 ;i.e. got to here without prematurely reaching the eof so first data point found OK
+
+ IF dtest[0] EQ '-1' THEN BEGIN ;need to determine the total number of data lines only (ndl)
+ IF NOT EOF(lu) THEN BEGIN
+ ;read through until eof or comment line or another VAR_NAME is reached
+ REPEAT BEGIN
+ READF,lu,dum
+ dum=STRTRIM(STRUPCASE(dum),2)
+ bi=WHERE(dum EQ STRUPCASE(vn))
+ IF (STRMID(dum,0,1) NE '!') AND (STRMID(dum,0,1) NE '#') AND $
+ ((dum NE '') OR ((sdata) AND (dum EQ ''))) AND (bi[0] EQ -1) THEN incnt=1L $
+ ELSE incnt=0L
+ ndl=ndl+incnt ;presume it is a valid data line so add to total
+ ENDREP UNTIL (incnt EQ 0L) OR (EOF(lu))
+ ENDIF
+ ENDIF ELSE BEGIN ;return dtest and ndl
+ di=WHERE(ndm[vc,0:7] NE 0L,ndim)
+ ;multiply together the array dimensions to determine expected total number of datalines
+ FOR i=0,ndim-1 DO ndl=ndl*ndm[vc,di[i]]
+ ;attempt to read any remaining data values
+ IF ndl GT 1L THEN BEGIN
+ valid=0 ;set to test for successful block read
+ dtest=STRARR(ndl-1) & READF,lu,dtest
+ valid=1 ;i.e. got to here without prematurely reaching the EOF so block read was OK
+ dtest=[dum,dtest] ;add first value to dtest
+ ENDIF ELSE dtest=[dum]
+ dtestup=STRUPCASE(STRTRIM(dtest,2)) ;& dtest=STRTRIM(dtest,2)
+ ;Test to see if any of the 'data' read in is a VAR_NAME - if so bcnt will NE 0 (and will generate error message)
+ bcnt=0 & i=0
+ WHILE (bcnt EQ 0) AND (i LE nvn-1) DO BEGIN
+ bi=WHERE(dtestup EQ STRUPCASE(vn[i]),bcnt)
+ IF bcnt EQ 0 THEN i=i+1
+ ENDWHILE
+ ;Test to see if any of the 'data' read in is not a valid data point - if so bcnt will NE 0
+ IF bcnt EQ 0 THEN $
+ bi=WHERE((STRMID(dtestup,0,1) EQ '!') OR (STRMID(dtestup,0,1) EQ '#') OR $
+ ((dtestup EQ '') AND (~sdata)) OR (dtestup EQ STRUPCASE(vntest)),bcnt)
+
+ IF bcnt NE 0 THEN BEGIN ;Non-valid data point found in the dataset
+ STOP_WITH_ERROR,o3[3]+proname+vn[vc]+': ',errtxt[4]+STRTRIM(bi[0]+1,2)+errtxt[3]+ $
+ STRTRIM(ndl,2),lu
+ RETURN
+ ENDIF
+
+ ;Read next line if not EOF to test for the correct number of data lines
+ IF NOT EOF(lu) THEN READF,lu,dum ELSE dum='!'
+ dum=STRTRIM(STRUPCASE(dum),2)
+ ;Test to see if the next line read in is another VAR_NAME - if not then bi[0] will equal -1
+ bi=WHERE(dum EQ STRUPCASE(vn))
+ IF bi[0] EQ vc THEN BEGIN ;Same VAR_NAME found twice in the data file
+ STOP_WITH_ERROR,o3[3]+proname+vn[vc]+': ',errtxt[5],lu & RETURN
+ ENDIF
+ IF (bi[0] EQ -1) AND (dum NE '!') THEN bi=WHERE(dum EQ STRUPCASE(vnchange))
+ ;Test to check if the next line read in is another data point - if so bcnt will be set to 1
+ IF (STRMID(dum,0,1) NE '!') AND (STRMID(dum,0,1) NE '#') AND $
+ ((dum NE '') OR ((sdata) AND (dum EQ ''))) AND (bi[0] EQ -1) THEN $
+ bcnt=1
+ IF bcnt NE 0 THEN BEGIN ;not enough data lines read in or next line in the file is invalid
+ IF vcnt NE 0 THEN STOP_WITH_ERROR,o3[3]+proname+'VAR_NAME='+vn[vc]+': ',errtxt[9],lu $
+ ELSE $
+ STOP_WITH_ERROR,o3[3]+proname+'VAR_NAME='+vn[vc]+': ',errtxt[6]+errtxt[3]+STRTRIM(ndl,2),lu
+ RETURN
+ ENDIF
+ ENDELSE
+ FREE_LUN,lu
+
+ TypeConversionError:
+ IF valid LE 0 THEN BEGIN ;EOF encountered but not expected
+ IF (dtest[0] EQ '-1') AND (valid EQ -1) THEN $
+ STOP_WITH_ERROR,o3[3]+proname,errtxt[0]+errtxt[2]+vn[vc],lu $
+ ELSE BEGIN
+ ;Check to see if it is due to a VAR_SIZE calculation error or not
+ IF vcnt NE 0 THEN STOP_WITH_ERROR,o3[3]+proname+'VAR_NAME='+vn[vc]+': ',errtxt[9],lu $
+ ELSE $
+ STOP_WITH_ERROR,o3[3]+proname+'VAR_NAME='+vn[vc]+': ',errtxt[0]+errtxt[3]+STRTRIM(ndl,2),lu
+ ENDELSE
+ RETURN
+ ENDIF
+ENDIF ELSE BEGIN ;sds is a structure
+ IF dtest[0] EQ '-1' THEN BEGIN ;determine the total number of data lines only
+ dtest=*sds[vc].data
+ ndl=LONG(N_ELEMENTS(dtest))
+ dtest=['-1']
+ ENDIF ELSE BEGIN ;return dtest and ndl
+ lu=-1
+ di=WHERE(ndm[vc,0:7] NE 0L,ndim)
+ ;multiply together the array dimensions to determine expected total number of datalines
+ FOR i=0,ndim-1 DO ndl=ndl*ndm[vc,di[i]]
+ dtest=*sds[vc].data
+ ;check structure contains the correct number of elements
+ IF N_ELEMENTS(dtest) NE ndl THEN BEGIN
+ STOP_WITH_ERROR,o3[3]+proname+vn[vc]+': ',errtxt[7]+STRTRIM(N_ELEMENTS(dtest),2)+'/'+STRTRIM(ndl,2),lu
+ RETURN
+ ENDIF
+ stest=SIZE(dtest) ;test to see if dtest is an array or scalar
+ IF stest[0] EQ 0 THEN dtest=[dtest] ELSE dtest=REFORM(dtest,ndl,/Overwrite) ;convert to a 1-D array if necessary
+ ENDELSE
+ENDELSE
+
+;Note: this conditional will only be satisfied if the routine is called by READ_DATA
+;(vu still not assigned VAR_UNITS value if called by SET_UP_STRUCTURE)
+IF STRUPCASE(vu[vc]) EQ 'MJD2K' THEN BEGIN
+ ;Convert time from ISO8601 to MJD2000 format if necessary
+ FOR i=0L,ndl-1L DO BEGIN
+ IF STRPOS(STRUPCASE(dtest[i]),'Z') NE -1 THEN BEGIN
+ mjd2000=JULIAN_DATE(dtest[i],/I,/M) ;return time in MJD2000 format
+ IF mjd2000 EQ -99999.d THEN BEGIN ;conversion error
+ STOP_WITH_ERROR,o3[3]+proname+vn[vc]+': ',errtxt[8]+dtest[i],lu & RETURN
+ ENDIF
+ dtest[i]=STRING(format='(d18.9)',mjd2000)
+ IF infowrite EQ 1 THEN BEGIN
+ IF qa_yes THEN itxt='cannot be in ISO8601 format' ELSE itxt='changed from ISO8601 to MJD2K format'
+ infotxt='2 Dataset values '+itxt+' for VAR_NAME='+vn[vc]
+ INFOTXT_OUTPUT,infotxt
+ infowrite=0
+ ENDIF
+ ENDIF
+ ENDFOR
+ENDIF
+
+END ;Procedure Extract_Data
+
+
+
+PRO set_up_structure, sds, inf
+;Procedure to do additional checks on VAR_NAME, VAR_DEPEND, VAR_DATA_TYPE, VAR_UNITS, and
+;VAR_FILL_VALUE metadata attributes and set up a structure which will be used to hold the
+;data values.
+; ----------
+;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
+;
+; History:
+; 20050802: Introduced to IDLCR8HDF, as Determine_Arr_Sizes. Data originally stored in 4
+; arrays according to the data type of the variable attributes - Version 1.0
+; 20050909: Additional checks performed on the metadata attributes as follows: that the
+; VAR_DEPEND variable name(s), themselves, have a VAR_DEPEND value of CONSTANT or
+; INDEPENDENT, and matching VAR_SIZE values; that VAR_DATA_TYPE is DOUBLE for
+; attributes with VAR_UNITS=MJD2000; that VAR_UNITS=MJD2000 when VAR_NAME=DATETIME
+; - Version 1.1
+; 20061012: Procedure renamed Set_Up_Structure. A structure using pointers used to store the
+; data instead of arrays. VAR_FILL_VALUE value saved in common array and, if
+; VAR_UNITS=MJD2000, VAR_FILL_VALUE converted to its MJD2000 form if it is in
+; ISO8601 format. Common variable definition WIDGET_WIN added - Version 2.0
+; 20080302: Can calculate the VAR_SIZE value(s) from the input data or from the VAR_SIZE
+; value(s) of the VAR_DEPEND variable name(s), if they are not present in the
+; metadata. Section added to account for the new variables, ALTITUDE/ALTITUDE.GPH/
+; PRESSURE.BOUNDARIES; Additional checks added for VIS_PLOT_TYPE, VIS_SCALE_TYPE,
+; VIS_SCALE_MIN, and VIS_SCALE_MAX values - Version 3.0
+; 20100205: Add RETURN command after all STOP_WITH_ERROR calls, which allows program to return to the
+; calling program if the reterr argument is included in the idlcr8hdf call - Version 3.09
+; 20101120: Account for GEOMS changes between v3.0 and v4.0; Remove checks on VAR_DIMENSION, and the
+; VIS attributes; Add new VAR_DATA_TYPEs (Byte and Short); Check for self-referencing
+; VAR_DEPEND values (which replace INDEPENDENT in that context); Remove requirement for
+; DATETIME to be of datatype DOUBLE - Version 4.0
+; 20140521: Account for new Dimension Ordering rules for *.BOUNDARIES variables - should be
+; INDEPENDENT;*[;DATETIME]. In the event that the input metadata specifies a data template,
+; then the template VAR_DEPEND ordering will apply (and be checked by the template checker),
+; otherwise full checks will be made - Version 4.0b21
+; 20140806: Allow VAR_UNITS=d/D to be changed to MJD2K for DATETIME checks - Version 4.0b22
+; 20150811: Bug fix - initialize the infotxt array if reporting an ISO8601 to MJD2K conversion
+; issue - Version 4.0b29
+; 20151109: Check that VAR_SIZE values are in INTEGER format, even though they are
+; written to the HDF file as a string - Version 4.0b34
+; 20161116: Bug fix - resvd array value set correctly when checking whether a dataset with
+; VAR_DEPEND=CONSTANT only has a single value (was 0 should have been 1) - Version 4.0b39
+; 20161130: Allow program to continue when VAR_UNITS is not set to MJD2K for DATETIME
+; and related datasets (previously called STOP_WITH_ERROR in some conditions -
+; Version 4.0b40
+; 20180218: Change Unexpected number of attribute values extracted error message so that it calls
+; INFOTXT_OUTPUT rather than STOP_WITH_ERROR - Version 4.0b45
+; 20210617: Add section to do VAR_DATA_TYPE checks on the datasets, VAR_VALID_MIN, VAR_VALID_MAX and
+; VAR_FILL_VALUE values, to ensure they are all matching - Version 4.0b59
+; 20231218: Change check that an axis variable must be listed in DATA_VARIABLES before any variable
+; that depends on it. The new check is that the axis variable is present anywhere in the file.
+; Text for errtxt[3] changed, code added to identify axis variables and determine their
+; VAR_SIZE - Version 4.0b61
+; 20240112: Remove test that INDEPENDENT can only be a second dimension variable if the variable includes
+; .BOUNDARIES in the name e.g. ALTITUDE.BOUNDARIES. This is too limiting e.g. for Aerosol Lidar
+; RANGE_INDEPENDENT_NORMALIZATION has VAR_DEPEND=INDEPENDENT;DATETIME - Version 4.0b62
+;
+; Inputs: sds - Either a structure containing the Variable Attributes and Data, or the input
+; data file
+; inf - flag identifying SDS type where, 0=structure; 1=data file
+; meta_arr - a string array containing the Global and Variable Attributes
+;
+; Outputs: meta_arr - some meta_arr values may be corrected or calculated in this routine, including:
+; VAR_DIMENSION, VAR_SIZE, and VAR_FILL_VALUE
+; nvn - an integer giving the number of Variable Attributes present in the input
+; vn - a string array of size nvn containing the VAR_NAME values
+; vu - a string array of size nvn containing the VAR_UNIT values
+; ndm - a (nvn,9) long array, where ndm(*,0:7) describes the dimensions of each dataset
+; (maximum of 8 dimensions allowed), and ndm(*,8) describes the VAR_DATA_TYPE
+; ds - the structure that will hold the data
+;
+; Called by: IDLCR8HDF
+;
+; Subroutines Called: EXTRACT_DATA (if it is necessary to determine VAR_SIZE from the input data)
+; STOP_WITH_ERROR (if error state detected); INFOTXT_OUTOUT
+; Possible Conditions for STOP_WITH_ERROR call (plus [line number] where called):
+; 1. Program expects to find the VAR_NAME in the metadata in the same order as that listed under
+; DATA_VARIABLES
+; 2. The VAR_DIMENSION value is greater than 8
+; 3. Number of VAR_SIZE and VAR_DEPEND attribute values is different
+; 4. The VAR_DEPEND value is not 'CONSTANT', 'INDEPENDENT' or self-referencing, and doesn't
+; match a previous VAR_NAME
+; 5. The VAR_DEPEND variable name must, itself, have a VAR_DEPEND value of 'CONSTANT' or
+; 'INDEPENDENT' or be self-referencing
+; 6. The VAR_DEPEND variable name must have a VAR_SIZE value matching the current VAR_NAME
+; 7. VAR_DATA_TYPE not BYTE, SHORT, INTEGER, LONG, REAL, DOUBLE, or STRING
+; 8. Unexpected number of ATTRIBUTE values extracted
+; 9. VAR_DATA_TYPE should be DOUBLE for VAR_UNITS=MJD2K - No longer a requirement
+; 10. VAR_UNITS should be MJD2K for VAR_NAME=DATETIME
+; 11. VAR_FILL_VALUE not a valid value - occurs on an unsuccessful call to JULIAN_DATE
+; 12. VAR_DEPEND values not valid - occurs when INDEPENDENT or CONSTANT is a VAR_DEPEND value
+; when it should not be
+; 13. First VAR_SIZE value is not '2' when VAR_NAME=ALTITUDE/ALTITUDE.GPH/PRESSURE.BOUNDARIES
+; 14. Axis variable cannot be dependent on another variable
+; 15. Axis variable in the vertical dimension already found
+; 16. Attribute which is not an axis variable has a self-referencing VAR_DEPEND value
+; 17. Type conversion error. Called when trying to convert a string, which is expected to be
+; numeric, to a number
+;
+; Information Conditions (when the program is able to make changes):
+; 1. 64-bit LONG Data Type is not currently supported in GEOMS. Will attempt to write data in 32-bit
+; Integer type instead
+; 2. Dataset/VAR_VALID_MIN/VAR_VALID_MAX/VAR_FILL_VALUE value(s) have data type that does not match
+; VAR_DATA_TYPE
+
+COMMON METADATA
+COMMON DATA
+COMMON WIDGET_WIN
+
+;possible error messages for this procedure
+errtxt=STRARR(17) & lu=-1L
+errtxt[0]='Does not match the equivalent VAR_NAME in DATA_VARIABLES: '
+errtxt[1]='Number of dimensions too high to make HDF file (max. allowed 8)'
+errtxt[2]='VAR_DEPEND attribute value missing or does not match the number of VAR_SIZE values'
+errtxt[3]='VAR_DEPEND value not CONSTANT,INDEPENDENT nor self-referencing, and doesn''t match any VAR_NAME: '
+errtxt[4]='VAR_DEPEND variable name(s) must have VAR_DEPEND value of CONSTANT or be self_referencing: '
+errtxt[5]='VAR_DEPEND variable name must have matching VAR_SIZE value (= '
+errtxt[6]='Data type not BYTE, SHORT, INTEGER, REAL, DOUBLE, or STRING: '
+errtxt[7]='Unexpected number of ATTRIBUTE values extracted: '
+errtxt[8]='VAR_DATA_TYPE must be DOUBLE for VAR_UNITS=MJD2K: '
+errtxt[9]='VAR_UNITS must be MJD2K for ' ;no longer used (infotxt message instead)
+errtxt[10]='VAR_FILL_VALUE not a valid value: '
+errtxt[11]=' not valid'
+errtxt[12]='Second VAR_SIZE value should be ''2'' for this VAR_NAME'
+errtxt[13]='Axis variable cannot be dependent on another variable: '
+errtxt[14]=' is already the axis variable in the vertical dimension'
+errtxt[15]='is not an axis variable so VAR_DEPEND value cannot be self-referencing'
+errtxt[16]='Type conversion error. Entry is not a valid number: '
+proname='Set_Up_Structure procedure: '
+
+lc=0 & vc=0 & writeonce=0 & bounderror=0
+allowable_data_types=['BYTE','SHORT','INTEGER','LONG64','REAL','DOUBLE','STRING']
+idl_data_type=[1,2,3,14,4,5,7] ;IDL type identifiers matching allowable_data_types
+num_atts=['VAR_VALID_MIN','VAR_VALID_MAX','VAR_FILL_VALUE']
+nna=N_ELEMENTS(num_atts)
+
+ON_IOERROR,TypeConversionError
+
+;Do a search for a DATA_TEMPLATE value. If not present will do all .BOUNDARIES checks, otherwise
+;these checks will be done by the Template QA program - interim until .BOUNDARIES dimension ordering
+;is consistent for all templates (FTIR retains original convention as of 20140521)
+dtfound=0 ;default to template not present
+dti=WHERE(STRMID(meta_arr,0,13) EQ 'DATA_TEMPLATE',dtcnt)
+IF dtcnt NE 0 THEN BEGIN ;check there is a value
+ res=STRSPLIT(meta_arr[dti[0]],' =;',/Extract,COUNT=rescnt)
+ IF rescnt EQ 2 THEN dtfound=1 ;valid DATA_TEMPLATE value found
+ENDIF
+
+;Determine a set of possible Axis Variables to be tested (i.e. all VAR_DEPEND values that aren't CONSTANT or INDEPENDENT)
+vdi=WHERE(STRMID(meta_arr,0,10) EQ 'VAR_DEPEND',vdcnt)
+vardeptest=[''] & vardepndim=INTARR(vdcnt) & vardepvals=STRARR(8,vdcnt)
+IF vdcnt NE 0 THEN BEGIN
+ FOR i=0,vdcnt-1 DO BEGIN
+ res=STRSPLIT(meta_arr[vdi[i]],' =;',/Extract,COUNT=rescnt)
+ vardepndim[i]=rescnt-1
+ IF rescnt GT 1 THEN BEGIN ;VAR_DEPEND values found
+ FOR j=1,rescnt-1 DO BEGIN
+ resuc=STRUPCASE(res[j]) & vardepvals[j-1,i]=res[j]
+ ni=WHERE(resuc EQ STRUPCASE(vardeptest),ncnt)
+ IF (ncnt EQ 0) AND (resuc NE 'INDEPENDENT') AND (resuc NE 'CONSTANT') THEN $
+ vardeptest=[vardeptest,res[j]] ;Add VAR_DEPEND value to list of axis-variables to be checked
+ ENDFOR
+ ENDIF
+ ENDFOR
+ENDIF
+;Remove '' from vardeptest
+gi=WHERE(vardeptest NE '',n_axv)
+IF n_axv NE 0 THEN BEGIN
+ vardeptest=vardeptest[gi] & vardepndl=lonarr(n_axv)
+ vni=WHERE(STRMID(meta_arr,0,8) EQ 'VAR_NAME',vncnt)
+ ndm=LONARR(vncnt,9) & vu=STRARR(vncnt) ;just needed, at thu stage as dummy inputs to EXTRACT_DATA
+ IF vncnt NE 0 THEN BEGIN
+ vn=STRARR(vncnt) & vserror=vn ;vserror needed for dummy input to EXTRACT_DATA
+ FOR i=0,vncnt-1 DO BEGIN
+ res=STRSPLIT(meta_arr[vni[i]],' =',/Extract,COUNT=rescnt)
+ IF rescnt EQ 2 THEN vn[i]=res[1]
+ ENDFOR
+ multidim=0B
+ FOR i=0,n_axv-1 DO BEGIN
+ vcx=WHERE(vardeptest[i] EQ vn,vcxcnt)
+ IF vcxcnt NE 0 THEN ndimx=vardepndim[vcx[0]] ELSE ndimx=0
+ IF (vcxcnt NE 0) AND (ndimx LE 1) THEN BEGIN
+ dtest=['-1'] ;-1 identifies to the EXTRACT_DATA routine that ndl is all that is required
+ EXTRACT_DATA,SDS,inf,vcx,dtest,ndl
+ vardepndl(i)=ndl
+ ENDIF ELSE IF ndimx GT 1 THEN multidim=1B ;axis variable has more than one dependency e.g. VAR_DEPEND=DATETIME;ALTITUDE for VAR_NAME=ALTITUDE
+ ENDFOR
+ IF multidim THEN BEGIN
+ FOR i=0,n_axv-1 DO BEGIN
+ vcx=WHERE(vardeptest[i] EQ vn,vcxcnt)
+ IF vcxcnt NE 0 THEN ndimx=vardepndim[vcx[0]] ELSE ndimx=0
+ IF (vcxcnt NE 0) AND (ndimx GT 1) THEN BEGIN
+ dtest=['-1'] ;-1 identifies to the EXTRACT_DATA routine that ndl is all that is required
+ EXTRACT_DATA,SDS,inf,vcx,dtest,ndl
+ ;Determine actual number of dataset values
+ FOR j=0,ndimx-1 DO BEGIN
+ IF vardepvals[j,vcx[0]] NE vn[vcx[0]] THEN BEGIN
+ di=WHERE(vardepvals[j,vcx[0]] EQ vardeptest,dcnt)
+ IF dcnt NE 0 THEN ndl=ndl/vardepndl[di[0]]
+ ENDIF
+ ENDFOR
+ vardepndl(i)=ndl
+ ENDIF
+ ENDFOR
+ ENDIF
+ ENDIF
+ENDIF
+
+;At this point have got an array of size n_axv containing the axis variable names (vardeptest) and their VAR_SIZE (vardepndl)
+
+notlong=0B ;Boolean to check whether VAR_SIZE values are of integer data type
+WHILE lc LT N_ELEMENTS(meta_arr) DO BEGIN
+ valid=0 ;set to test for successful string conversion to a number
+ IF STRMID(meta_arr[lc],0,14) EQ 'DATA_VARIABLES' THEN BEGIN ;determine number of variables reported
+ lcdv=lc
+ vn=STRSPLIT(meta_arr[lc],' =;',/Extract) ;string containing reported variables
+ nvn=N_ELEMENTS(vn)-1 ;total number of variables reported
+ vn=vn[1:nvn] ;remove the DATA_VARIABLES label
+ vu=STRARR(nvn) ;holding the VAR_UNITS values
+ vfvd=DBLARR(nna,nvn) ;holding array for floating point num_atts array values
+ vfvl=LON64ARR(nna,nvn) ;holding array for integer num_atts array values
+ ndm=LONARR(nvn,9) ;to contain the dimensions and data type for each dataset
+ vserror=STRARR(nvn) ;to hold variable names for datasets where the VAR_SIZE values cannot be determined
+ ENDIF
+
+ IF STRMID(meta_arr[lc],0,8) EQ 'VAR_NAME' THEN BEGIN
+ vname=meta_arr[lc] ;used in case of error
+ lcz=lc ;VAR_NAME index value
+ res=STRSPLIT(meta_arr[lc],' =',/Extract)
+ IF N_ELEMENTS(res) NE 2 THEN res=[res[0],'-1']
+ IF res[1] NE vn[vc] THEN BEGIN
+ STOP_WITH_ERROR,o3[3]+proname+vname+': ',errtxt[0]+vn[vc],lu & RETURN
+ ;Does not match the equivalent VAR_NAME in DATA_VARIABLES
+ ENDIF
+
+ REPEAT lc=lc+1 UNTIL STRMID(meta_arr[lc],0,8) EQ 'VAR_SIZE'
+ holdvs=meta_arr[lc] & lcx=lc ;do checks after determining VAR_DATA_TYPE
+ REPEAT lc=lc+1 UNTIL STRMID(meta_arr[lc],0,10) EQ 'VAR_DEPEND'
+ holdvd=meta_arr[lc] & lcy=lc ;do checks after determining VAR_DATA_TYPE
+ REPEAT lc=lc+1 UNTIL STRMID(meta_arr[lc],0,13) EQ 'VAR_DATA_TYPE'
+ res=STRSPLIT(STRUPCASE(meta_arr[lc]),' =;',/Extract)
+ IF N_ELEMENTS(res) EQ 2 THEN gi=WHERE(res[1] EQ allowable_data_types,gcnt) $ ;Check VAR_DATA_TYPE is valid
+ ELSE gcnt=0
+ IF gcnt NE 1 THEN BEGIN
+ STOP_WITH_ERROR,o3[3]+proname+vname+': ',errtxt[6]+meta_arr[lc],lu & RETURN
+ ;Data type not BYTE, SHORT, INTEGER, LONG64, REAL, DOUBLE, or STRING
+ ENDIF
+ IF gi[0] EQ 3 THEN BEGIN
+ IF writeonce EQ 0 THEN BEGIN
+ writeonce=1
+ infotxt='2 64-bit LONG Data Type is not currently supported|. '
+ infotxt=infotxt+'Will attempt to write data in 32-bit INTEGER Data Type'
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ gi[0]=2 & meta_arr[lc]='VAR_DATA_TYPE=INTEGER'
+ ENDIF
+ ndm[vc,8]=gi[0] ;index number representing the data type
+
+ ;Do VAR_SIZE and VAR_DEPEND checks
+ resvs=STRSPLIT(holdvs,' =;',/Extract,COUNT=nvs) ;VAR_SIZE entries
+ resvd=STRSPLIT(holdvd,' =;',/Extract,COUNT=nd) ;VAR_DEPEND entries
+ nvs=nvs-1 & nd=nd-1 ;Number of Dimensions based on number of VAR_SIZE/VAR_DEPEND values
+ vsadded=0
+
+ IF (nvs EQ 0) AND (nd NE 0) THEN BEGIN
+ ;Determine resvs values from vardeptest and vardepndl
+ FOR k=1,nd DO BEGIN
+ mi=WHERE(resvd[k] EQ vardeptest,mcnt)
+ IF mcnt NE 0 THEN BEGIN
+ resvs=[resvs,strtrim(vardepndl[mi[0]],2)]
+ nvs++
+ IF k NE nd THEN sctxt=';' ELSE sctxt=''
+ holdvs=holdvs+strtrim(vardepndl[mi[0]],2)+sctxt
+ vsadded=1
+ ENDIF
+ ENDFOR
+ meta_arr[lcx]=holdvs
+ ENDIF
+ IF nvs EQ 0 THEN ndmok=0 ELSE ndmok=1
+ ;If there are no VAR_SIZE value entries, need to determine from Data file or structure or
+ ;from the VAR_SIZE values of the VAR_DEPEND dependencies
+ IF ((nvs NE nd) AND (ndmok EQ 1)) OR (nd EQ 0) THEN BEGIN
+ STOP_WITH_ERROR,o3[3]+proname+vname+': ',errtxt[2],lu & RETURN
+ ;VAR_DEPEND attribute value missing or does not match the number of VAR_SIZE values
+ ENDIF ELSE IF nd GT 8 THEN BEGIN
+ STOP_WITH_ERROR,o3[3]+proname+vname+': ',errtxt[1],lu & RETURN
+ ;Number of dimensions too high to make HDF file (max. allowed 8)
+ ENDIF ELSE IF ndmok EQ 1 THEN BEGIN
+ ndm[vc,0:nd-1]=LONG(resvs[1:nvs]) ;save VAR_SIZE values in ndm array
+
+ ;Check that original input values are of integer data type
+ FOR k=1,nvs DO BEGIN
+ IF STRTRIM(resvs[k],2) NE STRTRIM(LONG(resvs[k]),2) THEN BEGIN
+ resvs[k]=STRTRIM(LONG(resvs[k]),2)
+ notlong=1B
+ ENDIF
+ ENDFOR
+ IF notlong THEN BEGIN
+ FOR k=1,nvs DO IF k EQ 1 THEN vshold=resvs[k] ELSE vshold=vshold+';'+resvs[k]
+ meta_arr[lcx]='VAR_SIZE='+vshold
+ holdvs=meta_arr[lcx]
+ ENDIF
+ ENDIF
+
+ vshold=LONARR(nd) ;to hold VAR_SIZE of the dependencies
+
+ ;Do checks on VAR_DEPEND values
+ ci=WHERE((STRUPCASE(resvd) EQ 'INDEPENDENT') OR (STRUPCASE(resvd) EQ 'CONSTANT'),ccnt)
+ IF ccnt NE 0 THEN resvd[ci]=STRUPCASE(resvd[ci])
+ ;Rules for .BOUNDARY VAR_NAME
+ IF STRPOS(vn[vc],'.BOUNDARIES') NE -1 THEN BEGIN
+ test1=0 & test2=0
+ IF ((nd EQ 2) OR (nd EQ 3)) AND (dtfound EQ 0) THEN BEGIN
+ ;First VAR_DEPEND value must be INDEPENDENT
+ test1=resvd[1] NE 'INDEPENDENT'
+ ;Second VAR_DEPEND value must be present in the Variable Name e.g. ALTITUDE for ALTITUDE.BOUNDARIES
+ secvdsplt=STRSPLIT(resvd[2],'.',/Extract)
+ test2=STRPOS(vn[vc],secvdsplt[0]) EQ -1
+ ENDIF ELSE IF (nd LE 1) OR (nd GE 4) THEN test2=1 ;incorrect number of VAR_DEPEND values
+ IF nd EQ 3 THEN IF resvd[3] NE 'DATETIME' THEN test1=1 ;Third VAR_DEPEND value must be DATETIME
+ IF (test1) OR (test2) THEN BEGIN
+ IF test1 EQ 1 THEN itxt=' in IDL/Fortran Dimension Ordering' ELSE itxt=''
+ infotxt='3 '+holdvd+' not valid for '+vname+itxt
+ INFOTXT_OUTPUT,infotxt
+ bounderror=1
+ ;STOP_WITH_ERROR,o3[3]+proname+vname+': ',holdvd+errtxt[11]+' for the given VAR_NAME'+itxt,lu
+ ;VAR_DEPEND value(s) not valid for the given VAR_NAME
+ ;RETURN
+ ENDIF
+ ENDIF
+
+ FOR i=1,nd DO BEGIN
+ IF i EQ 1 THEN BEGIN
+ ;Check for and Apply GEOMS rule changes for INDEPENDENT and axis variables
+ vdval=resvd[i] & vnval=vn[vc]
+ GEOMS_RULE_CHANGES,2,vardeptest,vnval,vdval,holdvd
+ IF vdval NE resvd[i] THEN BEGIN ;First VAR_DEPEND value has been changed
+ ;Rebuild VAR_DEPEND entry
+ holdvd='VAR_DEPEND='+vdval
+ IF nd GT 1 THEN FOR j=2,nd DO holdvd=holdvd+';'+resvd[j]
+ ENDIF
+ IF STRMID(holdvd,0,1) EQ 'E' THEN BEGIN ;error found while attempting to identify axis variables
+ CASE 1 OF
+ holdvd EQ 'E1': STOP_WITH_ERROR,o3[3]+proname+vname+': ',errtxt[13]+vdval,lu
+ holdvd EQ 'E3': STOP_WITH_ERROR,o3[3]+proname+vname+' ',errtxt[15],lu
+ ENDCASE
+ RETURN
+ ENDIF
+ resvd[i]=vdval
+ meta_arr[lcy]=holdvd
+ ENDIF
+ test1=(resvd[i] EQ 'CONSTANT') AND ((i GT 1) OR (nd GE 2)) ;CONSTANT can be the only VAR_DEPEND
+ test2=(resvd[i] EQ vn[vc]) AND ((i GT 1) OR (nd GT 2)) ;e.g. ALTITUDE;DATETIME are possible dependencies for ALTITUDE
+ test3=0B ;(resvd[i] EQ 'INDEPENDENT') AND ((i GT 1) OR (nd GE 2)) AND (STRPOS(vn[vc],'.BOUNDARIES') EQ -1) ;remove test as too specific
+ test4=(resvd[i] NE 'CONSTANT') AND (resvd[i] NE 'INDEPENDENT') AND (resvd[i] NE vn[vc])
+ ;Check for CONSTANT,INDEPENDENT or self-referencing and more than one VAR_DEPEND value
+ IF (test1) OR (test2) OR (test3) THEN BEGIN
+ STOP_WITH_ERROR,o3[3]+proname+vname+': ',holdvd+errtxt[11],lu & RETURN
+ ;VAR_DEPEND value(s) not valid
+ ENDIF
+ ;Check for valid dependencies, and hold VAR_SIZE values of the dependencies if required
+ ;Section modified 20231218 to allow axis variable to be after the variable that depends on it
+ CASE 1 OF
+ (vc EQ 0) AND (nvn GE 2): gi=WHERE(resvd[i] EQ vn[1:nvn-1],gcnt)
+ (vc EQ nvn-1) AND (nvn GE 2): gi=WHERE(resvd[i] EQ vn[0:nvn-2],gcnt)
+ nvn GE 3: BEGIN
+ gi=WHERE(resvd[i] EQ vn[0:vc-1],gcnt)
+ IF gcnt EQ 0 THEN BEGIN
+ gi=WHERE(resvd[i] EQ vn[vc+1:nvn-1],gcnt)
+ IF gcnt NE 0 THEN gi[0]=gi[0]+vc+1
+ ENDIF
+ END
+ ELSE: gcnt=0
+ ENDCASE
+ ;Original check
+ ;IF vc GE 1 THEN gi=WHERE(resvd[i] EQ vn[0:vc-1],gcnt) ELSE gcnt=0
+ IF (gcnt EQ 0) AND (test4) THEN BEGIN
+ STOP_WITH_ERROR,o3[3]+proname+vname+': ',errtxt[3]+resvd[i],lu & RETURN
+ ;VAR_DEPEND value not CONSTANT,INDEPENDENT nor Self-referencing, and doesn't match any VAR_NAME
+ ENDIF
+ IF gcnt NE 0 THEN BEGIN ;do remaining checks on VAR_DEPEND values
+ errfound=0
+ gi=WHERE(meta_arr EQ 'VAR_NAME='+resvd[i])
+ ;check VAR_DEPEND value of the VAR_DEPEND variable name
+ vi=WHERE(STRPOS(meta_arr[gi[0]:N_ELEMENTS(meta_arr)-1],'VAR_DEPEND') NE -1)
+ vex=STRSPLIT(STRUPCASE(meta_arr[gi[0]+vi[0]]),' =;',/Extract,COUNT=nvex)
+ IF nvex NE 2 THEN vex=[vex,''] ;missing VAR_DEPEND value so add dummy value
+ IF (vex[1] NE 'CONSTANT') AND (vex[1] NE 'INDEPENDENT') AND (vex[1] NE resvd[i]) THEN BEGIN
+ vert_var=['ALTITUDE','ALTITUDE.GPH','PRESSURE','DEPTH']
+ res=WHERE(resvd[i] EQ vert_var,vcnt) ;does VAR_DEPEND variable name reference a vertical axis
+ res=WHERE(vex[1] EQ vert_var,xcnt) ;does VAR_DEPEND of the VAR_DEPEND variable name reference a vertical axis
+ IF (vcnt NE 0) AND (xcnt EQ 0) THEN BEGIN ;Vertical axis so should be self-referencing
+ infotxt='2 '+meta_arr[gi[0]+vi[0]]+' should be self-referencing for axis variable'
+ infotxt=infotxt+' VAR_NAME='+resvd[i]+'|. VAR_DEPEND value changed in metadata'
+ INFOTXT_OUTPUT,infotxt
+ meta_arr[gi[0]+vi[0]]='VAR_DEPEND='+resvd[i]
+ IF nvex GT 2 THEN FOR j=2,nvex-1 DO meta_arr[gi[0]+vi[0]]=meta_arr[gi[0]+vi[0]]+';'+vex[j]
+ ENDIF ELSE BEGIN
+ ;VAR_DEPEND value of the VAR_DEPEND variable name must be CONSTANT or an axis variable
+ infotxt=STRARR(2)
+ infotxt[0]='3 '+meta_arr[lcy]+' variable name(s) for '+vname+' must have VAR_DEPEND'
+ infotxt[1]=' value of CONSTANT or be self-referencing'
+ INFOTXT_OUTPUT,infotxt
+ ENDELSE
+ ENDIF
+ IF vsadded EQ 0 THEN BEGIN
+ ;check VAR_SIZE value of the VAR_DEPEND variable name
+ vi=WHERE(STRPOS(meta_arr[gi[0]:N_ELEMENTS(meta_arr)-1],'VAR_SIZE') NE -1)
+ vex=STRSPLIT(meta_arr[gi[0]+vi[0]],' =;',/Extract,COUNT=nvex)
+
+ IF (nvex LT 2) OR (nvex GT 3) THEN errfound=1 $
+ ELSE IF (ndmok EQ 1) AND (LONG(vex[1]) NE ndm[vc,i-1]) THEN errfound=1
+
+ IF errfound EQ 1 THEN BEGIN
+ STOP_WITH_ERROR,o3[3]+proname+vname+': '+'VAR_DEPEND='+resvd[i]+': ', $
+ errtxt[5]+STRTRIM(ndm[vc,i-1],2)+'): '+meta_arr[gi[0]+vi[0]],lu
+ ;VAR_DEPEND variable name must have matching VAR_SIZE value
+ RETURN
+ ENDIF ELSE vshold[i-1]=LONG(vex[1])
+ ENDIF
+ ENDIF
+ ;If i eq 2 and value is INDEPENDENT then check that the corresponding VAR_SIZE value is 2
+ ;(if present), o/w generate errors
+ IF (i EQ 2) AND (STRUPCASE(resvd[i]) EQ 'INDEPENDENT') THEN BEGIN
+ IF (ndmok EQ 1) AND (ndm[vc,i-1] NE 2L) THEN BEGIN
+ STOP_WITH_ERROR,o3[3]+proname+vname+': '+holdvs+': ',errtxt[12],lu & RETURN
+ ;Second VAR_SIZE value should be '2' for this VAR_NAME
+ ENDIF
+ vshold[i-1]=2L ;set VAR_SIZE value in the event that it is not part of the Metadata
+ ENDIF
+ ENDFOR
+
+ IF (ndmok EQ 0) OR (vsadded EQ 1) THEN BEGIN ;no VAR_SIZE value so need to determine from the data or the vshold array
+ IF (vshold[0] EQ 0L) AND (vsadded EQ 0) THEN BEGIN
+ ;VAR_DEPEND is Self-Referencing or CONSTANT so determine VAR_SIZE from the Data file or structure
+ dtest=['-1'] ;-1 identifies to the EXTRACT_DATA routine that ndl is all that is required
+ EXTRACT_DATA,SDS,inf,vc,dtest,ndl
+ IF STRLEN(rerr[0]) GT 2 THEN RETURN
+ ;Check if any other VAR_SIZE values have already been determined
+ ;This could happen in the case VAR_DEPEND=ALTITUDE;DATETIME for VAR_NAME=ALTITUDE for e.g.
+ gi=WHERE(vshold NE 0L,gcnt)
+ ndltot=' ('+STRTRIM(ndl,2)+')'
+ IF gcnt NE 0 THEN BEGIN
+ FOR i=0,gcnt-1 DO ndl=ndl/FLOAT(vshold[gi[i]])
+ ENDIF
+ ;Make up VAR_SIZE string based on available values
+ IF vshold[0] EQ 0L THEN vsstr=' (VAR_SIZE=x' ELSE vsstr='VAR_SIZE='+STRTRIM(vshold[0],2)
+ IF nd GT 1 THEN FOR i=1,nd-1 DO $
+ IF vshold[i] EQ 0L THEN vsstr=vsstr+';x' ELSE vsstr=vsstr+';'+STRTRIM(vshold[i],2)
+ vsstr=vsstr+') '
+ vshold[0]=LONG(ndl)
+ bi=WHERE(vshold EQ 0,bcnt) ;Expected VAR_SIZE values are missing if bcnt not equal to 0
+ IF (ndl NE vshold[0]) OR (bcnt NE 0) THEN BEGIN
+ infotxt=STRARR(2)
+ infotxt[0]='3 Unable to determine missing VAR_SIZE value(s) for '+vname+' based on'
+ infotxt[1]=' '+holdvd+vsstr+'and the total number of dataset values'+ndltot
+ INFOTXT_OUTPUT,infotxt
+ vserror[vc]=vn[vc]
+ ENDIF
+ meta_arr[lcx]='VAR_SIZE='+STRTRIM(vshold[0],2)
+ ndm[vc,0:nd-1]=vshold
+ IF nd GT 1 THEN FOR i=1,nd-1 DO meta_arr[lcx]=meta_arr[lcx]+';'+STRTRIM(vshold[i],2)
+ ENDIF ELSE IF vsadded EQ 0 THEN BEGIN ;can determine VAR_SIZE from the vshold array
+ meta_arr[lcx]='VAR_SIZE='
+ FOR i=0,nd-1 DO BEGIN
+ IF i EQ 0 THEN meta_arr[lcx]=meta_arr[lcx]+STRTRIM(vshold[i],2) $
+ ELSE meta_arr[lcx]=meta_arr[lcx]+';'+STRTRIM(vshold[i],2)
+ ndm[vc,i]=vshold[i] ;EQ array dimensions from VAR_SIZE of the dependencies
+ ENDFOR
+ ENDIF
+ ;Check to see if VAR_SIZE calculation is valid or not based on contents of vserror
+ vsok=1
+ FOR i=1,nd DO BEGIN
+ ei=WHERE(resvd[i] EQ vserror,ecnt)
+ IF ecnt NE 0 THEN vsok=0
+ ENDFOR
+ IF (vsok EQ 1) OR (vsadded EQ 1) THEN BEGIN
+ IF qa_yes THEN qtxt=' not recorded. Calculated as '+meta_arr[lcx] ELSE qtxt=' added: '+meta_arr[lcx]
+ infotxt='2 VAR_SIZE value(s) for '+vname+qtxt
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ ENDIF
+
+ IF (resvd[1] EQ 'CONSTANT') AND (TOTAL(ndm[vc,0:7]) GT 1.) THEN BEGIN
+ infotxt='3 Dataset '+vn[vc]+' must only have a single value for VAR_DEPEND=CONSTANT: '+meta_arr[lcx]
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+
+ ;Do DATETIME/MJD2000 checks
+ REPEAT lc=lc+1 UNTIL STRMID(meta_arr[lc],0,9) EQ 'VAR_UNITS'
+ resv=STRSPLIT(meta_arr[lc],' =',/Extract,Count=rcnt)
+ IF rcnt EQ 1 THEN resv=[resv,'[MISSING]'] ;add filler for checks
+ test1=(vn[vc] EQ 'DATETIME') OR (vn[vc] EQ 'DATETIME.START') OR (vn[vc] EQ 'DATETIME.STOP')
+ IF (test1) AND (STRUPCASE(resv[1]) NE 'MJD2K') THEN BEGIN
+ lch=lc ;hold setting for VAR_UNITS
+ ;Find VAR_SI_CONVERSION values and test for correct values for MJD2K
+ REPEAT lch=lch+1 UNTIL STRMID(meta_arr[lch],0,17) EQ 'VAR_SI_CONVERSION'
+ res=STRSPLIT(meta_arr[lch],' =;',/EXTRACT)
+ test1=(STRLOWCASE(resv[1]) EQ 's') AND (FIX(res[1]) EQ 0) AND (LONG(res[2]) EQ 86400L) AND $
+ (STRLOWCASE(res[3]) EQ 's')
+ IF (resv[1] EQ '[MISSING]') OR (resv[1] EQ '1') OR (STRLOWCASE(resv[1]) EQ 'd') OR (test1) THEN BEGIN
+ errid='2 ' & itxt='|. Value changed in Metadata'
+ ENDIF ELSE BEGIN
+ errid='3 ' & itxt=''
+ ENDELSE
+ infotxt=errid+meta_arr[lc]+' must be VAR_UNITS=MJD2K for '+vname+itxt
+ INFOTXT_OUTPUT,infotxt
+ meta_arr[lc]='VAR_UNITS=MJD2K'
+ resv=['VAR_UNITS','MJD2K']
+ ;Change VAR_SI_CONVERSION value also
+ CASE 1 OF
+ ndm[vc,8] LE 3: meta_arr[lch]='VAR_SI_CONVERSION=0;86400;s
+ ndm[vc,8] LE 5: meta_arr[lch]='VAR_SI_CONVERSION=0.0;86400.0;s'
+ ELSE: meta_arr[lch]='VAR_SI_CONVERSION='
+ ENDCASE
+ ENDIF
+ vu[vc]=resv[1] ;used to check for MJD2K units when reading in data as well as NCSA units value
+ IF N_ELEMENTS(resv) GT 2 THEN $
+ FOR i=2,N_ELEMENTS(resv)-1 DO vu[vc]=vu[vc]+' '+resv[i]
+
+ ;Do checks on the numeric attributes
+ FOR i=0,nna-1 DO BEGIN
+ REPEAT lc=lc+1 UNTIL STRMID(meta_arr[lc],0,STRLEN(num_atts[i])) EQ num_atts[i]
+ resv=STRSPLIT(meta_arr[lc],' =;',/Extract,COUNT=nresv)
+ test1=(STRUPCASE(vu[vc]) NE 'MJD2K') OR (num_atts[i] EQ 'VAR_FILL_VALUE')
+ test2=(nresv NE 2) AND (ndm[vc,8] NE 6)
+ IF (test1) AND (test2) THEN BEGIN
+ infotxt=STRARR(2)
+ infotxt[0]='3 Unexpected number of ATTRIBUTE values extracted for '+vname+':'
+ infotxt[1]=' '+meta_arr[lc]
+ INFOTXT_OUTPUT,infotxt
+ resv=[resv,''] ;ensure at least 2 values
+ meta_arr[lc]=resv[0]+'='+resv[1] & nresv=2
+ ;STOP_WITH_ERROR,o3[3]+proname+vname+': ',errtxt[7]+meta_arr[lc],lu & RETURN
+ ;Unexpected number of ATTRIBUTE values extracted
+ ENDIF
+ IF (ndm[vc,8] NE 6) AND (nresv EQ 2) THEN BEGIN
+ ;If VAR_UNITS=MJD2000 then check for ISO8601 format and change to MJD2000 if necessary
+ IF (STRUPCASE(vu[vc]) EQ 'MJD2K') AND (STRPOS(STRUPCASE(resv[1]),'Z') NE -1) THEN BEGIN
+ mjd2000=JULIAN_DATE(resv[1],/I,/M) ;return time in MJD2000 format
+ IF mjd2000 EQ -99999.d THEN BEGIN
+ STOP_WITH_ERROR,o3[3]+proname+vn[vc]+': ',errtxt[10]+meta_arr[lc],lu & RETURN
+ ENDIF
+ vfvd[i,vc]=mjd2000
+ infotxt=STRARR(2)
+ meta_arr[lc]=resv[0]+'='+STRTRIM(STRING(format='(d18.9)',mjd2000),2)
+ IF qa_yes THEN qtxt=' must be in MJD2K format|' ELSE qtxt=' replaced with MJD2K formatted value '
+ infotxt[0]='2 '+resv[0]+'='+resv[1]+' for VAR_NAME='+vn[vc]
+ infotxt[1]=qtxt+meta_arr[lc]
+ INFOTXT_OUTPUT,infotxt
+ ENDIF ELSE BEGIN
+ ;Check that the VAR_VALID_MIN/MAX or VAR_FILL_VALUE is numeric and save to vfvl or vfvd arrays
+ lcx=lc
+ IF ndm[vc,8] LE 3 THEN BEGIN ;value is integer type
+ IF mv_lng[lc] NE 0LL THEN vfvl[i,vc]=mv_lng[lc] ELSE vfvl[i,vc]=LONG64(DOUBLE(resv[1]))
+ ENDIF ELSE BEGIN ;value is floating point type
+ IF mv_dbl[lc] NE 0.D THEN vfvd[i,vc]=mv_dbl[lc] ELSE vfvd[i,vc]=DOUBLE(resv[1])
+ ENDELSE
+ ENDELSE
+ ENDIF
+ ENDFOR
+ vc=vc+1
+ ENDIF
+ lc=lc+1 & valid=1
+ENDWHILE
+
+IF notlong THEN BEGIN ;write out INFORMATION or ERROR to log
+ IF qa_yes THEN itxt=' must be ' ELSE itxt=' changed to '
+ infotxt='2 VAR_SIZE values'+itxt+'integer format'
+ INFOTXT_OUTPUT,infotxt
+ENDIF
+
+;Save numeric metadata values to common arrays
+mv_lng=vfvl & mv_dbl=vfvd
+
+IF qa_yes THEN BEGIN
+ ON_IOERROR, NULL
+ ;Check VAR_VALID_MIN, VAR_VALID_MAX, VAR_FILL_VALUE and Dataset data types match VAR_DATA_TYPE
+ s_dims=SIZE(sds,/DIMENSIONS)
+ pchkd=PTR_VALID(sds.data)
+ pchkv=PTR_VALID(sds.va_v)
+ pchkl=PTR_VALID(sds.va_l)
+
+ IF s_dims[0] GE 1 THEN BEGIN
+ attnamechk=['VAR_NAME','VAR_DATA_TYPE','VAR_VALID_MIN','VAR_VALID_MAX','VAR_FILL_VALUE']
+ n_anc=N_ELEMENTS(attnamechk)
+ FOR i=0,s_dims[0]-1 DO BEGIN
+ vdtx=BYTARR(n_anc)+255B ;VAR_DATA_TYPE for the attnamechk values
+ vnh=''
+ FOR j=0,s_dims[1]-1 DO BEGIN
+ IF (pchkl[i,j]) AND (pchkv[i,j]) THEN BEGIN
+ res=*sds[i,j].va_l & res=STRTRIM(STRUPCASE(res),2)
+ ai=WHERE(res EQ attnamechk,acnt)
+ IF acnt NE 0 THEN BEGIN
+ attval=*sds[i,j].va_v
+ IF ai[0] EQ 0 THEN vnh=STRTRIM(STRUPCASE(attval),2) $
+ ELSE IF ai[0] EQ 1 THEN BEGIN
+ vdth=STRTRIM(STRUPCASE(attval),2)
+ vi=WHERE(vdth EQ allowable_data_types,vcnt)
+ IF vcnt NE 0 THEN vdtx[1]=idl_data_type[vi[0]]
+ ENDIF ELSE vdtx[ai[0]]=SIZE(*sds[i,j].va_v,/TYPE)
+ ENDIF
+ ENDIF
+ ENDFOR
+ IF pchkd[i,0] THEN dvdtx=SIZE(*sds[i,0].data,/TYPE) ELSE dvdtx=255B ;Data type of the dataset
+ ;PRINT,vnh,dvdtx,vdtx[1:4]
+ IF vdtx[1] NE 255B THEN BEGIN
+ IF (dvdtx NE vdtx[1]) AND (dvdtx NE 255B) THEN BEGIN
+ vi=WHERE(dvdtx EQ idl_data_type,vcnt)
+ IF vcnt NE 0 THEN svdt=allowable_data_types[vi[0]] ELSE svdt=''
+ IF svdt EQ '' THEN $
+ infotxt='3 '+vnh+' dataset has data type that does not match VAR_DATA_TYPE='+vdth $
+ ELSE $
+ infotxt='3 '+vnh+' dataset has data type '+svdt+', which does not match VAR_DATA_TYPE='+vdth
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ FOR j=2,n_anc-1 DO BEGIN
+ IF vdtx[j] NE vdtx[1] THEN BEGIN
+ vi=WHERE(vdtx[j] EQ idl_data_type,vcnt)
+ IF vcnt NE 0 THEN svdt=allowable_data_types[vi[0]] ELSE svdt=''
+ IF svdt EQ '' THEN $
+ infotxt='3 '+vnh+' '+attnamechk[j]+ $
+ ' value has data type that does not match VAR_DATA_TYPE='+vdth $
+ ELSE $
+ infotxt='3 '+vnh+' '+attnamechk[j]+' value has data type '+svdt+ $
+ ', which does not match VAR_DATA_TYPE='+vdth
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ ENDFOR
+ ENDIF
+ ENDFOR
+ ENDIF
+ENDIF
+
+;Define the HDF storage structure
+ds_set={data: PTR_NEW()} ;SDS data array
+ds=REPLICATE(ds_set,nvn) ;Dimension the structure to the size of the SDS datasets
+
+TypeConversionError:
+IF valid EQ 0 THEN BEGIN
+ STOP_WITH_ERROR,o3[3]+proname+vname+': ',errtxt[16]+meta_arr[lcx],lu & RETURN
+ENDIF
+END ;Procedure Set_Up_Structure
+
+
+
+PRO check_string_datatype, vc, dtest, ndl
+;Procedure to check that the data, and Variable Attributes are correctly configured when the
+;VAR_DATA_TYPE=STRING, including VAR_UNITS, VAR_VALID_MIN/MAX, and VAR_FILL_VALUE. The data
+;values are returned in the string datatype.
+; ----------
+;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
+;
+; History:
+; 20091203: Introduced to IDLCR8HDF routine - Version 3.08
+; 20101120: Account for GEOMS rule changes from v3.0 to v4.0 - Version 4.0b0
+; 20111220: Check dataset entries for non-ISO646-US ASCII Characters - Version 4.0b7
+; 20130116: Allow for a string dataset with maximum string length of 0, i.e. a set
+; of empty values - Version 4.0b15
+; 20150127: Use right padding to make strings the correct length (previously defaulted to
+; left padding); Save maximum string length of dataset in mv_str array
+; - Version 4.0b25
+; 20150217: VAR_FILL_VALUE reverts to being an empty space (made same length as the longest
+; dataset string in 4.0b25); improve checks on variable attribute values; Make
+; the first dataset entry equal to the maximum string length if writing to H5
+; due to apparent bug when writing datasets which means that the string
+; length is determined by the first entry (white space is removed when writing to
+; the H5 file) - Version 4.0b26
+; 20220805: Improve Information/Error message when attributes for string datasets are not
+; valid - Version 4.0b60
+;
+; Inputs: meta_arr - a string array containing the Global and Variable Attributes
+; vc - the index value of the vn array holding the VARIABLE_NAME being searched for in the data
+; file, or the index value of the sds structure which holds the data
+; dtest - the set of data values to be tested
+; ndl - The total number of data values i.e. the product of the VAR_SIZE values
+;
+; Outputs: meta_arr - in the event that the values being tested are changed, the array will be updated
+; dtest - correctly formatted dataset
+;
+; Called by: READ_DATA
+;
+; Subroutines Called: INFOTXT_OUTPUT
+; Information Conditions (when the program is able to make changes):
+; 1. The Metadata value is not valid for VAR_DATA_TYPE=STRING [2645]
+
+COMMON METADATA
+COMMON DATA
+COMMON WIDGET_WIN
+
+vi=WHERE(meta_arr EQ 'VAR_NAME='+vn[vc])
+dtestx=STRTRIM(dtest,0) ;remove trailing blanks only, for testing that string datasets are left justified
+dtest=STRTRIM(dtest,2)
+
+;Check for non ISO646-US ASCII Characters
+GEOMS_RULE_CHANGES,10,dtest,vn[vc]
+
+IF ~ARRAY_EQUAL(dtest,dtestx) THEN BEGIN ;format of the string dataset has been changed
+ IF qa_yes THEN $
+ infotxt='2 Dataset '+vn[vc]+' incorrectly formatted (text must be left-justified)' $
+ ELSE infotxt='2 Dataset '+vn[vc]+' reformatted so that text is left-justified'
+ INFOTXT_OUTPUT, infotxt
+ENDIF
+
+;Determine maximum string length of the dataset
+mxdatalen=0L ;default value of string length
+testdatalen=MAX(STRLEN(dtest))
+IF testdatalen GT mxdatalen THEN mxdatalen=testdatalen
+;Format the first data value to the maximum string length for H5 only (white space removed when writing to the file)
+IF (mxdatalen NE 0L) AND (STRPOS(o3[0],'H5') NE -1) THEN BEGIN
+ ;Done for H5 only because H5S_CREATE_SIMPLE function uses string length of the 1st entry to set length for all entries
+ vfsx='(A-'+STRTRIM(mxdatalen,2)+')' ;A- does left justified text instead of right justified
+ dtest[0]=STRING(format=vfsx,dtest[0])
+ENDIF
+
+label=['VAR_UNITS','VAR_SI_CONVERSION','VAR_VALID_MIN','VAR_VALID_MAX','VAR_FILL_VALUE']
+n_lab=N_ELEMENTS(label)
+mxlablen=MAX(STRLEN(label)) ;determine max label string length
+metahold=STRMID(meta_arr,0,mxlablen) ;subset of the meta_arr strings, containing first portion of the Metadata
+
+;Perform checks on the Attributes
+FOR i=0,n_lab-1 DO BEGIN
+ li=WHERE(STRPOS(metahold[vi[0]:N_ELEMENTS(metahold)-1],label[i]) NE -1)
+ lvi=li[0]+vi[0]
+ res=STRSPLIT(meta_arr[lvi],'=',/Extract,COUNT=nres)
+ IF nres EQ 1 THEN res=[res,'']
+ IF res[1] EQ ' ' THEN res[1]=''
+ IF (nres GT 2) OR (res[1] NE '') THEN BEGIN
+ ;print,'"'+res[1]+'"/"'+itxt+'"/'+strtrim(mxdatalen,2)
+ IF qa_yes THEN itxt='must be [' ELSE itxt='changed to ['
+ infotxt='2 '+vn[vc]+' '+meta_arr[lvi]+' attribute '+itxt+res[0]+'= ]'
+ INFOTXT_OUTPUT, infotxt
+ mv_str[vc]=mxdatalen
+ ENDIF
+ meta_arr[lvi]=res[0]+'= ' ;+string(ltxt)
+ IF res[0] EQ 'VAR_UNITS' THEN vu[vc]=' '
+ENDFOR
+
+END ;procedure Check_String_Datatype
+
+
+
+PRO check_min_max_fill, vc, dtest, ndl
+;Procedure to check that the data, VAR_VALID_MIN/MAX, and VAR_FILL_VALUE fit within the given data
+;type. That the VAR_FILL_VALUE falls within the GEOMS definition and that the data values are within
+;the valid minimum and maximum values (or the fill value). The VAR_VALID_MIN value is less than or
+;equal to the VAR_VALID_MAX value. The data values are returned in the data type given by VAR_DATA_TYPE.
+; ----------
+;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
+;
+; History:
+; 20050802: Original IDLCR8HDF routine - Version 1.0
+; 20050909: If VAR_NAME=DATETIME then checks that the first data value is not a fill value, and is
+; the lowest value; Change the methods for determining if a given data value falls within
+; the range allowed by integer or long data types; Add fix for the case where a dataset
+; consists of only fill values - Version 1.1
+; 20051107: Include VIS_SCALE_MIN/MAX values in the checks; Add fix to remove unwanted precision (to
+; prevent rounding errors), by converting values to formatted strings then back to numeric
+; values - Version 1.11
+; 20061012: Check that all DATETIME values are in chronological order (if more than one value), and
+; contain no fill values; ISO8601 DATETIME conversions to MJD2000 format (if necessary) moved
+; to the READ_DATA routine; Add additional checks when extracting the VIS_FORMAT values for
+; testing; Common variable definition WIDGET_WIN added - Version 2.0
+; 20080302: Fix bug so that unwanted precision from all data and relevant metadata values is removed
+; before doing QA checks, to avoid unnecessary error calls - Version 3.0
+; 20100205: Add RETURN command after all STOP_WITH_ERROR calls, which allows program to return to the
+; calling program if the reterr argument is included in the idlcr8hdf call - Version 3.09
+; 20101120: Account for GEOMS rule changes v3.0 to v4.0; Remove checks involving VIS_FORMAT; Modify
+; method for checking data type limitations - Version 4.0
+; 20120420: Fix CHECK_MATH check so that combined floating point errors are ignored for long integer
+; values being tested (error 160) - Version 4.0b9
+; 20120620: Remove CHECK_MATH checks, and just apply Type conversion checks, as it was not possible for
+; CHECK_MATH to accurately indicate the problem numeric value - Version 4.0b10
+; 20120829: Fix call to ROUND when the value being rounded is a string (ROUND can only be applied to
+; numeric datatypes) - Version 4.0b12
+; 20130327: Add additional check that VAR_VALID_MIN is less than or equal to the VAR_VALID_MAX in the
+; event that the dataset only contains fill values - Version 4.0b20
+; 20141110: Check for NaN in the dataset and change to the Fill Value - Version 4.0b24
+; 20170331: Add checks for VAR_VALID_MIN and MAX values for Latitude, Longitude, Zenith, Azimuth and
+; Wind direction datasets; check that the maximum dataset value for Azimuth datasets is
+; less than 360.0 degrees - Version 4.0b42
+; 20171130: Remove ds from STOP_WITH_ERROR parameters - Version 4.0b44
+; 20180901: Also check that DATETIME.START and DATETIME.STOP values are in chronological order -
+; Version 4.0b48
+; 20201211: Fix bug that caused an IDL Math Error (Floating Illegal Operand) when converting large
+; floating point values to integers - Version 4.0b58
+; 20220805: Allow program to continue if data type errors in the dataset and attributes are detected -
+; Version 4.0b60
+;
+; Inputs: meta_arr - a string array containing the Global and Variable Attributes
+; vc - the index value of the vn array holding the VARIABLE_NAME being searched for in the data
+; file, or the index value of the sds structure which holds the data
+; dtest - the set of data values to be tested
+; ndl - The total number of data values i.e. the product of the VAR_SIZE values
+;
+; Outputs: meta_arr - in the event that the VIS_FORMAT value string or the precision of the attribute
+; values being tested are changed, the array will be updated
+; dtest - dataset returned in correct data type (VAR_FILL_VALUE/VAR_VALID_MIN/VAR_VALID_MAX
+; also appended to the start of the array)
+;
+; Called by: READ_DATA
+;
+; Subroutines Called: STOP_WITH_ERROR (if error state detected)
+; INFOTXT_OUTPUT
+; Possible Conditions for STOP_WITH_ERROR call (plus [line number] where called):
+; 1. Value is not valid, or does not match VAR_DATA_TYPE [2826]
+; 2. A data value falls outside the range given by VAR_VALID_MIN or VAR_VALID_MAX [2846, 2851]
+; 3. The VAR_FILL_VALUE is not outside the valid minimum or maximum values - No longer a requirement
+; 4. The metadata or data value being tested is outside the data type range. For example,
+; if a data value is 40000, but VAR_DATA_TYPE=INTEGER (maximum allowable value of 32767)
+; [2786,2793,2800,2807,2813,2819]
+; 5. The DATETIME value cannot be a VAR_FILL_VALUE. Because DATETIME is an axis variable
+; it should not contain Fill Values [2862]
+; 6. The DATETIME values are not in chronological order [2868]
+; 7. Type conversion error. The program cannot convert a data value to a valid number [2874]
+;
+; Information Conditions (when the program is able to make changes):
+; 1. Fill value falls within the data range and is defined as a Default value [2836]
+; 2. NaN value in the dataset replaced with the fill value
+
+COMMON METADATA
+COMMON DATA
+COMMON WIDGET_WIN
+
+;Error messages for this procedure
+ON_IOERROR,TypeConversionError
+proname='Check_Min_Max_Fill procedure: '
+errtxt2=STRARR(7) & lu=-1
+errtxt2[0]=' not valid, or does not match VAR_DATA_TYPE'
+errtxt2[1]=' data value outside VAR_VALID_'
+errtxt2[2]='Fill data value not outside VAR_VALID_MIN or VAR_VALID_MAX: '
+errtxt2[3]=' outside the data type range: '
+errtxt2[4]='DATETIME value cannot be a VAR_FILL_VALUE. '
+errtxt2[5]='DATETIME values not in chronological order'
+errtxt2[6]='Type conversion error. Entry is not a valid value'
+
+vi=WHERE(meta_arr EQ 'VAR_NAME='+vn[vc])
+
+;Determine Var_Fill_Value, Var_Valid_Min, Var_Valid_Max, and data values,
+;and check that they fall within the range allowed by VAR_DATA_TYPE
+label=['VAR_VALID_MIN','VAR_VALID_MAX','VAR_FILL_VALUE','data value']
+n_lab=N_ELEMENTS(label)
+mxlablen=MAX(STRLEN(label[0:n_lab-2])) ;determine max label string length
+metahold=STRMID(meta_arr,0,mxlablen) ;subset of the meta_arr strings, containing first portion of the Metadata
+lvi=LONARR(n_lab-1) ;to hold index values of the VAR_FILL_VALUE/VALID_MIN/VALID_MAX attributes
+
+CASE ndm[vc,8] OF
+ 0: vhold=BYTARR(ndl+(n_lab-1L))
+ 1: vhold=INTARR(ndl+(n_lab-1L))
+ 2: vhold=LONARR(ndl+(n_lab-1L))
+ 3: vhold=LON64ARR(ndl+(n_lab-1L))
+ 4: vhold=FLTARR(ndl+(n_lab-1L))
+ 5: vhold=DBLARR(ndl+(n_lab-1L))
+ENDCASE
+
+IF SIZE(dtest,/TYPE) EQ 7 THEN convertstr=1 ELSE convertstr=0 ;dtest is in ASCII string format
+IF ndm[vc,8] LE 3 then convtolng=1 else convtolng=0
+
+nanfound=0 ;boolean setting so information/error message only written once
+FOR i=0L,ndl+(n_lab-2L) DO BEGIN
+ valid=0 ;set-up for possible Type Conversion Error
+ IF i LE n_lab-2L THEN BEGIN
+ lin=i ;label index
+ li=WHERE(STRPOS(metahold[vi[0]:N_ELEMENTS(metahold)-1],label[lin]) NE -1)
+ lvi[i]=li[0]+vi[0]
+ res=STRSPLIT(meta_arr[lvi[i]],'=',/Extract)
+ IF convtolng THEN lv=mv_lng[i,vc] ELSE lv=mv_dbl[i,vc]
+ tcerr=': '+meta_arr[lvi[i]] ;used for type conversion error
+ ENDIF ELSE BEGIN ;check individual data values
+ res=['',''] & lin=n_lab-1L ;label index
+ tcerr=' in the Data File: '+STRTRIM(dtest[i-lin],2) ;used for type conversion error
+ IF convertstr THEN BEGIN
+ IF convtolng THEN lv=ROUND(DOUBLE(dtest[i-lin]),/L64) ELSE lv=DOUBLE(dtest[i-lin]) ;LONG64(DOUBLE(dtest[i-lin])) ELSE lv=DOUBLE(dtest[i-lin])
+ ENDIF ELSE lv=dtest[i-lin]
+ ENDELSE
+ IF N_ELEMENTS(res) EQ 2 THEN mok=0 ELSE mok=-1
+
+ IF mok EQ 0 THEN BEGIN
+ ;test if value is 'NaN'. If so then change to the fill value if already determined
+ IF (FINITE(lv,/NAN)) AND (i GT 2L) THEN BEGIN ;data value is NaN
+ lv=vhold[2]
+ IF nanfound EQ 0 THEN BEGIN
+ infotxt='2 '+vn[vc]+' dataset contains NaN value(s)|. Replaced with Fill Value(s)'
+ INFOTXT_OUTPUT,infotxt
+ nanfound=1
+ ENDIF
+ ENDIF
+ CASE ndm[vc,8] OF
+ 0:BEGIN ;BYTE unsigned 8-bit integer (0 to 255)
+ ;do difference test to account for possible rounding errors
+ IF lv NE BYTE(lv) THEN BEGIN
+ infotxt='3 '+vn[vc]+' '+label[lin]+' outside 8-bit unsigned (BYTE) range: '+STRTRIM(lv,2)
+ INFOTXT_OUTPUT,infotxt
+ ENDIF ELSE vhold[i]=BYTE(lv)
+ END
+ 1:BEGIN ;SHORT signed 16-bit integer (-32,768 to 32,767)
+ ;do difference test to account for possible rounding errors
+ IF lv NE FIX(DOUBLE(lv)) THEN BEGIN
+ infotxt='3 '+vn[vc]+' '+label[lin]+' outside 16-bit signed (SHORT) range: '+STRTRIM(lv,2)
+ INFOTXT_OUTPUT,infotxt
+ ENDIF ELSE vhold[i]=FIX(DOUBLE(lv))
+ END
+ 2:BEGIN ;INTEGER signed 32-bit integer (-2.147e+9 to 2.147e+9)
+ ;do difference test to account for possible rounding errors
+ IF lv NE LONG(DOUBLE(lv)) THEN BEGIN
+ infotxt='3 '+vn[vc]+' '+label[lin]+' outside 32-bit signed (INTEGER) range: '+STRTRIM(lv,2)
+ INFOTXT_OUTPUT,infotxt
+ ENDIF ELSE vhold[i]=LONG(DOUBLE(lv))
+ END
+ 3:BEGIN ;INTEGER signed 64-bit integer (-9.223e+18 to 9.223e+18)
+ ;do difference test to account for possible rounding errors
+ IF ABS(DOUBLE(lv)-DOUBLE(lv)) GE 1. THEN BEGIN
+ infotxt='3 '+vn[vc]+' '+label[lin]+' outside 64-bit signed (LONG) range: '+STRTRIM(lv,2)
+ INFOTXT_OUTPUT,infotxt
+ ENDIF ELSE vhold[i]=LONG64(DOUBLE(lv))
+ END
+ 4:BEGIN
+ IF NOT FINITE(FLOAT(lv)) THEN BEGIN
+ infotxt='3 '+vn[vc]+' '+label[lin]+' outside 32-bit floating point (REAL) range: '+STRTRIM(lv,2)
+ INFOTXT_OUTPUT,infotxt
+ ENDIF ELSE vhold[i]=FLOAT(lv)
+ END
+ 5:BEGIN
+ IF NOT FINITE(DOUBLE(lv)) THEN BEGIN
+ infotxt='3 '+vn[vc]+' '+label[lin]+' outside 64-bit floating point (DOUBLE) range: '+STRTRIM(lv,2)
+ INFOTXT_OUTPUT,infotxt
+ ENDIF ELSE vhold[i]=DOUBLE(lv)
+ END
+ ENDCASE
+ ENDIF ELSE BEGIN
+ STOP_WITH_ERROR,o3[3]+proname+vn[vc],label[lin]+errtxt2[0]+tcerr,lu & RETURN
+ ENDELSE
+ valid=1 ;type conversion OK if got to here
+ENDFOR
+
+;Check if VAR_FILL_VALUE falls within VAR_VALID_MIN/MAX - if so it is defined
+;as a default, rather than missing/error value
+mnv=vhold[0] & mxv=vhold[1] & fv=vhold[2]
+IF (fv GE mnv) AND (fv LE mxv) THEN BEGIN
+ infotxt='0 Fill value for '+meta_arr[vi[0]]+' falls within the data'
+ infotxt=infotxt+' range and is defined as a ''Default'' value'
+ INFOTXT_OUTPUT,infotxt
+ENDIF
+
+dtest=vhold[n_lab-1L:ndl+(n_lab-2L)]
+
+;Do checks on VAR_VALID_MIN and MAX values for Latitude, Longitide, Zenith, Azimuth and Wind Direction datasets
+IF STRPOS(vn[vc],'LATITUDE') NE -1 THEN BEGIN
+ IF (mnv LT -90.0) OR (mxv GT 90.0) THEN BEGIN
+ itxt='-90.0 (South) and +90.0 (North) degrees'
+ infotxt='3 Valid Minimum and Maximum range values for '+vn[vc]+' are '+itxt
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ENDIF ELSE IF STRPOS(vn[vc],'LONGITUDE') NE -1 THEN BEGIN
+ IF (mnv LT -180.0) OR (mxv GT 180.0) THEN BEGIN
+ itxt='-180.0 (West) and +180.0 (East) degrees'
+ infotxt='3 Valid Minimum and Maximum range values for '+vn[vc]+' are '+itxt
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ENDIF ELSE IF (STRPOS(vn[vc],'ZENITH') NE -1) AND (STRPOS(vn[vc],'ANGLE') NE -1) THEN BEGIN
+ IF (mnv LT 0.0) OR (mxv GT 180.0) THEN BEGIN
+ itxt='0.0 and 180.0 degrees'
+ infotxt='3 Valid Minimum and Maximum range values for '+vn[vc]+' are '+itxt
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ENDIF ELSE IF (STRPOS(vn[vc],'AZIMUTH') NE -1) AND (STRPOS(vn[vc],'ANGLE') NE -1) THEN BEGIN
+ IF (mnv LT 0.0) OR (mxv GT 360.0) THEN BEGIN
+ itxt='0.0 (North) and 360.0 degrees (East = 90 degrees and so on)'
+ infotxt='3 Valid Minimum and Maximum range values for '+vn[vc]+' are '+itxt
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ENDIF ELSE IF STRPOS(vn[vc],'WIND.DIRECTION') NE -1 THEN BEGIN
+ IF (mnv NE 0.0) OR (mxv NE 360.0) THEN BEGIN
+ itxt='0.0 (Calm) and 360.0 degrees (use WMO definition)'
+ infotxt='3 Valid Minimum and Maximum range values for '+vn[vc]+' are '+itxt
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ENDIF
+
+;determine whether minimum and maximum data values are within VAR_VALID_MIN/MAX
+nfvi=WHERE(dtest NE fv,nfvcnt)
+IF nfvcnt NE 0 THEN BEGIN ;i.e. there are non-fill values in the data
+ minv=MIN(dtest(nfvi),minvi,MAX=maxv,SUBSCRIPT_MAX=maxvi)
+ IF minv LT mnv THEN BEGIN ;Minimum data value is less than VAR_VALID_MIN
+ IF ndm[vc,8] EQ 0 THEN itxt=STRTRIM(FIX(dtest[nfvi[minvi]]),2) $
+ ELSE itxt=STRTRIM(dtest[nfvi[minvi]],2)
+ infotxt='3 Minimum data value of '+itxt+' is outside '+meta_arr[lvi[0]]
+ infotxt=infotxt+' for dataset '+vn[vc]
+ INFOTXT_OUTPUT,infotxt
+ ;STOP_WITH_ERROR,o3[3]+proname+vn[vc]+': ','Minimum'+errtxt2[1]+ $
+ ; 'MIN: '+STRTRIM(dtest[nfvi[minvi]],2)+' ('+meta_arr[lvi[0]]+').',lu,ds
+ ;RETURN
+ ENDIF
+ IF maxv GT mxv THEN BEGIN ;Maximum data value is greater than VAR_VALID_MAX
+ IF ndm[vc,8] EQ 0 THEN itxt=STRTRIM(FIX(dtest[nfvi[maxvi]]),2) $
+ ELSE itxt=STRTRIM(dtest[nfvi[maxvi]],2)
+ infotxt='3 Maximum data value of '+itxt+' is outside '+meta_arr[lvi[1]]
+ infotxt=infotxt+' for dataset '+vn[vc]
+ INFOTXT_OUTPUT,infotxt
+ ;STOP_WITH_ERROR,o3[3]+proname+vn[vc]+': ','Maximum'+errtxt2[1]+ $
+ ; 'MAX: '+STRTRIM(dtest[nfvi[maxvi]],2)+' ('+meta_arr[lvi[1]]+').',lu,ds
+ ;RETURN
+ ENDIF ELSE IF (STRPOS(vn[vc],'AZIMUTH') NE -1) AND (STRPOS(vn[vc],'ANGLE') NE -1) THEN BEGIN
+ ;also check whether maximum dataset value = 360.0 degrees
+ IF LONG(maxv) EQ 360L THEN BEGIN
+ infotxt='3 Maximum data value of 360.0 degrees is an invalid value for dataset '+vn[vc]+' (North = 0.0)'
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ ENDIF
+ENDIF ELSE BEGIN ;dataset consists of fill values so check that VAR_VALID_MIN LE VAR_VALID_MAX
+ IF mnv GT mxv THEN BEGIN
+ infotxt='3 VAR_VALID_MIN value is greater than the VAR_VALID_MAX value for dataset '+vn[vc]
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ENDELSE
+
+;Do Checks with DATETIME attributes
+IF STRPOS(vn[vc],'DATETIME') EQ 0 THEN BEGIN ;i.e. DATETIME, DATETIME.START or DATETIME.STOP
+ ;Does DATETIME contain any fill values?
+ di=WHERE(dtest EQ fv,dcnt)
+ IF (vn[vc] EQ 'DATETIME') AND (dcnt NE 0) THEN BEGIN
+ ;'DTFVOK' = DATETIME fill value is OK
+ IF (o3[5] EQ 'DTFVOK') AND (dcnt NE ndl) THEN itxt='1 ' ELSE itxt='3 '
+ infotxt=itxt+'DATETIME contains fill value(s)'
+ INFOTXT_OUTPUT,infotxt
+ ;STOP_WITH_ERROR,o3[3]+proname,errtxt2[4]+STRTRIM(dtest[di[0]],2),lu,ds & RETURN
+ ENDIF
+ di=WHERE(dtest NE fv,dcnt)
+ IF dcnt GT 1 THEN BEGIN
+ ;Are DATETIME values in chronological order?
+ dsort=SORT(dtest[di])
+ IF ARRAY_EQUAL(dtest[di],dtest[di[dsort]]) EQ 0 THEN BEGIN
+ IF o3[5] EQ 'DTFVOK' THEN itxt='1 ' ELSE itxt='3 '
+ infotxt=itxt+vn[vc]+' values are not in chronological order'
+ INFOTXT_OUTPUT,infotxt
+ ;STOP_WITH_ERROR,o3[3]+proname,errtxt2[5],lu,ds & RETURN
+ ENDIF
+ ENDIF
+ENDIF
+
+TypeConversionError:
+IF valid EQ 0 THEN STOP_WITH_ERROR,o3[3]+proname+'VAR_NAME='+vn[vc]+': ',errtxt2[6]+tcerr,lu
+
+END ;procedure Check_Min_Max_Fill
+
+
+
+PRO read_data, sds, inf
+;Procedure to perform checks on the data file or structure. It looks for missing or invalid
+;VAR_VALID_MIN/MAX attribute values for datasets which have VAR_UNITS=MJD2K (including
+;start and stop times), and determines these values according to values in the data. It
+;also adds a comment to VAR_NOTES if averaging kernel data is present, to indicate the
+;proper array order (if this option is chosen on program start-up).
+; ----------
+;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
+;
+; History:
+; 20050802: Original IDLCR8HDF routine - Version 1.0
+; 20050909: Check that the input data file is not empty; test for the correct number of
+; datalines in the dataset being searched (e.g. there is not an additional data
+; line in the input file); Bug fix to save dtest as an array when there is only
+; one data value, and the input data is in a structure - Version 1.1
+; 20061012: Portion of routine originally extracting a dataset from the file or structure
+; moved to the EXTRACT_DATA routine; Code added to determine missing or invalid
+; minimum or maximum metadata values for DATETIME, or START/STOP.TIME attributes;
+; Code added to append a comment to VAR_NOTES if averaging kernel data is present,
+; to indicate proper array order (if /AVK option selected); Checked data written
+; to the ds structure instead of a series of arrays set-up according to data type;
+; Common variable definition WIDGET_WIN added - Version 2.0
+; 20080302: dtest initialized for the EXTRACT_DATA call - Version 3.0
+; 20100205: Add test to allow for RETURN to the calling program after STOP_WITH_ERROR calls
+; made by the external routines used by this this procedure - Version 3.09
+; 20101120: Account for new GEOMS rules regarding DATETIME.START/STOP and MJD2K; Do check for
+; averaging kernel vector instead of matrix, in which case do not add AVK comment,
+; if /AVK option chosen - Version 4.0
+; 20150127: Create mv_str array to hold maximum string length of dataset - Version 4.0b25
+; 20150217: Add check for mv_str so it is only created if input is via files; Change AVK
+; comment so it refers to the initial altitude level rather than lowest altitude
+; level - Version 4.0b26
+; 20150409: Strip trailing and leading spaces of VAR_VALID_MAX and VAR_VALID_MIN values
+; when doing checks for missing values for DATETIME datasets - Version 4.0b28
+; 20150811: Initialize infowrite value for call to EXTRACT_DATA - Version 4.0b29
+; 20150924: Add checks on ALTITUDE/PRESSURE ordering plus ordering of corresponding BOUNDARY
+; datasets (only if it is an axis variable). Also perform checks on variables that
+; should be grouped (e.g. LATITUDE and LONGITUDE) - Version 4.0b30
+; 20151012: Add ALTITUDE.GPH and DEPTH to the vertical variables to be checked. Check for FTIR
+; template version before doing some of the .BOUNDARIES checks - Version 4.0b31
+; 20151109: Allow for the possibility of missing variable attributes (i.e. VAR_NOTES) when
+; checking for variable attribute values based on the attr_arr_data array
+; ordering; Checks that VAR_NOTES label is present if the /AVK option is selected,
+; if not then create log message; Do check on *.BOUNDARIES INDEPENDENT dimension
+; regardless of whether the related vertical variable is an axis variable or not
+; - Version 4.0b34
+; 20170331: Update comment appended to VAR_NOTES to clarify AVK dimension ordering
+; -Version 4.0b42
+; 20171121: Change check identifying FTIR templates with incorrect .BOUNDARIES ordering (FTIR
+; changed to FTIR-0 to avoid conflict with other FTIR based templates); Change conditions
+; for identifying invalid wind speed or direction values - Version 4.0b43
+; 20201020: Add check for negative random uncertainty values (standard or relative only)
+; - Version 4.0b56
+;
+; Inputs: sds - Either a structure containing the Variable Attributes and Data, or the input
+; data file
+; inf - flag identifying SDS type where, 0=structure; 1=data file
+; meta_arr - a string array containing the Global and Variable Attributes
+;
+; Outputs: meta_arr - completes missing or invalid metadata values for attributes with
+; VAR_UNITS=MJD2K, as required, and also adds comments to VAR_NOTES
+; for averaging kernel data to indicate proper array order, if this
+; option is chosen
+; ds - checked datasets are written to the ds structure with correct dimensions and
+; data type
+;
+; Called by: IDLCR8HDF
+;
+; Subroutines Called: EXTRACT_DATA
+; CHECK_MIN_MAX_FILL
+
+COMMON METADATA
+COMMON DATA
+COMMON WIDGET_WIN
+
+;Go through variable names and check for DATETIME, DATETIME.START, or DATETIME.STOP
+;values and determine representative minimum and maximum datetime values
+mint=90000.D & maxt=-90000.D ;initialize min and max datetimes
+valfound=0 ;count variable that increments when DATETIME and related datasets are found
+errorfound=1 ;variable to indicate error in DATETIME checks
+infowrite=0 ;do not write ISO8601 to MJD2K information message in EXTRACT_DATA routine (will write on last EXTRACT_DATA call)
+;set-up holding array to hold number of bytes in string VAR_FILL_VALUE (note already created if input is via session memory)
+IF mv_str[0] EQ -1L THEN mv_str=LONARR(nvn)
+vchk=['VAR_VALID_MIN','VAR_VALID_MAX']
+nvchk=N_ELEMENTS(vchk) & mmi=INTARR(nvchk)
+FOR i=0,nvchk-1 DO BEGIN
+ mi=WHERE(attr_arr_data EQ vchk[i]) & mmi[i]=mi[0]
+ENDFOR
+vavk=INTARR(nvn) ;holding array for AVK attribute (used below)
+
+;Determine number of DATETIME values, and set arrays for DATETIME checks
+dtfv=DBLARR(3)-mint
+dti=WHERE(vn EQ 'DATETIME',dtcnt)
+IF dtcnt NE 0 THEN BEGIN
+ dtest=['0'] ;initializing dtest array for EXTRACT_DATA call
+ EXTRACT_DATA,sds,inf,dti[0],dtest,ndl,infowrite ;return the dataset
+ IF STRLEN(rerr[0]) GT 2 THEN RETURN
+ dtchk=N_ELEMENTS(dtest)
+ dtvals=DBLARR(3,dtchk)-mint
+ errorfound=0
+ENDIF ELSE BEGIN
+ dti=WHERE(vn EQ 'DATETIME.START',dtcnt)
+ dtsi=WHERE(vn EQ 'DATETIME.STOP',dtscnt)
+ IF (dtcnt NE 0) AND (dtscnt NE 0) THEN BEGIN
+ dtest=['0'] ;initializing dtest array for EXTRACT_DATA call
+ EXTRACT_DATA,sds,inf,dti[0],dtest,ndl,infowrite ;return the dataset
+ IF STRLEN(rerr[0]) GT 2 THEN RETURN
+ dtchk=N_ELEMENTS(dtest)
+ dtest=['0'] ;initializing dtest array for EXTRACT_DATA call
+ EXTRACT_DATA,sds,inf,dtsi[0],dtest,ndl,infowrite ;return the dataset
+ IF STRLEN(rerr[0]) GT 2 THEN RETURN
+ IF dtchk EQ N_ELEMENTS(dtest) THEN BEGIN
+ dtvals=DBLARR(3,dtchk)-mint
+ errorfound=0
+ ENDIF
+ ENDIF
+ENDELSE
+
+FOR vc=0,nvn-1 DO BEGIN
+ vnspl=STRSPLIT(vn[vc],'_',/Extract)
+ vnspl=[vnspl,'x','x'] ;ensures vnspl has a minimum of 3 values
+ IF (vnspl[1] EQ 'AVK') OR (vnspl[2] EQ 'AVK') THEN vavk[vc]=1 ;identifies a dataset containing AVK data
+ IF ((vn[vc] EQ 'DATETIME') OR (vn[vc] EQ 'DATETIME.START') OR $
+ (vn[vc] EQ 'DATETIME.STOP')) THEN BEGIN
+ dtest=['0'] ;initializing dtest array for EXTRACT_DATA call
+ EXTRACT_DATA,sds,inf,vc,dtest,ndl,infowrite ;return the dataset
+ IF STRLEN(rerr[0]) GT 2 THEN RETURN
+ IF errorfound EQ 0 THEN BEGIN
+ dtnel=N_ELEMENTS(dtest)
+ CASE 1 OF
+ vn[vc] EQ 'DATETIME.START': BEGIN
+ IF dtnel EQ dtchk THEN dtvals[0,*]=DOUBLE(dtest) $
+ ELSE IF dtnel EQ 1 THEN dtvals[0,0]=DOUBLE(dtest) $
+ ELSE errorfound=1
+ dtfv[0]=mv_dbl[2,vc]
+ END
+ vn[vc] EQ 'DATETIME': BEGIN
+ dtvals[1,*]=DOUBLE(dtest) & dtfv[1]=mv_dbl[2,vc]
+ END
+ ELSE: BEGIN
+ IF dtnel EQ dtchk THEN dtvals[2,*]=DOUBLE(dtest) $
+ ELSE IF dtnel EQ 1 THEN dtvals[2,dtchk-1]=DOUBLE(dtest) $
+ ELSE errorfound=1
+ dtfv[2]=mv_dbl[2,vc]
+ END
+ ENDCASE
+ ENDIF
+ gi=WHERE(DOUBLE(dtest) NE mv_dbl[2,vc],gcnt) ;edit out fill values
+ IF gcnt NE 0 THEN BEGIN
+ minth=MIN(DOUBLE(dtest[gi]),MAX=maxth)
+ IF minth LT mint THEN mint=minth
+ IF maxth GT maxt THEN maxt=maxth
+ ;IF vn[vc] EQ 'DATETIME.START' THEN mint=minth $
+ ;ELSE IF vn[vc] EQ 'DATETIME.STOP' THEN maxt=maxth
+ ENDIF
+ valfound=valfound+1
+ ENDIF
+ENDFOR
+
+IF (errorfound EQ 0) AND (valfound GT 1) THEN BEGIN
+ ;Make all missing values equal to fill values
+ FOR i=0,2 DO BEGIN
+ mi=WHERE(dtvals[i,*] EQ -90000.D,mcnt)
+ IF mcnt NE 0 THEN dtvals[i,mi]=dtfv[i]
+ ENDFOR
+ ;Check that DATETIME.START LE DATETIME LE DATETIME.STOP
+ i=0L
+ WHILE (errorfound EQ 0) AND (i LE dtchk-1L) DO BEGIN
+ test1=(dtvals[0,i] NE dtfv[0]) AND (dtvals[1,i] NE dtfv[1])
+ test2=(dtvals[1,i] NE dtfv[1]) AND (dtvals[2,i] NE dtfv[2])
+ test3=(dtvals[0,i] NE dtfv[0]) AND (dtvals[2,i] NE dtfv[2])
+ IF (test1) AND (dtvals[0,i] GT dtvals[1,i]) THEN errorfound=1
+ IF (test2) AND (dtvals[1,i] GT dtvals[2,i]) THEN errorfound=2
+ IF (test3) AND (dtvals[0,i] GT dtvals[2,i]) THEN errorfound=3
+ i=i+1L
+ ENDWHILE
+ IF errorfound NE 0 THEN BEGIN
+ v0=STRTRIM(STRING(format='(d18.9)',dtvals[0,i-1L]),2)
+ v1=STRTRIM(STRING(format='(d18.9)',dtvals[1,i-1L]),2)
+ v2=STRTRIM(STRING(format='(d18.9)',dtvals[2,i-1L]),2)
+ CASE 1 OF
+ errorfound EQ 1: itxt=['DATETIME.START='+v0,'DATETIME='+v1]
+ errorfound EQ 2: itxt=['DATETIME='+v1,'DATETIME.STOP='+v2]
+ ELSE: itxt=['DATETIME.START='+v0,'DATETIME.STOP='+v2]
+ ENDCASE
+ infotxt='3 '+itxt[0]+' is greater than '+itxt[1]
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ENDIF
+
+valfound=0 ;Boolean to indicate whether missing VAR_VALID_MIN/MAX values have been added
+;Add minimum and maximum values to Metadata as required
+di=WHERE(STRUPCASE(vu) EQ 'MJD2K',dcnt)
+IF dcnt NE 0 THEN BEGIN
+ FOR vc=0,dcnt-1 DO BEGIN
+ mi=WHERE(meta_arr EQ 'VAR_NAME='+vn[di[vc]])
+ FOR i=0,nvchk-1 DO BEGIN
+ IF STRMID(meta_arr[mi[0]+mmi[i]],0,STRLEN(vchk[i])) NE vchk[i] THEN acti=mmi[i]-1L ELSE acti=mmi[i] ;depends if VAR_NOTES is present or not
+ res=STRSPLIT(STRTRIM(meta_arr[mi[0]+acti],2),'=',/Extract,COUNT=nres)
+ IF nres EQ 1 THEN BEGIN ;no value so add Min or Max value to this attribute
+ IF i MOD 2 EQ 0 THEN BEGIN
+ IF mint NE 90000.d THEN BEGIN
+ meta_arr[mi[0]+acti]=vchk[i]+'='+STRTRIM(STRING(format='(d18.9)',mint),2)
+ valfound=1
+ ENDIF
+ mv_dbl[i,di[vc]]=mint
+ ENDIF ELSE BEGIN
+ IF maxt NE -90000.d THEN BEGIN
+ meta_arr[mi[0]+acti]=vchk[i]+'='+STRTRIM(STRING(format='(d18.9)',maxt),2)
+ valfound=1
+ ENDIF
+ mv_dbl[i,di[vc]]=maxt
+ ENDELSE
+ IF valfound EQ 1 THEN BEGIN
+ infotxt=STRARR(2)
+ infotxt[0]='2 Missing '+vchk[i]+' value for '+meta_arr[mi[0]]+'| added based on available'
+ infotxt[1]=' DATETIME[.START][.STOP] values: '+meta_arr[mi[0]+acti]
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ ENDIF
+ ENDFOR
+ ENDFOR
+ENDIF
+
+IF o3[1] EQ 'AVK' THEN BEGIN ;append sentence to VAR_NOTES in the AVK attribute
+ di=WHERE(vavk EQ 1,dcnt)
+ vnc=STRARR(nvn) ;holding array for comment
+ IF dcnt NE 0 THEN BEGIN
+ FOR vc=0,dcnt-1 DO BEGIN
+ mi=WHERE(meta_arr EQ 'VAR_NAME='+vn[di[vc]])
+ ;check to see if AVK is a matrix or vector (no need for comment if it us a vector)
+ add_comm=0 ;default is for no comment
+ xi=WHERE(attr_arr_data EQ 'VAR_DEPEND')
+ IF STRMID(meta_arr[mi[0]+xi[0]],0,10) NE 'VAR_DEPEND' THEN acti=xi[0]-1L ELSE acti=xi[0] ;depends if VAR_NOTES is present or not
+ res=STRSPLIT(meta_arr[mi[0]+acti],'=; ',/EXTRACT,COUNT=nres)
+ res=STRUPCASE(res)
+ IF nres GE 3 THEN BEGIN
+ FOR i=1,nres-1 DO BEGIN
+ ri=WHERE(res[i] EQ res,rcnt) ;i.e. repeated dependencies indicate an AVK matrix
+ IF rcnt GE 2 THEN add_comm=1
+ ENDFOR
+ ENDIF
+ IF add_comm THEN BEGIN
+ IF acti NE xi[0] THEN BEGIN ;No VAR_NOTES attribute present so generate error
+ infotxt='3 VAR_NOTES attribute not present under '+meta_arr[mi[0]]+' so unable to append Averaging Kernel information'
+ INFOTXT_OUTPUT,infotxt
+ ENDIF ELSE BEGIN
+ ;find relevant VAR_NOTES attribute label
+ xi=WHERE(attr_arr_data EQ 'VAR_NOTES')
+ eqpos=STRPOS(meta_arr[mi[0]+xi[0]],'=')
+ vnc[di[vc]]=STRTRIM(STRMID(meta_arr[mi[0]+xi[0]],eqpos+1),2) ;existing VAR_NOTES comment, if any
+ ;if comment exists, check to see if it ends with a period, if not add one
+ IF vnc[di[vc]] NE '' THEN BEGIN
+ IF STRMID(vnc[di[vc]],STRLEN(vnc[di[vc]])-1,1) NE '.' THEN $
+ vnc[di[vc]]=vnc[di[vc]]+'. ' ELSE vnc[di[vc]]=vnc[di[vc]]+' '
+ ENDIF
+ ;determine the number of dimensions
+ ni=WHERE(ndm[di[vc],0:7] NE 0,ndim)
+ IF ndim GT 2 THEN vncx='for the first measurement are:' ELSE vncx='are:'
+ ;append comment
+ vnc[di[vc]]=vnc[di[vc]]+'First three values of the initial averaging kernel '+vncx
+ ;append meta_arr index to comment
+ vnc[di[vc]]=STRTRIM(mi[0]+xi[0],2)+'_'+vnc[di[vc]]
+ ENDELSE
+ ENDIF
+ ENDFOR
+ ENDIF
+ENDIF
+
+;If required do checks on the vertical dimension plus correspondng .BOUNDARIES
+;Only do checks if variable is an axis variable - so must be in ascending or descending order
+bchks=['ALTITUDE','PRESSURE','ALTITUDE.GPH','DEPTH']
+sdata=ndm(vc,8) EQ 6
+
+;Do check for DATA_TEMPLATE and do not do all the .BOUNDARIES checks for FTIR 001 or 002 templates
+ti=WHERE(STRMID(meta_arr,0,13) EQ 'DATA_TEMPLATE',tcnt)
+ftirchk=0 ;default is that it is not an FTIR-001 or 002 measurement
+IF tcnt EQ 1 THEN BEGIN
+ res=STRSPLIT(meta_arr[ti[0]],'=',/EXTRACT,COUNT=nres)
+ IF nres EQ 2 THEN BEGIN
+ IF (STRPOS(res[1],'FTIR-0') NE -1) AND ((STRPOS(res[1],'001') NE -1) OR (STRPOS(res[1],'002') NE -1)) THEN ftirchk=1
+ ENDIF
+ENDIF
+
+FOR vc=0,N_ELEMENTS(bchks)-1 DO BEGIN
+ bci=WHERE(vn EQ bchks[vc],bccnt) & bbci=WHERE(vn EQ bchks[vc]+'.BOUNDARIES',bbccnt)
+ IF bccnt EQ 1 THEN BEGIN ;ALTITUDE or PRESSURE present so extract VAR_DEPEND and VAR_SIZE values
+ mi=WHERE(meta_arr EQ 'VAR_NAME='+vn[bci[0]])
+ xi=WHERE(attr_arr_data EQ 'VAR_DEPEND')
+ IF STRMID(meta_arr[mi[0]+xi[0]],0,10) NE 'VAR_DEPEND' THEN acti=xi[0]-1L ELSE acti=xi[0] ;depends if VAR_NOTES is present or not
+ res=STRSPLIT(meta_arr[mi[0]+acti],'=; ',/EXTRACT,COUNT=nres)
+ vdvals=STRUPCASE(res[1:nres-1])
+ di=WHERE(ndm[bci[0],0:7] NE 0L,ndim)
+ vsvals=ndm[bci[0],di]
+ bi=WHERE(vdvals EQ bchks[vc],bcnt) ;determine if the variable of interest is also self-referencing
+ nbi=WHERE(vdvals NE bchks[vc],nbcnt) ;determine if there are any other dependencies e.g. DATETIME
+ sdata=ndm[bci[0],8] EQ 6 ;test for string datatype
+ writeonce=[0,0] ;initialize array for writing information/error messages
+ IF (bcnt EQ 1) AND (ndim LE 2) AND (~sdata) THEN BEGIN ;it is self-referencing and there are 2 or less dependencies so do the checks
+ ascending=-1 ;set default value - will change to 0 for descending order and 1 for ascending order
+ writeonce=[1,1] ;ensure log text is only written once
+ EXTRACT_DATA,sds,inf,bci[0],dtest,ndl,infowrite
+ IF STRLEN(rerr[0]) GT 2 THEN RETURN
+ bvals=DOUBLE(REFORM(dtest,ndm[bci[0],di])) ;put into array order and make numeric
+ IF nbcnt EQ 1 THEN n_sec=ndm[bci[0],nbi[0]] ELSE n_sec=1L ;Number of datasets to be read through
+ IF bi[0] NE 0 THEN bvals=TRANSPOSE(bvals) ;make ALTITUDE or PRESSURE array the first index
+ FOR i=0L,n_sec-1L DO BEGIN ;e.g. number of DATETIME values or single loop if only one dimension
+ bvtest=bvals[*,i] ;set of PRESSURES or ALTITUDES to be tested
+ asc=-1
+ IF ARRAY_EQUAL(bvtest,bvtest[SORT(bvtest)]) THEN asc=1 $ ;ascending order
+ ELSE IF ARRAY_EQUAL(bvtest,bvtest[REVERSE(SORT(bvtest))]) THEN asc=0 ;descending order
+ IF ascending EQ -1 THEN ascending=asc
+ IF (asc EQ -1) OR (ascending NE asc) THEN BEGIN ;altitudes/pressures not in any order
+ IF (ascending NE -1) AND (writeonce[1] EQ 1) THEN BEGIN ;more than one dimension and error in one or more of the sets
+ IF ascending EQ 1 THEN itxt='increasing' ELSE itxt='decreasing'
+ infotxt='3 '+bchks[vc]+' values not in '+itxt+' order for every '+vdvals[nbi[0]]+' for '+meta_arr[mi[0]]
+ INFOTXT_OUTPUT,infotxt
+ writeonce[1]=0
+ ENDIF ELSE IF writeonce[0] EQ 1 THEN BEGIN
+ infotxt='3 '+bchks[vc]+' values not monotonically increasing or decreasing for '+meta_arr[mi[0]]
+ INFOTXT_OUTPUT,infotxt
+ writeonce[0]=0
+ ENDIF
+ ENDIF
+ ENDFOR
+ ENDIF
+
+ IF bbccnt EQ 1 THEN BEGIN ;*.BOUNDARIES present so check that INDEPENDENT index has VAR_SIZE=2
+ mi=WHERE(meta_arr EQ 'VAR_NAME='+vn[bbci[0]])
+ xi=WHERE(attr_arr_data EQ 'VAR_DEPEND')
+ IF STRMID(meta_arr[mi[0]+xi[0]],0,10) NE 'VAR_DEPEND' THEN acti=xi[0]-1L ELSE acti=xi[0] ;depends if VAR_NOTES is present or not
+ res=STRSPLIT(meta_arr[mi[0]+acti],'=; ',/EXTRACT,COUNT=nres)
+ vdvals=STRUPCASE(res[1:nres-1])
+ di=WHERE(ndm[bbci[0],0:7] NE 0L,ndim)
+ vsvals=ndm[bbci[0],di]
+ bi=WHERE(vdvals EQ bchks[vc],bcnt) ;determine index of the variable of interest is also self-referencing
+ IF bcnt EQ 1 THEN n_alt=ndm[bbci[0],bi[0]]
+ ibi=WHERE(vdvals EQ 'INDEPENDENT',ibcnt)
+ IF ibcnt EQ 1 THEN BEGIN
+ n_ind=ndm[bbci[0],ibi[0]]
+ IF n_ind NE 2 THEN BEGIN
+ infotxt='3 INDEPENDENT index must have VAR_SIZE=2 for '+meta_arr[mi[0]]
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ ENDIF
+ ENDIF
+
+ oksofar=(writeonce[0] EQ 1) AND (writeonce[1] EQ 1)
+ IF (bbccnt EQ 1) AND (oksofar) THEN BEGIN ;Also need to check that xx.BOUNDARIES is in the same order (ascending or descending)
+ ;Note: Only do checks if checks on axis variable were all OK
+ nbi=WHERE((vdvals NE bchks[vc]) AND (vdvals NE 'INDEPENDENT'),nbcnt) ;determine if there are any other dependencies e.g. DATETIME
+ IF nbcnt EQ 1 THEN n_sec=ndm[bbci[0],nbi[0]] ELSE n_sec=1
+ sdata=(ndm[bbci[0],8] EQ 6) ;test for string datatype
+ IF (bcnt EQ 1) AND ((ndim EQ 2) OR (ndim EQ 3)) AND (n_ind EQ 2) AND (~sdata) THEN BEGIN
+ ;it is dependent on ALTITUDE/PRESSURE and there are 2 or less other dependencies so do the checks
+ EXTRACT_DATA,sds,inf,bbci[0],dtest,ndl,infowrite
+ IF STRLEN(rerr[0]) GT 2 THEN RETURN
+ bvals=DOUBLE(REFORM(dtest,ndm[bbci[0],di])) ;put into array order and make numeric
+
+ ;Determine the actual ordering of the array
+ ;Option 1 = 2-D e.g. ALTITUDE;INDEPENDENT or INDEPENDENT;ALTITUDE
+ ;Option 2 = 3-D with axis variable as first or last index and INDEPENDENT is the middle index e.g. ALT;IND;DAT or DAT;IND;ALT
+ ;Option 3 = 3-D with axis variable as first or last index and INDEPENDENT is the first or last index e.g. ALT;DAT;IND or IND;DAT;ALT
+ ;Option 4 = 3-D with INDEPENDENT variable as first or last index and ALTITUDE is the middle index e.g. IND;ALT;DAT or DAT;ALT;IND
+ IF (bcnt EQ 1) AND (ibcnt EQ 1) AND (ndim EQ 2) THEN BEGIN
+ option=1
+ IF bi[0] NE 0 THEN bvals=TRANSPOSE(bvals) ;make ALTITUDE or PRESSURE array the first index
+ ENDIF ELSE IF (bcnt EQ 1) AND (ibcnt EQ 1) AND (ndim EQ 3) THEN BEGIN
+ IF ((bi[0] EQ 0) OR (bi[0] EQ 2)) AND (ibi[0] EQ 1) THEN BEGIN
+ option=2
+ IF bi[0] EQ 2 THEN bvals=TRANSPOSE(bvals) ;make ALTITUDE or PRESSURE array the first index
+ ENDIF ELSE IF ((bi[0] EQ 0) OR (bi[0] EQ 2)) AND (nbi[0] EQ 1) THEN BEGIN
+ option=3
+ IF bi[0] EQ 2 THEN bvals=TRANSPOSE(bvals) ;make ALTITUDE or PRESSURE array the first index
+ ENDIF ELSE BEGIN
+ option=4
+ IF ibi[0] EQ 2 THEN bvals=TRANSPOSE(bvals) ;make INDEPENDENT array the first index (ALTITUDE or PRESSURE is the 2nd index)
+ ENDELSE
+ ENDIF ELSE option=0
+
+ IF option NE 0 THEN BEGIN
+ writeonceb=[1,1]
+ FOR i=0L,n_sec-1L DO BEGIN
+ FOR j=0,n_ind-1 DO BEGIN ;check that order of the boundary sets matches the axis variable ordering
+ CASE option of ;set of PRESSURES or ALTITUDES to be tested
+ 1: bvtest=bvals[*,j]
+ 2: bvtest=bvals[*,j,i]
+ 3: bvtest=bvals[*,i,j]
+ 4: bvtest=bvals[j,*,i]
+ ENDCASE
+ asc=-1
+ IF ARRAY_EQUAL(bvtest,bvtest[SORT(bvtest)]) THEN asc=1 $ ;ascending order
+ ELSE IF ARRAY_EQUAL(bvtest,bvtest[REVERSE(SORT(bvtest))]) THEN asc=0 ;descending order
+
+ IF (ascending NE asc) AND (writeonceb[0] EQ 1) THEN BEGIN
+ ;order of the boundaries dataset does not match axis dataset
+ IF ascending EQ 1 THEN itxt='increasing' ELSE itxt='decreasing'
+ infotxt='3 '+bchks[vc]+'.BOUNDARIES order does not match '+bchks[vc]+' '+itxt+' ordering for '+meta_arr[mi[0]]
+ INFOTXT_OUTPUT,infotxt
+ writeonceb[0]=0
+ ENDIF
+ ENDFOR
+ ENDFOR
+ ;Check that the boundary ordering is also the same as the axis variable for every ALTITUDE/PRESSURE
+ FOR i=0L,n_sec-1L DO BEGIN
+ FOR j=0L,n_alt-1L DO BEGIN
+ CASE option OF ;set of boundary pairs to be tested (INDEPENDENT index)
+ 1: bvtest=bvals[j,*]
+ 2: bvtest=bvals[j,*,i]
+ 3: bvtest=bvals[j,i,*]
+ 4: bvtest=bvals[*,j,i]
+ ENDCASE
+ asc=-1
+ IF ARRAY_EQUAL(bvtest,bvtest[SORT(bvtest)]) THEN asc=1 $ ;ascending order
+ ELSE IF ARRAY_EQUAL(bvtest,bvtest[REVERSE(SORT(bvtest))]) THEN asc=0 ;descending order
+
+ IF (ascending NE asc) AND (writeonceb[1] EQ 1) AND (ftirchk EQ 0) THEN BEGIN
+ ;order of the boundaries values does not match axis dataset
+ IF ascending EQ 1 THEN itxt='increasing' ELSE itxt='decreasing'
+ infotxt='3 Boundary index order does not match '+bchks[vc]+' '+itxt+' ordering for '+meta_arr[mi[0]]
+ INFOTXT_OUTPUT,infotxt
+ writeonceb[1]=0
+ ENDIF
+ ENDFOR
+ ENDFOR
+ ENDIF
+ ENDIF
+ ENDIF
+ ENDIF
+ENDFOR
+
+;Test that random uncertainty values are positive
+posun=['UNCERTAINTY.RANDOM.STANDARD','UNCERTAINTY.RANDOM.STANDARD.RELATIVE']
+n_p=N_ELEMENTS(posun)
+vnx=vn
+;vnx consists of only the last sub-name of vn
+FOR i=0,nvn-1 DO $
+ IF STRPOS(vnx[i],'_') NE -1 THEN vnx[i]=STRMID(vnx[i],STRPOS(vnx[i],'_',/REVERSE_SEARCH)+1)
+FOR i=0,n_p-1 DO BEGIN ;number of values to check
+ pi=WHERE(vnx EQ posun[i],pcnt)
+ IF pcnt NE 0 THEN BEGIN
+ FOR j=0,pcnt-1 DO BEGIN ;test for negative data values
+ sdata=(ndm[pi[j],8] EQ 6) ;test for string datatype
+ IF ~sdata THEN BEGIN
+ EXTRACT_DATA,sds,inf,pi[j],dtest,ndl,infowrite
+ IF STRLEN(rerr[0]) GT 2 THEN RETURN
+
+ CASE ndm[pi[j],8] OF ;identifies the VAR_DATA_TYPE
+ 0:BEGIN
+ fv=BYTE(mv_lng[2,pi[j]]) & urd=BYTE(dtest) & zero=0
+ END
+ 1:BEGIN
+ fv=FIX(mv_lng[2,pi[j]]) & urd=FIX(dtest) & zero=0
+ END
+ 2:BEGIN
+ fv=LONG(mv_lng[2,pi[j]]) & urd=LONG(dtest) & zero=0L
+ END
+ 3:BEGIN
+ fv=mv_lng[2,pi[j]] & urd=LONG64(dtest) & zero=0LL
+ END
+ 4:BEGIN
+ fv=FLOAT(mv_dbl[2,pi[j]]) & urd=FLOAT(dtest) & zero=0.
+ END
+ 5:BEGIN
+ fv=mv_dbl[2,pi[j]] & urd=DOUBLE(dtest) & zero=0.d
+ END
+ ENDCASE
+ ni=WHERE((urd LT zero) AND (urd NE fv),ncnt)
+ IF ncnt NE 0 THEN BEGIN
+ infotxt='3 '+vn[pi[j]]+' values cannot be negative'
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ ENDIF
+ ENDFOR
+ ENDIF
+ENDFOR
+
+;Test 'grouped' datasets, i.e. datasets that must both be in the file if one of them is present
+;Note that there might be more than one variable in a particular group e.g. LATITUDE and LATITUDE.INSTRUMENT
+grouped=[['LATITUDE','LONGITUDE'],['WIND.DIRECTION','WIND.SPEED']]
+g_dim=SIZE(grouped,/DIMENSIONS)
+FOR i=0,g_dim(1)-1 DO BEGIN ;number of grouped pairs
+ p1i=WHERE(STRPOS(STRMID(vn,0,STRLEN(grouped[0,i])),grouped[0,i]) NE -1,p1cnt)
+ p2i=WHERE(STRPOS(STRMID(vn,0,STRLEN(grouped[1,i])),grouped[1,i]) NE -1,p2cnt)
+ IF p1cnt NE 0 THEN BEGIN
+ FOR j=0,p1cnt-1 DO BEGIN ;test all variations
+ extrapart=STRMID(vn[p1i[j]],STRLEN(grouped[0,i])) & extrapart=STRTRIM(extrapart,2)
+ mi=WHERE(vn EQ grouped[1,i]+extrapart,mcnt)
+ IF mcnt EQ 0 THEN BEGIN ;matching variable name not found
+ infotxt='3 VAR_NAME='+grouped[1,i]+extrapart+' must also be present together with VAR_NAME='+vn[p1i[j]]
+ INFOTXT_OUTPUT,infotxt
+ ENDIF ELSE BEGIN
+ ;Do tests on grouped values
+ ;1. VAR_SIZE values must be the same. If this is the case then for wind direction and wind speed,
+ ;2. If wind direction EQ 0 then corresponding wind speed must be 0
+ ;3. If wind direction NE 0 then corresponding wind speed can't be 0
+ IF ~ARRAY_EQUAL(ndm[p1i[j],0:7],ndm[mi[0],0:7]) THEN BEGIN
+ infotxt='3 VAR_SIZE values for VAR_NAME='+vn[p1i[j]]+' must be the same as VAR_SIZE values for VAR_NAME='+vn[mi[0]]
+ INFOTXT_OUTPUT,infotxt
+ ENDIF ELSE BEGIN ;arrays match up so do wind direction/speed checks
+ sdata=(ndm[p1i[j],8] EQ 6) OR (ndm[mi[0],8] EQ 6) ;test for string datatype
+ IF (grouped[0,i] EQ 'WIND.DIRECTION') AND (~sdata) THEN BEGIN
+ EXTRACT_DATA,sds,inf,p1i[j],dtest,ndl,infowrite
+ IF STRLEN(rerr[0]) GT 2 THEN RETURN
+ wdd=DOUBLE(dtest)
+ EXTRACT_DATA,sds,inf,mi[0],dtest,ndl,infowrite
+ IF STRLEN(rerr[0]) GT 2 THEN RETURN
+ wsd=DOUBLE(dtest)
+ ni=WHERE(wdd EQ 0.d,ncnt)
+ IF ncnt NE 0 THEN BEGIN
+ si=WHERE(wsd[ni] GT 0.d,scnt)
+ IF scnt NE 0 THEN BEGIN ;Wind direction = 0.0 but wind speed NE 0.0
+ infotxt='3 '+vn[mi[0]]+' values must be 0.0 for corresponding '+vn[p1i[j]]+' 0.0 values'
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ ENDIF
+ ni=WHERE((wdd GT 0.d) AND (wdd LE 360.d),ncnt)
+ IF ncnt NE 0 THEN BEGIN
+ si=WHERE(wsd[ni] EQ 0.d,scnt)
+ IF scnt NE 0 THEN BEGIN ;Wind direction NE 0.0 but wind speed = 0.0
+ infotxt='3 '+vn[p1i[j]]+' values must be 0.0 for corresponding '+vn[mi[0]]+' 0.0 values'
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ ENDIF
+ ENDIF
+ ENDELSE
+ ENDELSE
+ ENDFOR
+ ENDIF
+ IF p2cnt NE 0 THEN BEGIN ;now do the reverse check
+ FOR j=0,p2cnt-1 DO BEGIN ;test all variations
+ extrapart=STRMID(vn[p2i[j]],STRLEN(grouped[1,i])) & extrapart=STRTRIM(extrapart,2)
+ mi=WHERE(vn EQ grouped[0,i]+extrapart,mcnt)
+ IF mcnt EQ 0 THEN BEGIN ;matching variable name not found
+ infotxt='3 VAR_NAME='+grouped[0,i]+extrapart+' must also be present together with VAR_NAME='+vn[p2i[j]]
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ ENDFOR
+ ENDIF
+ENDFOR
+
+FOR vc=0,nvn-1 DO BEGIN
+ dtest=['0'] ;initializing dtest array for EXTRACT_DATA call
+ infowrite=1 ;write information message in EXTRACT_DATA routine if required
+ ;Extract the dataset
+ EXTRACT_DATA,sds,inf,vc,dtest,ndl,infowrite
+ IF STRLEN(rerr[0]) GT 2 THEN RETURN
+ ;perform checks on the attribute and data values
+ IF ndm[vc,8] EQ 6 THEN CHECK_STRING_DATATYPE,vc,dtest $ ;Test for String Datatype
+ ELSE BEGIN
+ CHECK_MIN_MAX_FILL,vc,dtest,ndl ;Test for Numeric Datatype
+ IF STRLEN(rerr[0]) GT 2 THEN RETURN
+ ENDELSE
+
+ ;if AVK dataset extracted then add comment to VAR_NOTES if required
+ IF o3[1] EQ 'AVK' THEN BEGIN
+ IF (vnc[vc] NE '') AND (ndm[vc,0] GE 3) THEN BEGIN
+ res=STRSPLIT(vnc[vc],'_',/Extract) & mi=FIX(res[0])
+ meta_arr[mi]='VAR_NOTES='+res[1]
+ FOR i=0,2 DO meta_arr[mi]=meta_arr[mi]+' '+STRTRIM(dtest[i],2)
+ ENDIF
+ ENDIF
+
+ ;Copy data to the data structure with correct dimensions and data type
+ arrchk=SIZE(dtest) ;put it into array form if it is a scalar
+ IF arrchk[0] EQ 0 THEN dtest=[dtest]
+ gi=WHERE(ndm[vc,0:7] NE 0)
+ vs=TRANSPOSE(ndm[vc,gi])
+ dtest=REFORM(dtest,vs,/Overwrite)
+ ds[vc].data=PTR_NEW(dtest)
+ENDFOR
+
+END ;Procedure Read_Data
+
+
+
+PRO find_hdf_filename, hdffilename
+;Procedure to do the following:
+;1. Compare the DATA_START_DATE and DATA_STOP_DATE with the first and last entries under
+; DATETIME. If necessary create/change DATA_START_DATE and DATA_STOP_DATE to match the
+; DATETIME entries, and put in ISO8601 format.
+;2. Compare filename created by the program from the DATASET_ATTRIBUTES with that under the
+; FILE_NAME entry (if present). If necessary create/change the FILE_NAME entry to match the
+; created filename
+;3. Create/change the FILE_GENERATION_DATE and ensure it is in ISO8601 format.
+; ----------
+;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
+;
+; History:
+; 20050802: Original IDLCR8HDF routine - Version 1.0
+; 20050909: Remove requirement for a 'v' to be at the start of the DATA_FILE_VERSION
+; (e.g. v1.0 or 1.0 acceptable); Change the format that DATA_START_DATE and
+; FILE_GENERATION_DATE values are saved in the HDF file from MJD2000 to
+; ISO8601 - Version 1.1
+; 20051107: Move VIS_SCALE_MIN/MAX checks, for correct format of DATETIME values, to the
+; CHECK_MIN_MAX_FILL routine - Version 1.11
+; 20061012: Move the FILE_GENERATION_DATE checks to the main loop which determines the file
+; name to facilitate the future option of including the file generation date in the
+; file name (this option is not currently implemented by the AVDC); Common variable
+; definition WIDGET_WIN added - Version 2.0
+; 20080302: Do search for either DATA_LEVEL or DATA_TYPE, depending on whether an AVDC style
+; TAV file or original Envisat table.dat file has been read in; Common variable
+; definition TABLEDATA added (tab_type used to differentiate between the databases)
+; - Version 3.0
+; 20090610: Allow for the Global Variable DATA_STOP_DATE and calculate the value if required
+; - Version 3.07
+; 20100205: Add RETURN command after all STOP_WITH_ERROR calls, which allows program to return to the
+; calling program if the reterr argument is included in the idlcr8hdf call - Version 3.09
+; 20101120: Account for new GEOMS changes to filename composition - Version 4.0
+; 20151104: Check that ISO8601 format time is all uppercase - Version 4.0b33
+; 20151109: Check for seconds value of 60 in ISO8601 datetime value - Version 4.0b34
+; 20151215: Fix bug if DATE_TIME_START or DATE_TIME_STOP value is missing - Version 4.0b36
+; 20161130: Allow the program to continue running if an ISO8601 error is found (previously
+; called STOP_WITH_ERROR) - Version 4.0b40
+; 20171130: Remove ds from STOP_WITH_ERROR parameters; Change some error and information text
+; statements - Version 4.0b44
+; 20180314: Fix bug when checking DATA_START|STOP_DATE values. Criteria for difference with
+; MJD2K values change to GT 0.99d/86400.d instead of GE 1.0d/86400.d -
+; Version 4.0b46
+; 20190807: Allow .nc and .nc4 filename extensions when doing QA on a file that has been read
+; into session memory using the HDF5 routines (idlcr8ascii set up to use H5 routines
+; to read netCDF4 files as well as HDF5) - Version 4.0b51
+; 20190824: Fix bug in 4.0b51 that caused the file name to be written to the DATA_FILE_VERSION
+; attribute - Version 4.0b52
+; 20231218: Always recalculate FILE_GENERATION_DATE if creating a GEOMS file so that it aligns
+; with the FILE_META_VERSION - Version 4.0b61
+; 20241029: Add extra resolution when checking for discrepancies between the first and last
+; DATETIME[.START][.STOP] values and DATA_[START|STOP]_DATE (was 0.99d/86400.d and
+; now 0.9999d/86400.d). This should be equivalent to the NDACC QA criteria, which
+; checks against a floored and ceiling second value - Version 4.0b65
+;
+; Inputs: meta_arr - a string array containing the Global and Variable Attributes
+; hfdfilename - a string holding the extension of the eventual HDF filename, either
+; '.hdf' for HDF4 or '.h5' for HDF5 or '.nc' for netCDF3/4
+;
+; Outputs: meta_arr - DATA_START_DATE, FILE_GENERATION_DATE and FILE_NAME values will be
+; (re)written to the array based on values computed by the routine
+; hdffilename - a string holding the full HDF file name determined by the routine
+;
+; Called by: IDLCR8HDF
+;
+; Subroutines Called: STOP_WITH_ERROR (if error state detected); INFOTXT_OUTPUT
+; Possible Conditions for STOP_WITH_ERROR call (plus [line number] where called):
+; 1. An incorrect number of sub-values found in meta_arr for DATA_DISCIPLINE,
+; DATA_SOURCE, DATA_LOCATION, or DATA_FILE_VERSION [3129]
+; 2. Error encountered when converting ISO8601 datetime values to MJD2000 format [3138,3179]
+; 3. Type conversion error encountered when a datetime value is expected to be in
+; MJD2000 format, but is not numeric [3216]
+;
+; Information Conditions (when the program is able to make changes):
+; 1. DATA_START_DATE or DATA_STOP_DATE changed to match relevant DATETIME entries [3167]
+; 2. Filename determined from DATASETS attributes [3208]
+
+COMMON TABLEDATA
+COMMON METADATA
+COMMON DATA
+COMMON WIDGET_WIN
+
+;Possible error messages for this procedure
+ON_IOERROR,TypeConversionError
+errtxt=STRARR(4) & lu=-1L
+proname='Find_HDF_Filename procedure: '
+errtxt[0]='Number of sub-values under DATA_'
+errtxt[1]=' should be '
+errtxt[2]=' ISO8601 format not valid: '
+errtxt[3]='Type conversion error. MJD2K entry is not valid: '
+
+fni=WHERE(attr_arr_glob EQ 'FILE_NAME')
+res=STRSPLIT(meta_arr[fni[0]],' =',/Extract)
+IF N_ELEMENTS(res) EQ 1 THEN res=[res,'-1']
+fn_val=res[1]
+
+IF (qa_yes) AND (hdffilename EQ '.h5') AND (fn_val NE '-1') THEN BEGIN
+ ;netCDF4 files read in using H5 routines so .h5 extension is assumed. Allow files to
+ ;also have .nc or .nc4 extensions for files being QA'd
+ hext=['.h5','.nc','.nc4']
+ fn_ext=STRMID(fn_val,STRPOS(fn_val,'.',/REVERSE_SEARCH))
+ ei=WHERE(fn_ext EQ hext,ecnt)
+ IF ecnt NE 0 THEN hdffilename=hext[ei[0]]
+ENDIF
+
+fhold='' & iso='' & mjd2000=0.d
+dtmjd=0.d & dtiso='' & itxt=' '
+dt=['DISCIPLINE','SOURCE','LOCATION','START_DATE','STOP_DATE','FILE_GENERATION_DATE','FILE_VERSION']
+valid=0 ;test for Type conversion errors
+FOR i=0,N_ELEMENTS(dt)-1 DO BEGIN
+ IF i EQ 0 THEN nei=4 ELSE nei=2
+ IF i EQ 5 THEN ai=WHERE(attr_arr_glob EQ dt[i]) $
+ ELSE ai=WHERE(attr_arr_glob EQ 'DATA_'+dt[i])
+
+ res=STRSPLIT(meta_arr[ai[0]],' =;',/Extract,COUNT=cres)
+ IF (cres NE nei) AND ((i LE 2) OR (i GE 6)) THEN BEGIN
+ ;infotxt='3 '+errtxt[0]+dt[i]+errtxt[1]+nes
+ ;INFOTXT_OUTPUT,infotxt
+ STOP_WITH_ERROR,o3[3]+proname+meta_arr[ai[0]]+': ',errtxt[0]+dt[i]+errtxt[1]+STRTRIM(nei-1,2),lu
+ RETURN
+ ENDIF
+
+ IF i EQ 6 THEN BEGIN ;Do DATA_FILE_VERSION checks
+ dfvv=res[1]
+ GEOMS_RULE_CHANGES,4,dfvv
+ res[1]=dfvv & meta_arr[ai[0]]='DATA_FILE_VERSION='+dfvv
+ ENDIF
+
+ IF (i EQ 3) OR (i EQ 4) THEN BEGIN ;calculate the ISO or MJD2000 value as required
+ ;Find lowest or highest DATETIME values from Variable Attributes
+ IF i EQ 3 THEN BEGIN
+ di=WHERE(vn EQ 'DATETIME.START',dcnt)
+ itxt= ' first ' & etxt='DATETIME.START '
+ ENDIF ELSE BEGIN
+ di=WHERE(vn EQ 'DATETIME.STOP',dcnt)
+ itxt= ' last ' & etxt='DATETIME.STOP '
+ ENDELSE
+ IF dcnt NE 0 THEN BEGIN
+ darr=*ds[di[0]].data ;extract the DATETIME values
+ ;remove any fill values
+ fi=WHERE(darr NE mv_dbl[2,di[0]],dcnt)
+ ENDIF
+ IF dcnt EQ 0 THEN BEGIN
+ di=WHERE(vn EQ 'DATETIME',dcnt) & etxt='DATETIME '
+ IF dcnt NE 0 THEN BEGIN
+ darr=*ds[di[0]].data ;extract the DATETIME values
+ ;remove any fill values
+ fi=WHERE(darr NE mv_dbl[2,di[0]],dcnt)
+ ENDIF
+ ENDIF
+ IF dcnt NE 0 THEN BEGIN
+ darr=darr[fi]
+ IF N_ELEMENTS(darr) EQ 1 THEN itxt=' '
+ IF i EQ 3 THEN dtmjd=MIN(darr) ELSE dtmjd=MAX(darr)
+ dtiso=JDF_2_DATETIME(dtmjd,/M,/S) ;returns datetime in ISO8601 format
+ dtisochk=0B ;boolean to check for 60 as seconds value
+ ;Extract DATA_START/STOP_DATE from the Global Attributes, if present
+ IF N_ELEMENTS(res) EQ nei THEN BEGIN ;a date/time value is present
+ IF STRPOS(STRUPCASE(res[1]),'Z') NE -1 THEN BEGIN
+ IF (STRPOS(res[1],'Z') EQ -1) OR (STRPOS(res[1],'T') EQ -1) THEN BEGIN ;ISO8601 string must be in upper case
+ IF qa_yes then qtxt=' must be ' ELSE qtxt=' made '
+ infotxt='2 '+res[0]+' entry'+qtxt+'uppercase for ISO8601 compliance'
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ res[1]=STRUPCASE(res[1])
+ mjd2000=JULIAN_DATE(res[1],/I,/M) ;return ISO8601 time in MJD2000 format
+ IF mjd2000 EQ -99999.d THEN BEGIN ;conversion error
+ infotxt='3 '+res[0]+errtxt[2]+res[1]
+ INFOTXT_OUTPUT,infotxt
+ ;STOP_WITH_ERROR,o3[3]+proname,res[0]+errtxt[2]+res[1],lu,ds & RETURN
+ ENDIF
+ dtisochk=STRMID(res[1],13,2) EQ '60' ;existing time in ISO8601 format
+ ENDIF ELSE BEGIN
+ mjd2000=DOUBLE(res[1])
+ iso=JDF_2_DATETIME(mjd2000,/M,/S) ;return MJD2000 time in ISO8601 format
+ res[1]=iso
+ IF qa_yes THEN qtxt=' is not in ' ELSE qtxt= ' converted to '
+ infotxt='2 '+meta_arr[ai[0]]+qtxt+'ISO8601 format'
+ INFOTXT_OUTPUT,infotxt
+ ENDELSE
+ res1=res[1]
+ ENDIF ELSE BEGIN
+ mjd2000=-99999.0D & res1='-99999.0'
+ ENDELSE
+ ;Check that DATA_START_DATE matches the first value under DATETIME.START or DATETIME
+ ;and DATA_STOP_DATE matches the last value under DATETIME.STOP or DATETIME (if present)
+ IF (ABS(mjd2000-dtmjd) GT 0.9999D/86400.D) OR ((dtisochk) AND (dtiso NE res1)) THEN BEGIN
+ IF N_ELEMENTS(res) EQ nei THEN BEGIN
+ IF qa_yes THEN qtxt=' must match' ELSE qtxt=' changed to match'
+ infotxt='2 Date in DATA_'+dt[i]+qtxt+itxt+etxt+'entry: '+res[1]+' -> '+dtiso
+ ENDIF ELSE BEGIN
+ IF qa_yes THEN qtxt='should be '+dtiso ELSE qtxt=dtiso+' added'
+ infotxt='2 Missing DATA_'+dt[i]+' value '+qtxt+' based on'+itxt+etxt+'entry'
+ ENDELSE
+ INFOTXT_OUTPUT, infotxt
+ res=[res[0],dtiso] ;new ISO time for filename
+ ENDIF
+ meta_arr[ai[0]]=res[0]+'='+res[1] ;rewrite (correct) value in meta_arr in ISO8601 format
+ ENDIF ELSE IF N_ELEMENTS(res) EQ 1 THEN res=[res,'[MISSING]']
+ ENDIF
+ IF i EQ 5 THEN BEGIN
+ ;check format of FILE_GENERATION_DATE value and create/change if necessary
+ IF ~qa_yes THEN BEGIN
+ ;Not doing QA so calculate new FILE_GENERATION_DATE of file creation so it aligns with FILE_META_VERSION (added 20231218)
+ mjd2000=SYSTIME(/Julian,/UTC)-2451544.5D
+ mjds=JDF_2_DATETIME(mjd2000,/M,/S) ;returns datetime in ISO8601 format
+ meta_arr[ai[0]]=res[0]+'='+mjds ;write date in ISO8601 format
+ infotxt='0 '+res[0]+' updated with current system time'
+ INFOTXT_OUTPUT,infotxt
+ IF cres EQ 1 THEN res=[res,mjds] ELSE res[1]=mjds
+ ENDIF ELSE BEGIN
+ IF N_ELEMENTS(res) EQ 1 THEN res=[res,'-1']
+ IF STRPOS(STRUPCASE(res[1]),'Z') NE -1 THEN BEGIN
+ IF (STRPOS(res[1],'Z') EQ -1) OR (STRPOS(res[1],'T') EQ -1) THEN BEGIN ;ISO8601 string must be in upper case
+ IF qa_yes THEN qtxt=' must be ' ELSE qtxt=' made '
+ infotxt='2 '+res[0]+' entry'+qtxt+'uppercase for ISO8601 compliance'
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ mjd2000=JULIAN_DATE(res[1],/I,/M) ;return time in MJD2000 format
+ IF mjd2000 EQ -99999.d THEN res=[res[0],'-2'] $ ;conversion error
+ ELSE IF STRMID(res[1],13,2) EQ '60' THEN res=[res[0],'-3'] $
+ ELSE meta_arr[ai[0]]=res[0]+'='+STRTRIM(STRUPCASE(res[1]),2) ;rewrite date in ISO8601 format
+ ENDIF
+ IF STRPOS(STRUPCASE(res[1]),'Z') EQ -1 THEN BEGIN
+ ;FILE_GENERATION_DATE not present or not in ISO8601 format so recalculate
+ mjd2000=SYSTIME(/Julian,/UTC)-2451544.5D
+ mjds=JDF_2_DATETIME(mjd2000,/M,/S) ;returns datetime in ISO8601 format
+ meta_arr[ai[0]]=res[0]+'='+mjds ;write date in ISO8601 format
+ IF res[1] EQ '-1' THEN itxt='not present' $
+ ELSE IF res[1] EQ '-3' THEN itxt='must have seconds value between 0 and 59' $
+ ELSE itxt='not in ISO8601 format'
+ infotxt='2 '+res[0]+' value '+itxt+'|. Recalculated using the system time'
+ INFOTXT_OUTPUT,infotxt
+ res[1]=mjds
+ ENDIF
+ ENDELSE
+ ;Test to see if DATA_STOP_DATE is greater than FILE_GENERATION_DATE
+ IF dtmjd GT mjd2000 THEN BEGIN
+ infotxt='3 Calculated DATA_STOP_DATE='+dtiso+' is later than '+meta_arr[ai[0]]
+ INFOTXT_OUTPUT,infotxt
+ ENDIF
+ ENDIF
+ IF i NE 5 THEN BEGIN
+ ;i.e. don't include file generation date in the filename
+ IF fhold EQ '' THEN fhold=fhold+STRLOWCASE(res[nei-1]) $
+ ELSE fhold=fhold+'_'+STRLOWCASE(res[nei-1])
+ ENDIF
+ENDFOR
+hdffilename=fhold+hdffilename
+
+;check to see if the resulting filename is the same as that under FILE_NAME
+IF hdffilename NE fn_val THEN BEGIN
+ IF fn_val NE '-1' THEN BEGIN
+ infotxt=STRARR(3)
+ infotxt[0]='2 FILE_NAME entry under File Attributes does not match the filename'
+ infotxt[0]=infotxt[0]+' determined from the Global Attributes'
+ infotxt[1]=' '+fn_val+' -> '+hdffilename+'|'
+ infotxt[2]=' Filename determined from Global Attributes used'
+ ENDIF ELSE BEGIN
+ IF qa_yes THEN qtxt='should be '+hdffilename ELSE qtxt=hdffilename+' added'
+ infotxt=STRARR(2)
+ infotxt[0]='2 Missing FILE_NAME value '+qtxt
+ infotxt[1]=' based on Global Attribute values'
+ ENDELSE
+ INFOTXT_OUTPUT, infotxt
+ meta_arr[fni[0]]='FILE_NAME='+hdffilename ;change FILE_NAME entry
+ENDIF
+valid=1 ;Type conversions performed OK
+
+TypeConversionError:
+IF valid EQ 0 THEN BEGIN
+ infotxt='3 '+errtxt[3]+res[1]
+ INFOTXT_OUTPUT, infotxt
+ ;STOP_WITH_ERROR,o3[3]+proname+res[0]+': ',errtxt[3]+res[1],lu & RETURN
+ENDIF
+
+END ;Procedure Find_HDF_Filename
+
+
+
+PRO makean, hdf_fn, obj_id, obj_type, label, value
+;Procedure to read the contents of a file given as an attribute value and write
+;it directly to the HDF Scientific Dataset or to the annotation file description
+;section (depending on its size). Currently set up to always write the contents
+;of the file to HDF_SD as the text file size limit is also 4096 bytes.
+; ----------
+;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
+;
+; History:
+; 20050802: Original IDLCR8HDF routine - Version 1.0
+;
+; Inputs: hdf_fn - The file handle identifier for the HDF file
+; obj_id - The Scientific Dataset (SD) identifier of the HDF file or a newly
+; created dataset
+; obj_type - a string identifying the type of annotation (set to 'FILE')
+; label - a string containing the Attribute label from the metadata
+; value - a string containing the Attribute value from the metadata
+;
+; Output: Nil
+;
+; Called by: AVDC_HDF_WRITE
+;
+; Subroutines Called: None
+
+nchar=4096 ;strlen maximum for writing directly to SD (instead of AN)
+
+;Extract name of file
+pos1=STRPOS(value,'"')
+pos2=STRPOS(value,'"',/REVERSE_SEARCH)
+data_notes_file=STRMID(value,pos1+1,pos2-pos1-1)
+OPENR,lu,data_notes_file,/GET_LUN
+
+;read each line and count number of characters in file
+dum='' & text_in_file=''
+i=0
+WHILE NOT EOF(lu) DO BEGIN
+ READF,lu,dum
+ text_in_file=text_in_file+dum+STRING(10B) ;+STRING(13B)
+ i=i+1
+ENDWHILE
+FREE_LUN,lu
+n1=STRLEN(text_in_file)
+
+;The text-buffer contains n1 characters which will be written either
+;to the annotation file description section or directly to the HDF file as
+;the value of the current attribute
+IF n1 LE nchar THEN HDF_SD_ATTRSET,obj_id,label,text_in_file,n1,/DFNT_CHAR $
+ELSE BEGIN
+ ;open the HDF output file for AN write operations
+ ;Initialize the HDF AN interface for the specified file
+ an_id=HDF_AN_START(hdf_fn)
+
+ IF obj_type EQ 'FILE' THEN BEGIN
+ ;Get id for the file label annotation
+ an_labl_id=HDF_AN_CREATEF(an_id,2)
+ ;Get id for the file description annotation
+ an_desc_id=HDF_AN_CREATEF(an_id,3)
+ ENDIF ELSE BEGIN
+ ;Get id for the data object label annotation
+ an_labl_id=HDF_AN_CREATE(an_id,DFTAG_NDG,HDF_SD_IDTOREF(obj_id),0)
+ ;Get id for the data object description annoatation
+ an_desc_id=HDF_AN_CREATE(an_id,DFTAG_NDG,HDF_SD_IDTOREF(obj_id),1)
+ ENDELSE
+
+ ;Write the label annotation
+ status=HDF_AN_WRITEANN(an_labl_id,label)
+ ;Terminate access to the label annotation
+ HDF_AN_ENDACCESS,an_labl_id
+
+ ;Write the description annotation
+ status=HDF_AN_WRITEANN(an_desc_id,text_in_file)
+ ;Terminate access to the description annotation
+ HDF_AN_ENDACCESS,an_desc_id
+
+ ;Terminate access to the AN interface
+ HDF_AN_END,an_id
+ENDELSE
+
+END ;Procedure MakeAN
+
+
+
+PRO avdc_hdf5_write, hdffilename, natts, av, geoms_output
+;IDL subroutine to write AVDC-type global attributes and datasets to the Root
+;Group in an HDF5 file.
+;##########################################################################
+;Note: In versions of IDL earlier than 6.2, this procedure may not compile
+;and the HDF5 write option will not be available.
+;##########################################################################
+; ----------
+;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
+;
+; History:
+; 20061012: Introduced to IDLCR8HDF - Version 2.0
+; 20080302: Removed portion of the code which created the DIMENSIONLIST attribute
+; containing the object reference pointers to the variables referred to
+; in the VAR_DEPEND values - Version 3.0
+; 20150127: For string datasets, change the VAR_FILL_VALUE to an empty string that
+; is the same length as the individual strings - Version 4.0b25
+; 20150217: Change rule from 4.0b25 so the VAR_FILL_VALUE is an empty string; For
+; string datasets do a STRTRIM of the dataset when writing to the file, as
+; the first dataset entry may have had whitespace added so that the
+; maximum dataset length is correctly determined - Version 4.0b26
+; 20181221: Allow compression and shuffle options when creating HDF5 files
+; - Version 4.0b49
+;
+; Inputs: hdffilename - a string holding the name of the HDF file being created
+; and written to
+; natts - an integer giving the number of Variable Attributes in each
+; dataset (i.e. the number of elements in attr_arr_data)
+; av - a string array of size (nvn,natts+7) containing the attribute and
+; NCSA values for each dataset
+; ds - structure containing the data
+; meta_arr - a string array containing the Global and Variable Attributes
+; vn - a string array containing the VAR_NAME values
+; ndm - a long array describing the dimensions of each dataset and the
+; VAR_DATA_TYPE
+;
+; Outputs: An HDF5 file with the name hdffilename
+;
+; Called by: AVDC_HDF_WRITE
+;
+; Subroutines Called: None
+
+COMMON METADATA
+COMMON DATA
+
+;Check for compression setting
+comp_fac=0
+IF STRLEN(geoms_output) EQ 4 THEN BEGIN
+ val_chk=STRMID(geoms_output,3,1)
+ IF IS_A_NUMBER_HDF(val_chk) THEN comp_fac=FIX(val_chk)
+ENDIF
+
+;Create an HDF5 file
+hdf_file_id=H5F_CREATE(hdffilename)
+H5F_CLOSE, hdf_file_id
+
+;open the HDF5 file for writing.
+hdf_file_id=H5F_OPEN(hdffilename,/WRITE)
+;The H5G_OPEN function opens an existing group within an HDF5 file
+sd_id=H5G_OPEN(hdf_file_id,'/')
+
+;Write the Global Attributes to file
+;Section 4.1 (Bojkov et al., 2002): Originator attributes
+;Section 4.2 (Bojkov et al., 2002): Dataset attributes
+;Section 4.3 (Bojkov et al., 2002): File attributes
+FOR i=0,N_ELEMENTS(attr_arr_glob)-1 DO BEGIN
+ ;separate out the Meta Attribute entry
+ eqpos=STRPOS(meta_arr[i],'=')
+ IF eqpos EQ STRLEN(meta_arr[i])-1 THEN attribute_value=' ' $
+ ELSE attribute_value=STRMID(meta_arr[i],eqpos+1)
+ ;get attribute type and space, needed to create the attribute
+ atype_id=H5T_IDL_CREATE(attribute_value)
+ aspace_id=H5S_CREATE_SCALAR()
+ ;create attribute in the output file
+ aset_id=H5A_CREATE(sd_id,attr_arr_glob[i],atype_id,aspace_id)
+ ;write attribute to dataset
+ H5A_WRITE,aset_id,attribute_value
+ ;close all open identifiers
+ H5A_CLOSE,aset_id
+ H5S_CLOSE,aspace_id
+ H5T_CLOSE,atype_id
+ENDFOR
+
+;Write the Datasets (SDS) to file
+;Section 5.1 (Bojkov et al, 2002): Variable description attributes.
+al=STRARR(natts)
+al[0:natts-1]=attr_arr_data
+FOR i=0,nvn-1 DO BEGIN
+ CASE ndm[i,8] OF ;identifies the VAR_DATA_TYPE
+ 0:BEGIN
+ valid_range=[BYTE(mv_lng[0,i]),BYTE(mv_lng[1,i])]
+ fill_value=BYTE(mv_lng[2,i])
+ END
+ 1:BEGIN
+ valid_range=[FIX(mv_lng[0,i]),FIX(mv_lng[1,i])]
+ fill_value=FIX(mv_lng[2,i])
+ END
+ 2:BEGIN
+ valid_range=[LONG(mv_lng[0,i]),LONG(mv_lng[1,i])]
+ fill_value=LONG(mv_lng[2,i])
+ END
+ 3:BEGIN
+ valid_range=[mv_lng[0,i],mv_lng[1,i]]
+ fill_value=mv_lng[2,i]
+ END
+ 4:BEGIN
+ valid_range=[FLOAT(mv_dbl[0,i]),FLOAT(mv_dbl[1,i])]
+ fill_value=FLOAT(mv_dbl[2,i])
+ END
+ 5:BEGIN
+ valid_range=[mv_dbl[0,i],mv_dbl[1,i]]
+ fill_value=mv_dbl[2,i]
+ END
+ 6:BEGIN
+ valid_range=[' ',' ']
+ fill_value=' ' ;STRING(format='(A-'+STRTRIM(mv_str[i],2)+')','') ;bytarr(mv_str[i])
+ END
+ ENDCASE
+
+ ;Make up data array for writing to HDF5 file
+ data=*ds[i].data
+ ;may have lost array VAR_SIZE information if the last value is '1' e.g. 41,1
+ gi=WHERE(ndm[i,0:7] NE 0L) & vs=TRANSPOSE(ndm[i,gi]) ;transpose to get array values in a vector
+ ;get dataset type and space, needed to create the dataset
+ dtype_id=H5T_IDL_CREATE(data)
+ dspace_id=H5S_CREATE_SIMPLE(vs)
+ ;create dataset in the output file
+ IF comp_fac EQ 0 THEN dset_id=H5D_CREATE(sd_id,vn[i],dtype_id,dspace_id) $
+ ELSE BEGIN
+ ;Perform compression and shuffling of the dataset
+ dset_id=H5D_CREATE(sd_id,vn[i],dtype_id,dspace_id,CHUNK_DIMENSIONS=vs,GZIP=comp_fac,/SHUFFLE)
+ ENDELSE
+ ;write data to dataset - for string datasets remove unwanted white space
+ IF ndm[i,8] EQ 6 THEN H5D_WRITE,dset_id,STRTRIM(data,2) ELSE H5D_WRITE,dset_id,data
+
+ ;write out variable attributes
+ ninc=0 & j=0
+ mi=WHERE(meta_arr EQ attr_arr_data[0]+'='+vn[i])
+ WHILE j LE natts-1 DO BEGIN
+ ma=mi[0]+ninc
+ ;Extract Attribute Label
+ eqpos=STRPOS(meta_arr[ma],'=')
+ res=STRMID(meta_arr[ma],0,eqpos)
+ WHILE res[0] NE al[j] DO j=j+1 ;in case of missing optional variable attributes
+ CASE 1 OF
+ ;attr_arr_data[j] EQ 'VAR_SIZE': athold=vs
+ attr_arr_data[j] EQ 'VAR_VALID_MIN': athold=valid_range[0]
+ attr_arr_data[j] EQ 'VAR_VALID_MAX': athold=valid_range[1]
+ attr_arr_data[j] EQ 'VAR_FILL_VALUE': athold=fill_value
+ ELSE: athold=av[i,j]
+ ENDCASE
+ ;get attribute type and space, needed to create the attribute
+ atype_id=H5T_IDL_CREATE(athold)
+ IF N_ELEMENTS(athold) EQ 1 THEN aspace_id=H5S_CREATE_SCALAR() $
+ ELSE aspace_id=H5S_CREATE_SIMPLE(N_ELEMENTS(athold))
+ ;create attribute in the output file
+ aset_id=H5A_CREATE(dset_id,al[j],atype_id,aspace_id)
+ ;write attribute to dataset
+ H5A_WRITE,aset_id,athold
+ ;close all open attribute identifiers
+ H5A_CLOSE,aset_id
+ H5S_CLOSE,aspace_id
+ H5T_CLOSE,atype_id
+ ninc=ninc+1 & j=j+1
+ ENDWHILE
+ ;close all open dataset identifiers
+ H5D_CLOSE,dset_id
+ H5S_CLOSE,dspace_id
+ H5T_CLOSE,dtype_id
+ENDFOR
+
+;The H5G_CLOSE procedure closes the specified group and releases resources used by it.
+H5G_CLOSE,sd_id
+;The H5F_CLOSE procedure closes the HDF file associated with the given file handle.
+H5F_CLOSE,hdf_file_id
+
+END ;Procedure AVDC_HDF5_Write
+
+
+
+PRO avdc_nc_write, hdffilename, natts, av, var_depend, var_size
+;IDL subroutine to write AVDC-type global attributes and datasets to a netCDF file.
+; ----------
+;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
+;
+; History:
+; 20111120: Introduced to IDLCR8HDF - Version 4.0
+; 20130116: Account for string datasets which contain only empty string values (make
+; string length = 1) - Version 4.0b15
+; 20150127: For string datasets, change the VAR_FILL_VALUE to an empty string that
+; is the same length as the individual strings - Version 4.0b25
+; 20150217: Change rule from 4.0b25 so the VAR_FILL_VALUE is an empty string
+; - Version 4.0b26
+; 20161230: Assign a dimension ID to STRING datasets with VAR_DEPEND=CONSTANT so that
+; the string length is correctly accounted for (previously only wrote the
+; first character of the string) - Version 4.0b41
+;
+; Inputs: hdffilename - a string holding the name of the netCDF file being created
+; and written to
+; natts - an integer giving the number of Variable Attributes in each
+; dataset (i.e. the number of elements in attr_arr_data)
+; av - a string array of size (nvn,natts+7) containing the attribute and
+; NCSA values for each dataset
+; var_depend - a string array of size (nvn) holding the VAR_DEPEND attribute
+; values, used to assign DIM information
+; var_size - a string array of size (nvn) holding the VAR_SIZE attribute
+; values, used to assign DIM information
+; ds - structure containing the data
+; meta_arr - a string array containing the Global and Variable Attributes
+; vn - a string array containing the VAR_NAME values
+; ndm - a long array describing the dimensions of each dataset and the
+; VAR_DATA_TYPE
+;
+; Outputs: A netCDF file with the name hdffilename
+;
+; Called by: AVDC_HDF_WRITE
+;
+; Subroutines Called: None
+
+COMMON METADATA
+COMMON DATA
+
+;Create a netCDF file
+df_file_id=NCDF_CREATE(hdffilename,/CLOBBER)
+
+;Write the Global Attributes to file
+;Section 4.1 (Bojkov et al., 2002): Originator attributes
+;Section 4.2 (Bojkov et al., 2002): Dataset attributes
+;Section 4.3 (Bojkov et al., 2002): File attributes
+FOR i=0,N_ELEMENTS(attr_arr_glob)-1 DO BEGIN
+ ;separate out the Meta Attribute entry
+ eqpos=STRPOS(meta_arr[i],'=')
+ IF eqpos EQ STRLEN(meta_arr[i])-1 THEN attribute_value=' ' $
+ ELSE attribute_value=STRMID(meta_arr[i],eqpos+1)
+ ;get attribute type and space, needed to create the attribute
+ NCDF_ATTPUT,df_file_id,/GLOBAL,attr_arr_glob[i],attribute_value
+ENDFOR
+
+;Write the Datasets (SDS) to file
+;Section 5.1 (Bojkov et al, 2002): Variable description attributes.
+al=STRARR(natts)
+al[0:natts-1]=attr_arr_data
+vdh=['CONSTANT','INDEPENDENT']
+dim_id_list=LONARR(2)-999L
+constfound=0
+
+FOR i=0,nvn-1 DO BEGIN
+
+ ;Make up data array for writing to netCDF file
+ data=*ds[i].data
+ ;may have lost array VAR_SIZE information if the last value is '1' e.g. 41,1
+ gi=WHERE(ndm[i,0:7] NE 0L) & vs=TRANSPOSE(ndm[i,gi]) ;transpose to get array values in a vector
+
+ ;Determine Dimension IDs for the Variable
+ vdl=STRSPLIT(var_depend[i],' ;',/EXTRACT,COUNT=vdcount)
+ vsl=STRSPLIT(var_size[i],' ;',/EXTRACT) & vsl=LONG(vsl)
+ IF ndm[i,8] EQ 6 THEN BEGIN ;need to include string length dimension variable
+ vdcount++
+ IF MAX(STRLEN(data)) EQ 0 THEN mxstrlen=1 ELSE mxstrlen=MAX(STRLEN(data))
+ vdl=['STRLEN',vdl] & vsl=[mxstrlen,vsl]
+ ENDIF
+ dims=LONARR(vdcount)-999L & const=0
+ FOR j=0,vdcount-1 DO BEGIN
+ ndi=WHERE(STRUPCASE(vdl[j]) EQ vdh,ndc)
+ IF vdl[j] EQ 'STRLEN' THEN BEGIN
+ ;dataset is of type string so add string length dimension variable to vdh array
+ dim_id=NCDF_DIMDEF(df_file_id,vn[i]+'_STRLEN',vsl[j])
+ vdh=[vdh,STRUPCASE(vn[i])+'_STRLEN'] ;add dimension variable to vdh array
+ dim_id_list=[dim_id_list,dim_id]
+ dims[j]=dim_id
+ ENDIF ELSE IF ndc EQ 0 THEN BEGIN ;new dimension found for file
+ vdh=[vdh,STRUPCASE(vdl[j])] ;add dimension variable to vdh array
+ dim_id=NCDF_DIMDEF(df_file_id,vdl[j],vsl[j])
+ dim_id_list=[dim_id_list,dim_id]
+ dims[j]=dim_id
+ ENDIF ELSE BEGIN ;dimension already has ID or is CONSTANT or INDEPENDENT
+ IF ndi[0] EQ 0 THEN BEGIN
+ ;CONSTANT so no dimension is assigned
+ IF constfound EQ 0 THEN BEGIN
+ dim_id=NCDF_DIMDEF(df_file_id,'CONSTANT',1)
+ dim_id_list[0]=dim_id ;assign a dim_id for CONSTANT - needed for STRING datasets
+ constfound=1
+ ENDIF
+ const=1
+ dims[j]=dim_id_list[0]
+ ENDIF ELSE IF ndi[0] EQ 1 THEN BEGIN
+ ;INDEPENDENT or BOUNDARIES so assign INDEPENDENT/BOUNDARIES_%vs as dimension name
+ IF (j EQ 1) AND (STRPOS(vn[i],'.BOUNDARIES') NE -1) THEN itxt='BOUNDARIES_' $
+ ELSE itxt='INDEPENDENT_'
+ dim_id=NCDF_DIMDEF(df_file_id,itxt+STRTRIM(vsl[j],2),vsl[j])
+ vdh=[vdh,itxt+STRTRIM(vsl[j],2)] ;add dimension variable to vdh array
+ dim_id_list=[dim_id_list,dim_id]
+ dims[j]=dim_id
+ ENDIF ELSE dims[j]=dim_id_list[ndi[0]]
+ ENDELSE
+ ENDFOR
+
+ ;If multi-dimensional reverse dimension IDs as the dataset is automatically transposed
+ IF N_ELEMENTS(vs) GT 1 THEN dims=REVERSE(dims)
+
+ CASE ndm[i,8] OF ;identifies the VAR_DATA_TYPE
+ 0:BEGIN
+ valid_range=[BYTE(mv_lng[0,i]),BYTE(mv_lng[1,i])]
+ fill_value=BYTE(mv_lng[2,i])
+ IF const THEN var_id=NCDF_VARDEF(df_file_id,vn[i],/BYTE) $
+ ELSE var_id=NCDF_VARDEF(df_file_id,vn[i],dims,/BYTE)
+ END
+ 1:BEGIN
+ valid_range=[FIX(mv_lng[0,i]),FIX(mv_lng[1,i])]
+ fill_value=FIX(mv_lng[2,i])
+ IF const THEN var_id=NCDF_VARDEF(df_file_id,vn[i],/SHORT) $
+ ELSE var_id=NCDF_VARDEF(df_file_id,vn[i],dims,/SHORT)
+ END
+ 2:BEGIN
+ valid_range=[LONG(mv_lng[0,i]),LONG(mv_lng[1,i])]
+ fill_value=LONG(mv_lng[2,i])
+ IF const THEN var_id=NCDF_VARDEF(df_file_id,vn[i],/LONG) $
+ ELSE var_id=NCDF_VARDEF(df_file_id,vn[i],dims,/LONG)
+ END
+ 3:BEGIN
+ valid_range=[mv_lng[0,i],mv_lng[1,i]]
+ fill_value=mv_lng[2,i]
+ IF const THEN var_id=NCDF_VARDEF(df_file_id,vn[i],/LONG) $
+ ELSE var_id=NCDF_VARDEF(df_file_id,vn[i],dims,/LONG)
+ END
+ 4:BEGIN
+ valid_range=[FLOAT(mv_dbl[0,i]),FLOAT(mv_dbl[1,i])]
+ fill_value=FLOAT(mv_dbl[2,i])
+ IF const THEN var_id=NCDF_VARDEF(df_file_id,vn[i],/FLOAT) $
+ ELSE var_id=NCDF_VARDEF(df_file_id,vn[i],dims,/FLOAT)
+ END
+ 5:BEGIN
+ valid_range=[mv_dbl[0,i],mv_dbl[1,i]]
+ fill_value=mv_dbl[2,i]
+ IF const THEN var_id=NCDF_VARDEF(df_file_id,vn[i],/DOUBLE) $
+ ELSE var_id=NCDF_VARDEF(df_file_id,vn[i],dims,/DOUBLE)
+ END
+ 6:BEGIN
+ valid_range=[' ',' ']
+ fill_value=' ' ;bytarr(mv_str[i]) ;STRING(format='(A-'+STRTRIM(mv_str[i],2)+')','')
+ var_id=NCDF_VARDEF(df_file_id,vn[i],dims,/CHAR)
+ END
+ ENDCASE
+
+ ;write out variable attributes
+ ninc=0 & j=0
+ mi=WHERE(meta_arr EQ attr_arr_data[0]+'='+vn[i])
+ WHILE j LE natts-1 DO BEGIN
+ ma=mi[0]+ninc
+ ;Extract Attribute Label
+ eqpos=STRPOS(meta_arr[ma],'=')
+ res=STRMID(meta_arr[ma],0,eqpos)
+ WHILE res[0] NE al[j] DO j=j+1 ;in case of missing optional variable attributes
+ CASE 1 OF
+ attr_arr_data[j] EQ 'VAR_VALID_MIN': athold=valid_range[0]
+ attr_arr_data[j] EQ 'VAR_VALID_MAX': athold=valid_range[1]
+ attr_arr_data[j] EQ 'VAR_FILL_VALUE': athold=fill_value
+ ELSE: athold=av[i,j]
+ ENDCASE
+ NCDF_ATTPUT,df_file_id,var_id,attr_arr_data[j],athold
+ ninc=ninc+1 & j=j+1
+ ENDWHILE
+
+ IF ndm[i,8] NE 6 THEN BEGIN
+ NCDF_ATTPUT,df_file_id,var_id,'units',vu[i]
+ NCDF_ATTPUT,df_file_id,var_id,'valid_range',valid_range
+ NCDF_ATTPUT,df_file_id,var_id,'_Fillvalue',fill_value
+ ENDIF
+
+ NCDF_CONTROL,df_file_id,/ENDEF ;Enter Data Mode
+ NCDF_VARPUT,df_file_id,var_id,data ;Write Data to file
+ NCDF_CONTROL,df_file_id,/REDEF ;Enter Define Mode
+ENDFOR
+
+NCDF_CLOSE,df_file_id
+
+END ;Procedure AVDC_NC_Write
+
+
+
+PRO avdc_hdf_write, hdffilename, geoms_output
+;IDL subroutine to write AVDC-type global attributes and datasets to a Scientific
+;Dataset (SDS) in HDF4 (Bojkov et al.,2004) or HDF5. This routine creates the HDF4
+;file if requested, and calls the AVDC_HDF5_WRITE routine to create the HDF5 file if
+;that option is chosen.
+; ----------
+;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
+;
+; History:
+; 20050802: Originally split into two routines to separately write the Global
+; attributes (AVDC_HDF_GLOBAL_ATTRIBUTES) and the Variable attributes
+; and data (AVDC_HDF_SDS_VARIABLES) to an HDF4 file - Version 1.0
+; 20061012: Two routines combined into the AVDC_HDF_WRITE routine; Call to the
+; AVDC_HDF5_WRITE routine if HDF5 file option is chosen; HDF4 file not
+; closed between writing the global and variable attributes to file;
+; Data now written to file from the ds structure instead of from arrays
+; made according to VAR_DATA_TYPE; the av array made to hold attribute
+; values to write to the HDF file, instead of writing the values to
+; file directly from the meta_arr array (to avoid duplication of code
+; in AVDC_HDF5_WRITE) - Version 2.0
+; 20130116: Account for string datasets which contain only empty string values (make
+; string length = 1) - Version 4.0b15
+; 20150127: For string datasets, change the VAR_FILL_VALUE to an empty string that
+; is the same length as the individual strings - Version 4.0b25
+; 20150217: Change rule from 4.0b25 so the VAR_FILL_VALUE is an empty string
+; - Version 4.0b26
+;
+; Inputs: hdffilename - a string holding the name of the HDF file being created
+; and written to
+; ds - structure containing the data
+; meta_arr - a string array containing the Global and Variable Attributes
+; vn - a string array containing the VAR_NAME values
+; ndm - a long array describing the dimensions of each dataset and the
+; VAR_DATA_TYPE
+;
+; Outputs: An HDF4 file with the name hdffilename
+;
+; Called by: IDLCR8HDF
+;
+; Subroutines Called: MAKEAN
+; AVDC_HDF5_WRITE (if HDF5 option is chosen)
+; INFOTXT_OUTPUT
+; Information Conditions (when the program is able to make changes):
+; 1. HDF4.2R1 or earlier library and SDS name has greater than 63-characters
+; - in which case the SDS name will be truncated [3573]
+
+COMMON METADATA
+COMMON DATA
+
+nodimset=1 ;added from version 3.06 to stop dimension information being written to the HDF4 file
+ ;while still retaining original code
+
+;Determine HDF format (ftype is either 'hdf', 'h5' or 'nc')
+ftype=STRMID(hdffilename,STRPOS(hdffilename,'.',/Reverse_Search)+1)
+
+;List of dim. attributes to be assigned to the SDS dimensions.
+da_list=['VAR_NAME','VAR_UNITS']
+da_att=INDGEN(N_ELEMENTS(da_list),/String) & natts=N_ELEMENTS(attr_arr_data)
+av=STRARR(nvn,natts) ;array containing attribute and NCSA values for each dataset
+var_depend=STRARR(nvn) ;holding array for the VAR_DEPEND attributes, used to assign DIM information
+var_size=STRARR(nvn) ;holding array for the VAR_SIZE attributes, used to assign netCDF DIM information
+
+;Write all Dataset Attribute values to an array
+FOR i=0,nvn-1 DO BEGIN ;nvn EQ number of variable names
+ mi=WHERE(meta_arr EQ attr_arr_data[0]+'='+vn[i])
+ ninc=0 & j=0
+ WHILE j LE natts-1 DO BEGIN
+ ma=mi[0]+ninc
+ ;separate out the Meta entry into two components
+ eqpos=STRPOS(meta_arr[ma],'=')
+ res=STRMID(meta_arr[ma],0,eqpos)
+ WHILE res NE attr_arr_data[j] DO j=j+1 ;in case optional variable attributes are not included
+ IF eqpos EQ STRLEN(meta_arr[ma])-1 THEN av[i,j]=' ' $
+ ELSE av[i,j]=STRMID(meta_arr[ma],eqpos+1)
+ ;Reverse Values for VAR_DEPEND and VAR_SIZE to match order values will be saved in HDF
+ IF (res EQ 'VAR_DEPEND') OR (res EQ 'VAR_SIZE') THEN BEGIN
+ atval=STRSPLIT(av[i,j],' ;',/EXTRACT,COUNT=avcnt)
+ IF avcnt GT 1 THEN BEGIN
+ atval=REVERSE(atval)
+ FOR k=0,avcnt-1 DO IF k EQ 0 THEN ahold=atval[k] ELSE ahold=ahold+';'+atval[k]
+ av[i,j]=ahold
+ ENDIF
+ IF res EQ 'VAR_DEPEND' THEN var_depend[i]=av[i,j] ELSE var_size[i]=av[i,j]
+ ENDIF
+ ninc=ninc+1 & j=j+1
+ ENDWHILE
+ENDFOR
+
+IF ftype EQ 'hdf' THEN BEGIN
+
+ ;Determine HDF library used by this program
+ HDF_LIB_INFO,MAJOR=mj,MINOR=mn,RELEASE=rl
+ hdf4lib=(mj*100)+(mn*10)+rl
+ hdf4txt='HDF'+STRTRIM(mj,2)+'.'+STRTRIM(mn,2)+'R'+STRTRIM(rl,2)
+
+ ;Create and open the HDF4 file for writing
+ hdf_file_id=HDF_OPEN(hdffilename,/ALL)
+ ;The HDF_SD_START function opens or creates an HDF4 file and initializes the SD interface.
+ sd_id=HDF_SD_START(hdffilename,/RDWR)
+
+ ;Write the Global Attributes to file
+ ;Section 4.1 (Bojkov et al., 2002): Originator attributes
+ ;Section 4.2 (Bojkov et al., 2002): Dataset attributes
+ ;Section 4.3 (Bojkov et al., 2002): File attributes
+ FOR i=0,N_ELEMENTS(attr_arr_glob)-1 DO BEGIN
+ ;separate out the Meta Attribute entry
+ eqpos=STRPOS(meta_arr[i],'=')
+ IF eqpos EQ STRLEN(meta_arr[i])-1 THEN attribute_value=' ' $
+ ELSE attribute_value=STRMID(meta_arr[i],eqpos+1)
+ ;check for Free text attribute, and filename entry
+ res=STRMID(meta_arr[i],0,eqpos)
+ fai=WHERE(res EQ attr_free,facnt)
+ IF (facnt NE 0) AND (STRMID(STRUPCASE(attribute_value),0,6) EQ 'FILE("') THEN $
+ MAKEAN,hdf_file_id,sd_id,'FILE',res,attribute_value $
+ ELSE HDF_SD_ATTRSET,sd_id,attr_arr_glob[i],attribute_value
+ ENDFOR
+
+ ;Write the Datasets (SDS) to file
+ ;Section 5.1 (Bojkov et al, 2002): Variable description attributes.
+ FOR i=0,nvn-1 DO BEGIN ;nvn EQ number of variable names
+
+ ;If hdf4 library version is less than 4.2R2 then check length of VAR_NAME
+ IF hdf4lib LT 422 THEN BEGIN
+ IF STRLEN(vn[i]) GT 63 THEN BEGIN
+ infotxt=STRARR(4)
+ infotxt[0]='2 '+hdf4txt+' truncates the dataset name if its'
+ infotxt[0]=infotxt[0]+' length is greater than 63-characters.'
+ infotxt[1]=' If possible update the HDF4 library to HDF4.2R2 or newer| '
+ infotxt[1]=infotxt[1]+'(refer to program documentation for procedure to do this).
+ infotxt[2]=' The Archive Data Center may also be set up to update the file'
+ infotxt[2]=infotxt[2]+' on submission.'
+ infotxt[3]=' VAR_NAME='+vn[i]
+ INFOTXT_OUTPUT, infotxt
+ ENDIF
+ ENDIF
+
+ ;create data array from structure and create and define dataset for an HDF4 file
+ data=*ds[i].data
+ ;may have lost array VAR_SIZE information if the last value is '1' e.g. 41,1
+ gi=WHERE(ndm[i,0:7] NE 0L) & vs=TRANSPOSE(ndm[i,gi])
+ type=HDF_IDL2HDFTYPE(SIZE(data,/Type))
+ IF ndm[i,8] EQ 6 THEN BEGIN
+ mxstrlen=MAX(STRLEN(data)) ;determine maximum stength length
+ IF mxstrlen EQ 0 THEN mxstrlen=1
+ sds_id_1=HDF_SD_CREATE(sd_id,vn[i],[mxstrlen,vs],HDF_TYPE=type)
+ ENDIF ELSE sds_id_1=HDF_SD_CREATE(sd_id,vn[i],vs,HDF_TYPE=type)
+
+ mi=WHERE(meta_arr EQ attr_arr_data[0]+'='+vn[i])
+ CASE ndm[i,8] OF ;identifies the VAR_DATA_TYPE
+ 0:BEGIN
+ valid_range=[BYTE(mv_lng[0,i]),BYTE(mv_lng[1,i])]
+ fill_value=BYTE(mv_lng[2,i])
+ END
+ 1:BEGIN
+ valid_range=[FIX(mv_lng[0,i]),FIX(mv_lng[1,i])]
+ fill_value=FIX(mv_lng[2,i])
+ END
+ 2:BEGIN
+ valid_range=[LONG(mv_lng[0,i]),LONG(mv_lng[1,i])]
+ fill_value=LONG(mv_lng[2,i])
+ END
+ 3:BEGIN
+ valid_range=[mv_lng[0,i],mv_lng[1,i]]
+ fill_value=mv_lng[2,i]
+ END
+ 4:BEGIN
+ valid_range=[FLOAT(mv_dbl[0,i]),FLOAT(mv_dbl[1,i])]
+ fill_value=FLOAT(mv_dbl[2,i])
+ END
+ 5:BEGIN
+ valid_range=[mv_dbl[0,i],mv_dbl[1,i]]
+ fill_value=mv_dbl[2,i]
+ END
+ 6:BEGIN
+ valid_range=[' ',' ']
+ fill_value=' ' ;bytarr(mv_str[i]) ;STRING(format='(A-'+STRTRIM(mv_str[i],2)+')','')
+ ;Note: this seems to give the same result as using 0B and saving it as type string in the HDF_SD_ATTRSET call
+ END
+ ENDCASE
+ ninc=0 & j=0
+ WHILE j LE natts-1 DO BEGIN
+ ma=mi[0]+ninc
+ ;Extract Attribute Label
+ eqpos=STRPOS(meta_arr[ma],'=')
+ res=STRMID(meta_arr[ma],0,eqpos)
+ WHILE res[0] NE attr_arr_data[j] DO j=j+1
+ CASE 1 OF
+ ;attr_arr_data[j] EQ 'VAR_SIZE': HDF_SD_ATTRSET,sds_id_1,attr_arr_data[j],vs
+ attr_arr_data[j] EQ 'VAR_VALID_MIN': HDF_SD_ATTRSET,sds_id_1,attr_arr_data[j],valid_range[0]
+ attr_arr_data[j] EQ 'VAR_VALID_MAX': HDF_SD_ATTRSET,sds_id_1,attr_arr_data[j],valid_range[1]
+ attr_arr_data[j] EQ 'VAR_FILL_VALUE': HDF_SD_ATTRSET,sds_id_1,attr_arr_data[j],fill_value
+ ELSE: HDF_SD_ATTRSET,sds_id_1,attr_arr_data[j],av[i,j]
+ ENDCASE
+ ninc=ninc+1 & j=j+1
+ ENDWHILE
+ ;Set information about the dataset (note no pre-defined attributes for String Datatype)
+ IF ndm[i,8] NE 6 THEN $
+ HDF_SD_SETINFO,sds_id_1,UNIT=vu[i],RANGE=valid_range,FILL=fill_value
+
+ IF nodimset EQ 0 THEN BEGIN
+ ;Assign the VAR_DEPEND dimension(s) attribute information
+ vd_out=STRSPLIT(var_depend[i],' ;',/Extract) ;Read the variable dependencies
+ vdim=N_ELEMENTS(vd_out)
+ vd_out=STRTRIM(vd_out,2)
+ ;Start search of information
+ FOR j=0,vdim-1 DO BEGIN
+ IF (STRUPCASE(vd_out[j]) NE 'CONSTANT') AND (STRUPCASE(vd_out[j]) NE 'INDEPENDENT') THEN BEGIN
+ ;Return the index of the var_depend array SDS dataset
+ index=HDF_SD_NAMETOINDEX(sd_id,vd_out[j])
+ ;Access the dataset
+ sds_id_tmp=HDF_SD_SELECT(sd_id,index)
+ FOR k=0,N_ELEMENTS(da_list)-1 DO BEGIN
+ ;Find the dataset attribute in da_list
+ dindex=HDF_SD_ATTRFIND(sds_id_tmp,da_list[k])
+ ;Read attribute info and assign to da_att
+ HDF_SD_ATTRINFO,sds_id_tmp,dindex,DATA=att_data
+ da_att[k]=att_data
+ ENDFOR
+ ;End access of SDS search
+ HDF_SD_ENDACCESS,sds_id_tmp
+
+ ;Assign dimension to the current SDS
+ dim_id=HDF_SD_DIMGETID(sds_id_1,j)
+ HDF_SD_DIMSET,dim_id,/BW_INCOMP,NAME=da_att[0],UNIT=da_att[1]
+ ENDIF
+ ENDFOR
+ ENDIF
+
+ ;Write data to HDF4 file
+ HDF_SD_ADDDATA,sds_id_1,data
+ HDF_SD_ENDACCESS,sds_id_1
+ ENDFOR
+
+ ;The HDF_SD_END function closes the SD interface to an HDF4 file.
+ HDF_SD_END,sd_id
+ ;The HDF_CLOSE procedure closes the HDF4 file associated with the given file handle.
+ HDF_CLOSE,hdf_file_id
+ENDIF ELSE IF ftype EQ 'h5' THEN AVDC_HDF5_WRITE,hdffilename,natts,av,geoms_output $
+ELSE AVDC_NC_WRITE,hdffilename,natts,av,var_depend,var_size
+
+END ;Procedure AVDC_HDF_Write
+
+
+
+PRO idlcr8hdf, ga, sds, tav, odir, reterr, H5=o1, AVK=o2, LOG=o4, POPUP=o5, QA=o6, NOHDF=o7, NC=o8, DATETIME=o9, $
+ C1=o11, C2=o12, C3=o13, C4=o14, C5=o15, C6=o16, C7=o17, C8=o18, C9=o19
+;Main IDL program to create HDF4 or HDF5 format files for submission to the AVDC, NDACC, or
+;NILU (ESA Envisat) databases.
+;
+;Program documentation, idlcr8hdf-v4.0_Readme.pdf, available from http://avdc.gsfc.nasa.gov.
+;
+;Program sub-version 4.0b65 (20241029)
+; ----------
+;Written by Ian Boyd for the EVDC/AVDC - iboyd@bryanscientific.org
+;
+; History:
+; 20050802: Original Release - Version 1.0
+; 20050909: No change to routine - Version 1.1
+; 20051107: No change to routine - Version 1.11
+; 20061012: Make the code suitable for running on a licensed version of IDL (using the .pro
+; and .sav versions of the code) and on IDL Virtual Machine (using the .sav version
+; of the code); Change command line keyword options, remove the /Help (window now
+; opened if there are no command line parameters) and /File options (now automatically
+; identifies whether input is from session memory or files), and add the /H5 (write
+; HDF5), /AVK (for Averaging Kernel Datasets add sentence to VAR_NOTES indicating
+; averaging kernel order), /Log (append input/output information to a log file), and
+; /Popup options (append input/output, error and warning information to a pop-up
+; display window); If input is from files, the program can now handle multiple data
+; files (either as a string array or as a file spec), together with a Metadata template
+; file and the TAV file; Input/output information as well as errors and warnings
+; included in the IDL DE output log window and/or to a log file; The user has the
+; option of continuing to run the program with file inputs from the 'Introduction'
+; window; Common variable definition WIDGET_WIN added - Version 2.0
+; 20080302: Remove HELP,/TRACEBACK call, which identified how the program was called. This was used
+; to stop output to the IDLDE output window in the event that IDL VM was used, but it is
+; not required as IDL VM ignores these print calls; Add DIALOG_BOX to show completion
+; of the program if program inputs are via the INTRO box, and logging window option
+; isn't requested - Version 3.0
+; 20100205: Add optional reterr parameter which allows program to return to a calling program with
+; an error message, rather than stop - Version 3.09
+; 20101120: Adopt GEOMS metadata standard; New structure format required when input is by session
+; memory; New arrays created to hold numeric metadata values (mv_lng and mv_dbl) in
+; either LONG64 or DOUBLE format (changed to their actual format before writing to file);
+; Add QA keyword, to perform QA function on HDF file read into session memory (does not
+; create an HDF file) - Version 4.0
+; 20130114: Add check for IDL being run in DEMO mode by using LMGR command. Replace PRINTF,-1
+; statements (i.e. when dux[0] eq -1) with PRINT (o/w causes DEMO mode to stop. Also disable
+; /LOG and /NC keywords in DEMO mode - Version 4.0b14
+; 20140327: Add check and log message for VAR_SIZE values not being of type string - Version 4.0b20
+; 20150127: When input is by structure, stop removing leading and trailing spaces from variable
+; attribute values when creating metafile array - Version 4.0b25
+; 20150217: Initialize mv_str if input is via files - Version 4.0b26
+; 20150302: Fix Bug which incorrectly set mv_str value if input is via structure - Version 4.0b27
+; 20151012: Create qa_yes common variable to streamline checks on whether program called in QA mode
+; or not - Version 4.0b31
+; 20160725: Fix bug so that numeric metadata sub-values read in from a structure will be written to
+; the metafile array OK - Version 4.0b38
+; 20161130: Fix issue when a metadata value is not numeric or a string e.g. a structure when the
+; inputs are in the form of a structure - Version 4.0b40
+; 20180218: Ensure variable attribute value (vav) is only a single variable when calling
+; PRE_DEFINED_ATT_CHECKS - Version 4.0b45
+; 20181221: Allow compression and shuffle options when creating HDF5 files - Version 4.0b49
+;
+; Inputs: DIALOG PROMPTS WITH IDL VIRTUAL MACHINE (File input only)
+; Metadata template file - The program will fill in missing spaces based on data file and
+; TAV inputs
+; Data file(s) - the user has the option of picking multiple data files
+; tav - the current Table Attribute Values (TAV) file
+; outdir - the directory which will hold the HDF file
+;
+; COMMAND LINE PARAMETER OPTION AVAILABLE WITH FULL IDL VERSION
+; ga - either a string array containing the Global Attributes, or a Metadata template file
+; (as above)
+; sds - either a heap structure using pointers containing the Data (DS.Data) and the Variable
+; Attributes (DS.VA_L and DS.VA_V) for a single file, or a string array or file spec of
+; file(s) containing the data (as above)
+; tav - the current Table Attribute Values (TAV) file
+; outdir - the directory to which any HDF and log files will be written
+; reterr - optional string input that, if included, returns an error message to the calling
+; program rather than stop the program. Note that the Pop-up option will be
+; deselected if present together with this argument.
+;
+; OPTIONS
+; H5 - output will be as an HDF5 file instead of the standard HDF4
+; NC - output will be as a netCDF3 file instead of the standard HDF4
+; AVK - to add a sentence to VAR_NOTES indicating array order for Averaging Kernel
+; datasets
+; Log - to append input/output information as well as warnings and errors to a log file
+; named idlcr8hdf.log, or idlcr8qa.log if QA option chosen
+; Popup - to append input/output information as well as warnings and errors to a pop-up
+; display window
+; QA - to perform QA function on an HDF file passed to the program using the session
+; memory option. The logfile is automatically generated and will be called
+; idlcr8qa.log instead of idlcr8hdf.log
+; NoHDF - an HDF file will not be created, but the logfile idlcr8hdf.log will automatically
+; be generated
+; Cn - to enable compression and shuffling of HDF5 files only (ignored for netCDF3 and HDF4)
+;
+; Output: An HDF file formatted to GEOMS standards. Keyword options can also result
+; in an output log file (idlcr8hdf.log), or a pop-up box showing log output. If the
+; reterr string input parameter is included then reterr will return an error message to the
+; calling program, if encountered.
+;
+; Called by: Main Routine
+;
+; Subroutines Called: INTRO (if program called without command line parameters)
+; READ_TABLEFILE
+; READ_METADATA
+; CHECK_METADATA
+; SET_UP_STRUCTURE
+; READ_DATA
+; FIND_HDF_FILENAME
+; AVDC_HDF_WRITE
+; STOP_WITH_ERROR (if error state detected)
+; INFOTXT_OUTPUT (if information conditions detected)
+; Possible Conditions for STOP_WITH_ERROR call:
+; 1. If Metadata, Data, or Table Attribute Values file(s), or Output Directory
+; selection is not valid
+; 2. If input is via session memory and the array holding the global attributes
+; is not of type string, or is not one dimensional
+; 3. If input is via session memory and the input heap structure is not valid
+; 4. If input is via session memory and the array sizes of the heap structures
+; do not match
+; 5. If the H5 option is chosen and the IDL version does not support HDF5 write
+; routines (less than v6.2)
+; 6. If the NC option is chosen and IDL was called in IDL DEMO mode
+;
+; Information Conditions (when the program is able to make changes):
+; 1. /POPUP keyword cannot be used together with the 'reterr' argument
+; 2. Argument 'reterr' must be a scalar variable of type string
+; 3. /LOG keyword cannot be used in IDL DEMO mode
+
+COMMON TABLEDATA
+COMMON METADATA
+COMMON DATA
+COMMON WIDGET_WIN
+
+;Possible error message for this procedure
+proname='IDLcr8HDF procedure: ' & lu=-1L
+errtxt=STRARR(6)
+errtxt[0]=' selection not valid'
+errtxt[1]='Global Attributes Array is not of type string, or is not one dimensional'
+errtxt[2]='structure is not valid ('
+errtxt[3]='Array sizes of the heap structure do not match: '
+errtxt[4]=' does not support HDF5 Write Routines'
+errtxt[5]='IDLcr8HDF requires session memory input to perform the QA function'
+
+demomode=LMGR(/DEMO) ;Boolean to check for IDL being run in demo mode (disable /LOG option)
+IF N_PARAMS() GE 2 THEN intype=SIZE(sds,/TYPE) $ ;Is SDS a structure (8) or a string (7)?
+ELSE IF demomode THEN intype=-3 $
+ELSE intype=-1
+IF (intype NE 7) AND (intype NE 8) AND (intype NE -1) AND (intype NE -3) THEN intype=-2
+;Command line parameter neither string nor structure
+dux=[-1,-1,1] ;initialize flags for output log to IDLDE Output Window and/or to file
+;dux[0] default allows output to the IDLDE output window (ignored by IDL VM)
+;dux[1] is the program assigned file unit for sending log output to file
+;dux[2] is the step between dux[0] and dux[1]
+lineno=0L ;count variable to put curser at end of text pop-up window
+rerr=['NA','NA'] ;initialize return error string
+
+IF intype LT 0 THEN BEGIN ;either no input parameters or invalid second parameter
+ o3=['0','0','0','0','0','0']
+ lvals=['None','1','2','3','4','5','6','7','8','9'] ;compression level for HDF5 option
+ INTRO,intype ;Open Intro Box and determine HDF output format (HDF4 or HDF5)
+ IF o3[0] EQ '0' THEN BEGIN
+ STOP_WITH_ERROR,'','',lu & RETURN
+ ENDIF
+ IF o3[3] EQ 'Pop' THEN o3[3]='' ELSE o3[3]='D_'
+ENDIF ELSE BEGIN ;Set options (HDF4, HDF5, netCDF, AVK, LOGFILE, POP-UP, QA, NOHDF)
+ IF KEYWORD_SET(o1) THEN BEGIN
+ o3=['H5','0','0','D_','0','0']
+ CASE 1 OF
+ KEYWORD_SET(o11): o3[0]='H5_1'
+ KEYWORD_SET(o12): o3[0]='H5_2'
+ KEYWORD_SET(o13): o3[0]='H5_3'
+ KEYWORD_SET(o14): o3[0]='H5_4'
+ KEYWORD_SET(o15): o3[0]='H5_5'
+ KEYWORD_SET(o16): o3[0]='H5_6'
+ KEYWORD_SET(o17): o3[0]='H5_7'
+ KEYWORD_SET(o18): o3[0]='H5_8'
+ KEYWORD_SET(o19): o3[0]='H5_9'
+ ELSE:
+ ENDCASE
+ ;PRINT,o3
+ ENDIF ELSE IF KEYWORD_SET(o8) THEN o3=['NC','0','0','D_','0','0'] $
+ ELSE o3=['H4','0','0','D_','0','0']
+ IF KEYWORD_SET(o2) THEN o3[1]='AVK' ;AVK option
+ IF KEYWORD_SET(o4) THEN o3[2]='idlcr8hdf.log' ;Log option
+ IF KEYWORD_SET(o5) THEN o3[3]='' ;POP-UP option
+ IF KEYWORD_SET(o6) THEN o3[2]='idlcr8qa.log' ;QA option
+ IF KEYWORD_SET(o7) THEN o3[4]='NOHDF' ;NoHDF option
+ IF KEYWORD_SET(o9) THEN o3[5]='DTFVOK' ;DATETIME Fill Value OK
+ENDELSE
+IF o3[0] EQ 'NC' THEN dftxt='netCDF' ELSE dftxt='HDF'
+
+;If reterr included allows error output to return to a calling program
+n_it=5
+infotxt=STRARR(2,n_it)
+IF N_PARAMS() GE 5 THEN BEGIN
+ ;IF (ARG_PRESENT(reterr)) AND (SIZE(reterr,/Type) EQ 7) THEN BEGIN
+ IF SIZE(reterr,/Type) EQ 7 THEN BEGIN
+ rerr=['','']
+ IF o3[3] EQ '' THEN BEGIN
+ ;Deselect POPUP option and write INFORMATION message
+ o3[3]='D_'
+ infotxt[0,0]='0 /POPUP keyword cannot be used together with the ''reterr'' argument.'
+ infotxt[1,0]=' The request for a POPUP window has been ignored.'
+ ENDIF
+ ENDIF ELSE BEGIN
+ ;reterr input included but is of the incorrect type, or parameter cannot be returned
+ ;to the calling program
+ infotxt[0,1]='0 Argument ''reterr'' must be of type string.' ;a returnable variable of type string.'
+ infotxt[1,1]=' idlcr8hdf will stop normally if an error is encountered.'
+ ENDELSE
+ENDIF
+
+;Do check for /LOG and /NC keywords in IDL DEMO Mode and if necessary disable or stop the program
+IF demomode THEN BEGIN
+ IF o3[2] EQ 'idlcr8hdf.log' THEN BEGIN
+ o3[2]='0'
+ infotxt[0,2]='0 /LOG keyword cannot be used in IDL DEMO mode and has been ignored.'
+ ENDIF
+ IF o3[0] EQ 'NC' THEN BEGIN
+ o3[4]='NOHDF'
+ infotxt[0,3]='3 NetCDF file create feature is disabled in IDL DEMO mode. '
+ infotxt[1,3]=' To generate files please use the free IDL Virtual Machine or a licenced version of IDL.'
+ ENDIF
+ENDIF
+
+qa_yes=STRPOS(o3[2],'idlcr8qa.log') NE -1 ;Set Boolean for doing QA checks
+
+IF (KEYWORD_SET(o1)) AND (FLOAT(!Version.Release) LT 6.2) THEN BEGIN ;No HDF5 library for IDL6.1 or less
+ STOP_WITH_ERROR,'D_'+proname,'IDL Version '+!Version.Release+errtxt[4],lu
+ IF STRLEN(rerr[0]) GT 2 THEN reterr=rerr[0] & RETURN
+ENDIF
+
+;Check TAV file and Output Directory inputs
+tablefile='' & outdir=''
+CD,CURRENT=path ;identify current working directory
+IF N_PARAMS() GE 3 THEN BEGIN ;check TAV file
+ IF (SIZE(tav,/N_ELEMENTS) EQ 1) AND (SIZE(tav,/TYPE) EQ 7) THEN BEGIN
+ file_exist=FILE_TEST(tav,/READ)
+ IF (file_exist EQ 1) AND (tav NE '') THEN tablefile=tav
+ ENDIF
+ENDIF
+IF N_PARAMS() GE 4 THEN BEGIN ;check output directory
+ IF (SIZE(odir,/N_ELEMENTS) EQ 1) AND (SIZE(odir,/TYPE) EQ 7) THEN BEGIN
+ file_exist=FILE_TEST(odir,/DIRECTORY)
+ IF (file_exist EQ 1) AND (odir NE '') THEN BEGIN
+ outdir=FILE_SEARCH(odir,/FULLY_QUALIFY_PATH,/MARK_DIRECTORY)
+ path=outdir
+ ENDIF ELSE IF intype NE 8 THEN outdir=STRUPCASE(STRTRIM(odir,2))
+ ENDIF
+ENDIF
+
+IF (intype LT 0) OR (intype EQ 7) THEN BEGIN ;Input via files
+ IF KEYWORD_SET(o6) THEN BEGIN
+ ;session memory inputs required for QA option
+ STOP_WITH_ERROR,'D_'+proname,errtxt[5],lu
+ IF STRLEN(rerr[0]) GT 2 THEN reterr=rerr[0] & RETURN
+ ENDIF
+ metafile='' & inf=1 & mv_str=[-1L]
+ IF N_PARAMS() GE 2 THEN BEGIN
+ ;Check that input files exist. If not then user will be prompted for new inputs
+ IF SIZE(ga,/N_ELEMENTS) EQ 1 THEN BEGIN
+ file_exist=FILE_TEST(ga,/READ)
+ IF (file_exist EQ 1) AND (ga NE '') THEN $
+ metafile=FILE_DIRNAME(ga,/MARK_DIRECTORY)+FILE_BASENAME(ga)
+ ENDIF
+ arorfs=SIZE(sds,/DIMENSIONS) ;Is datafile input Filespec or String Array?
+ IF arorfs[0] EQ 0 THEN BEGIN ;input is filespec (scalar)
+ IF sds NE '' THEN sds=FINDFILE(sds)
+ isize=SIZE(sds,/DIMENSIONS)
+ IF isize[0] EQ 0 THEN sds=[''] ;no files found matching filespec
+ ENDIF
+ IF N_ELEMENTS(arorfs) GT 1 THEN sds=REFORM(sds,N_ELEMENTS(sds),/OVERWRITE) ;Convert to 1-D array
+ file_exist=FILE_TEST(sds,/READ)
+ gi=WHERE(file_exist EQ 1,nfile)
+ IF nfile NE 0 THEN BEGIN
+ sds=sds[gi] & sds=FILE_DIRNAME(sds,/MARK_DIRECTORY)+FILE_BASENAME(sds)
+ ENDIF ELSE sds=[''] ;contains all valid datafiles
+ ENDIF ELSE sds=['']
+ IF metafile EQ '' THEN BEGIN
+ metafile=DIALOG_PICKFILE(Filter=['*.meta','meta*.txt','*.txt'], PATH=path, $
+ /MUST_EXIST,Title='Select Metadata Template File')
+ IF metafile EQ '' THEN BEGIN
+ STOP_WITH_ERROR,'D_'+proname,'Metadata file'+errtxt[0],lu
+ IF STRLEN(rerr[0]) GT 2 THEN reterr=rerr[0] & RETURN
+ ENDIF
+ path=FILE_DIRNAME(metafile,/MARK_DIRECTORY)
+ END
+ IF sds[0] EQ '' THEN BEGIN
+ datafile=DIALOG_PICKFILE(Filter=['*.data','*.dat','*.txt'], PATH=path, $
+ /MUST_EXIST,/Multiple_Files,Title='Select Data File(s)')
+ gi=WHERE(datafile NE '',nfile)
+ IF nfile EQ 0 THEN BEGIN
+ STOP_WITH_ERROR,'D_'+proname,'Data file(s)'+errtxt[0],lu
+ IF STRLEN(rerr[0]) GT 2 THEN reterr=rerr[0] & RETURN
+ ENDIF ELSE BEGIN
+ datafile=datafile[gi] & dsort=SORT(datafile) & sds=datafile[dsort]
+ path=FILE_DIRNAME(sds[0],/MARK_DIRECTORY)
+ ENDELSE
+ ENDIF
+ IF outdir EQ 'M' THEN outdir=FILE_DIRNAME(metafile,/MARK_DIRECTORY) $
+ ELSE IF outdir EQ 'D' THEN outdir=FILE_DIRNAME(sds[0],/MARK_DIRECTORY)
+ENDIF ELSE BEGIN ;SDS is a data structure, so do initial checks on parameters
+ ;check GA array
+ inf=0 & nfile=1
+ as=SIZE(ga) & n_ga=N_ELEMENTS(ga)
+ IF (as[0] NE 1) OR (as[2] NE 7) THEN BEGIN
+ STOP_WITH_ERROR,'D_'+proname,errtxt[1],lu
+ IF STRLEN(rerr[0]) GT 2 THEN reterr=rerr[0] & RETURN
+ ENDIF
+
+ vdtype=[1,2,3,4,5,7,12,13,14,15] ;valid IDL data types
+ badlabel=[''] ;list of attribute labels where the values are not valid (to avoid repeated log messages)
+ it5=[''] ;array holding infotxt[*,5] error messages
+
+ ;check pointer array (heap structure)
+ IF N_TAGS(sds) EQ 0 THEN BEGIN
+ STOP_WITH_ERROR,'D_'+proname,'Input heap '+errtxt[2]+'expecting VA_L, VA_V, and DATA tags).',lu
+ IF STRLEN(rerr[0]) GT 2 THEN reterr=rerr[0] & RETURN
+ ENDIF
+ tagnames=STRUPCASE(TAG_NAMES(sds))
+ si=WHERE((tagnames EQ 'VA_L') OR (tagnames EQ 'VA_V') OR (tagnames EQ 'DATA'),scnt)
+ IF scnt NE 3 THEN BEGIN
+ STOP_WITH_ERROR,'D_'+proname,'Input heap '+errtxt[2]+'missing VA_L, VA_V, and/or DATA tags).',lu
+ IF STRLEN(rerr[0]) GT 2 THEN reterr=rerr[0] & RETURN
+ ENDIF
+ ;Check for number and size of dimensions, and invalid pointer arguments for the data
+ FOR i=0,2 DO BEGIN
+ CASE 1 OF
+ i EQ 0: BEGIN
+ pchk=PTR_VALID(sds.data) & sdstxt='SDS.DATA '
+ END
+ i EQ 1: BEGIN
+ pchk=PTR_VALID(sds.va_v) & sdstxt='SDS.VA_V '
+ pchkv=pchk
+ END
+ ELSE: BEGIN
+ pchk=PTR_VALID(sds.va_l) & sdstxt='SDS.VA_L '
+ END
+ ENDCASE
+ ;Check that the SDS structure has two dimensions
+ s_ndim=SIZE(pchk,/N_DIMENSIONS) & s_dim=SIZE(pchk,/DIMENSIONS)
+ IF s_ndim NE 2 THEN BEGIN
+ STOP_WITH_ERROR,'D_'+proname,sdstxt+errtxt[2]+'two dimensions expected).',lu
+ IF STRLEN(rerr[0]) GT 2 THEN reterr=rerr[0] & RETURN
+ ENDIF ELSE IF i EQ 0 THEN BEGIN
+ pi=WHERE(pchk[*,0] NE 1,pcnt)
+ IF pcnt NE 0 THEN BEGIN
+ STOP_WITH_ERROR,'D_'+proname,sdstxt+errtxt[2]+'null pointer arguments).',lu
+ IF STRLEN(rerr[0]) GT 2 THEN reterr=rerr[0] & RETURN
+ ENDIF
+ ENDIF ELSE BEGIN
+ IF i EQ 2 THEN test=s_dim[1] NE s_dim0[1] ELSE test=0
+ IF (s_dim[0] NE s_dim0[0]) OR (test) THEN BEGIN
+ dimv='['+STRTRIM(s_dim0[0],2)+','+STRTRIM(sdim0[1],2)+']/['+ $
+ STRTRIM(s_dim[0],2)+','+STRTRIM(sdim[1],2)+'].'
+ STOP_WITH_ERROR,'D_'+proname,errtxt[3]+dimv,lu
+ IF STRLEN(rerr[0]) GT 2 THEN reterr=rerr[0] & RETURN
+ ENDIF
+ ENDELSE
+ s_dim0=s_dim
+ ENDFOR
+
+ ;create Metafile by combining global and variable attributes, also make
+ ;accompanying arrays to hold any numeric metadata values
+ vi=WHERE(pchk EQ 1,vatot) ;total number of Variable Attribute labels
+ metafile=STRARR(n_ga+vatot) & mv_lng=LON64ARR(n_ga+vatot) & mv_dbl=DBLARR(n_ga+vatot)
+ pcnt=LONG(n_ga) & mv_str=LONARR(vatot)
+ metafile[0:pcnt-1L]=ga
+ FOR i=0,s_dim[0]-1 DO BEGIN ;s_dim[0] is the number of datasets
+ dset=*sds[i,0].data & dtype=SIZE(dset,/TYPE)
+ FOR j=0,s_dim[1]-1 DO BEGIN ;s_dim[1] is the number of attributes
+ IF pchk[i,j] EQ 1 THEN BEGIN
+ IF pchkv[i,j] EQ 0 THEN metafile[pcnt]=*sds[i,j].va_l+'=' $
+ ELSE BEGIN
+ vavt=SIZE(*sds[i,j].va_v,/TYPE)
+ gti=WHERE(vavt EQ vdtype,g_vavt) ;test to see if the data type is valid
+ IF g_vavt EQ 1 THEN BEGIN ;it is OK to transfer the attribute value to a variable
+ vav=*sds[i,j].va_v
+ n_vav=N_ELEMENTS(vav)
+ FOR k=0,n_vav-1 DO BEGIN
+ IF k EQ 0 THEN BEGIN
+ IF vavt EQ 1 THEN metafile[pcnt]=*sds[i,j].va_l+'='+STRING(FIX(vav[k])) $
+ ELSE IF vavt EQ 7 THEN metafile[pcnt]=*sds[i,j].va_l+'='+STRING(vav[k]) $ ;save all values as a string representation
+ ELSE metafile[pcnt]=*sds[i,j].va_l+'='+STRTRIM(vav[k],2)
+ ENDIF ELSE BEGIN
+ IF vavt EQ 1 THEN metafile[pcnt]=metafile[pcnt]+';'+STRING(FIX(vav[k])) $
+ ELSE IF vavt EQ 7 THEN metafile[pcnt]=metafile[pcnt]+';'+STRING(vav[k]) $
+ ELSE metafile[pcnt]=metafile[pcnt]+';'+STRTRIM(vav[k],2)
+ ENDELSE
+ ENDFOR
+ IF vavt NE 7 THEN BEGIN ;i.e. if the metadata value is not of STRING datatype
+
+ ;Do check for VAR_SIZE value written as a numeric type instead of string
+ vs_chk=*sds[i,j].va_l
+ IF (STRUPCASE(vs_chk) EQ 'VAR_SIZE') AND (infotxt[0,4] EQ '') THEN BEGIN
+ ;need to generate infotxt message
+ IF qa_yes THEN itxt=' must be of ' ELSE itxt=' changed to '
+ infotxt[0,4]='2 VAR_SIZE attribute value(s)'+itxt+'type STRING'
+ ENDIF
+
+ IF n_vav EQ 1 THEN BEGIN ;save numeric values to the appropriate numeric array (LONG64 or DOUBLE)
+ CASE 1 OF
+ ((vavt GE 1) AND (vavt LE 3)) OR ((vavt GE 12) AND (vavt LE 14)): mv_lng[pcnt]=vav
+ (vavt GE 4) AND (vavt LE 5): mv_dbl[pcnt]=vav
+ ELSE:
+ ENDCASE
+ ENDIF
+ ENDIF
+ ENDIF ELSE BEGIN ;invalid data type for the attribute label
+ IF qa_yes THEN itxt='' ELSE itxt=' and will be ignored'
+ val=*sds[i,j].va_l & val=STRTRIM(val,2)
+ bli=WHERE(val EQ badlabel,blcnt)
+ IF blcnt EQ 0 THEN BEGIN
+ badlabel=[badlabel,val]
+ IF it5[0] EQ '' THEN it5[0]='2 Value for Attribute Label '+val+' is an invalid Data Type'+itxt $
+ ELSE it5=[it5,'2 Value for Attribute Label '+val+' is an invalid Data Type'+itxt]
+ ENDIF
+ ENDELSE
+ ENDELSE
+ pcnt=pcnt+1L
+ ENDIF
+ ENDFOR
+ ENDFOR
+ENDELSE
+
+;If necessary open DIALOG_BOXES for the TAV File and Output Directory Inputs
+IF tablefile EQ '' THEN BEGIN
+ tablefile=DIALOG_PICKFILE(Filter=['table*.dat','*.dat'], PATH=path, $
+ /MUST_EXIST,Title='Select Table Attribute Values File')
+ IF tablefile EQ '' THEN BEGIN
+ STOP_WITH_ERROR,'D_'+proname,'Table Attribute Values file'+errtxt[0],lu
+ IF STRLEN(rerr[0]) GT 2 THEN reterr=rerr[0] & RETURN
+ ENDIF
+ path=FILE_DIRNAME(tablefile,/MARK_DIRECTORY)
+ENDIF
+IF outdir EQ '' THEN BEGIN
+ outdir=DIALOG_PICKFILE(/DIRECTORY,Title='Select Directory for Output '+dftxt+' file(s)', PATH=path)
+ IF outdir EQ '' THEN BEGIN
+ STOP_WITH_ERROR,'D_'+proname,dftxt+' file(s) output directory'+errtxt[0],lu
+ IF STRLEN(rerr[0]) GT 2 THEN reterr=rerr[0] & RETURN
+ ENDIF
+ENDIF
+
+;Set dux values according to output options
+IF o3[2] NE '0' THEN BEGIN ;open idlcr8hdf/idlcr8qa.log
+ o3[2]=outdir+o3[2]
+ res=FILE_TEST(o3[2],/WRITE)
+ IF res EQ 0 THEN OPENW,du,o3[2],/GET_LUN $
+ ELSE BEGIN
+ OPENW,du,o3[2],/Append,/GET_LUN & FOR i=0,1 DO PRINTF,du,''
+ ENDELSE
+ dux[1]=du & dux[2]=dux[1]-dux[0]
+ENDIF
+FOR i=dux[0],dux[1],dux[2] DO BEGIN
+ IF i EQ -1 THEN BEGIN
+ PRINT,dftxt+' File Input/Output Log - Program Started on '+SYSTIME(0) & PRINT,''
+ ENDIF ELSE IF (i EQ dux[0]) OR ((i EQ dux[1]) AND (STRPOS(o3[2],'idlcr8hdf.log') NE -1)) THEN BEGIN
+ PRINTF,i,dftxt+' File Input/Output Log - Program Started on '+SYSTIME(0)
+ PRINTF,i,''
+ ENDIF
+ENDFOR
+
+IF o3[3] EQ '' THEN BEGIN
+ ;Set-up output file display widget and log file
+ base=WIDGET_BASE(Title=dftxt+' File Input/Output Log',Units=2,yoffset=3,xoffset=3,Tlb_Frame_Attr=1,/COLUMN) ;,Tab_Mode=1)
+ wtxt=WIDGET_TEXT(base,/Scroll,xsize=100,ysize=12)
+ base2=WIDGET_BASE(base)
+ b1=WIDGET_BUTTON(base2,value='Finish',uvalue='DONE',frame=3,Sensitive=0) ;Accelerator='Return')
+ WIDGET_CONTROL,wtxt,/REALIZE
+ WIDGET_CONTROL,base,/REALIZE
+ WIDGET_CONTROL,wtxt,set_value='',/APPEND
+ WIDGET_CONTROL,wtxt,set_value='Input TAV File: '+tablefile,/APPEND
+ENDIF
+
+;Call the procedure to read the TAV file (returns tab_arr)
+FOR i=dux[0],dux[1],dux[2] DO $
+ IF i EQ -1 THEN PRINT,'Input TAV File: '+tablefile ELSE PRINTF,i,'Input TAV File: '+tablefile
+READ_TABLEFILE,tablefile
+IF STRLEN(rerr[0]) GT 2 THEN BEGIN
+ reterr=rerr[0] & RETURN
+ENDIF
+
+;Check (and write out) any earlier infotxt statements
+FOR i=0,3 DO BEGIN
+ IF infotxt[0,i] NE '' THEN BEGIN
+ IF i EQ 2 THEN INFOTXT_OUTPUT,infotxt[0,i] ELSE INFOTXT_OUTPUT,infotxt[*,i]
+ ENDIF
+ENDFOR
+
+FOR ndf=0,nfile-1 DO BEGIN
+ IF o3[3] EQ '' THEN BEGIN
+ lineno=lineno+3L
+ WIDGET_CONTROL,wtxt,set_value='',/Append
+ ENDIF
+ FOR i=dux[0],dux[1],dux[2] DO IF i EQ -1 THEN PRINT,'' ELSE PRINTF,i,''
+ IF inf EQ 1 THEN BEGIN
+ IF o3[3] EQ '' THEN BEGIN
+ lineno=lineno+2L
+ WIDGET_CONTROL,wtxt,set_value='Input Metadata File: '+metafile,/Append
+ WIDGET_CONTROL,wtxt,set_value='Input Data File: '+sds[ndf],/Append
+ ENDIF
+ FOR i=dux[0],dux[1],dux[2] DO BEGIN
+ IF i EQ -1 THEN BEGIN
+ PRINT,'Input Metadata File: '+metafile & PRINT,'Input Data File: '+sds[ndf]
+ ENDIF ELSE BEGIN
+ PRINTF,i,'Input Metadata File: '+metafile
+ PRINTF,i,'Input Data File: '+sds[ndf]
+ ENDELSE
+ ENDFOR
+ ENDIF ELSE BEGIN
+ IF o3[3] EQ '' THEN $
+ WIDGET_CONTROL,wtxt,set_value='Reading input Global Attribute array and Data Structure',/Append
+ FOR i=dux[0],dux[1],dux[2] DO BEGIN
+ IF i EQ -1 THEN PRINT,'Reading input Global Attribute array and Data Structure' $
+ ELSE IF (i EQ dux[0]) OR ((i EQ dux[1]) AND (STRPOS(o3[2],'idlcr8hdf.log') NE -1)) THEN $
+ PRINTF,i,'Reading input Global Attribute array and Data Structure'
+ ENDFOR
+ ;check for numeric VAR_SIZE value message
+ IF infotxt[0,4] NE '' THEN INFOTXT_OUTPUT,infotxt[0,4]
+
+ ;check for invalid variable attribute value data type
+ IF it5[0] NE '' THEN BEGIN
+ FOR i=0,N_ELEMENTS(it5)-1 DO INFOTXT_OUTPUT,it5[i]
+ ENDIF
+
+ ;Do stage 1 of HDF Pre-Defined attributes check (checking units,valid_range,_FillValue only)
+ ncsa=['x','x','units','_fillvalue','valid_range','valid_range']
+ var_atts=['VAR_NAME','VAR_DATA_TYPE','VAR_UNITS','VAR_FILL_VALUE','VAR_VALID_MIN','VAR_VALID_MAX']
+ FOR i=0,s_dim[0]-1 DO BEGIN ;s_dim[0] is the number of datasets
+ vdtv='' & vnx='' & verror=INTARR(2)
+ FOR k=0,N_ELEMENTS(var_atts)-1 DO BEGIN
+ vav='' & nav=''
+ FOR j=0,s_dim[1]-1 DO BEGIN ;s_dim[1] is the number of attributes
+ IF (pchk[i,j] EQ 1) AND (pchkv[i,j] EQ 1) THEN BEGIN
+ CASE 1 OF
+ STRUPCASE(STRTRIM(*sds[i,j].va_l,2)) EQ var_atts[k]: vav=*sds[i,j].va_v
+ STRLOWCASE(STRTRIM(*sds[i,j].va_l,2)) EQ ncsa[k]: nav=*sds[i,j].va_v
+ ELSE:
+ ENDCASE
+ ENDIF
+ ENDFOR
+ IF k EQ 0 THEN vnx=STRTRIM(vav[0],2) $ ;VAR_NAME
+ ELSE IF k EQ 1 THEN vdtv=STRUPCASE(STRTRIM(vav[0],2)) $ ;VAR_DATA_TYPE
+ ELSE IF k GE 4 THEN BEGIN ;separate out min and max valid_range values
+ IF ((SIZE(nav[0],/TYPE) EQ 7) AND (STRTRIM(nav[0],2) NE '')) OR (SIZE(nav[0],/TYPE) NE 7) THEN BEGIN
+ IF k EQ 4 THEN nav=nav[0] $
+ ELSE IF N_ELEMENTS(nav) GT 1 THEN nav=nav[1]
+ ENDIF
+ vav=vav[0]
+ ENDIF ELSE BEGIN
+ vav=vav[0] & nav=nav[0]
+ ENDELSE
+ IF k GE 2 THEN PRE_DEFINED_ATT_CHECKS,k-2,nav,vav,vnx,vdtv,verror
+ ENDFOR
+ ENDFOR
+ ENDELSE
+
+ ;Call the procedure to read the metadata (returns meta_arr)
+ READ_METADATA,metafile,inf
+ IF STRLEN(rerr[0]) GT 2 THEN BEGIN
+ reterr=rerr[0] & RETURN
+ ENDIF
+ ;Call Procedure to check inputs in meta_arr with those from the TAV file
+ CHECK_METADATA
+ IF STRLEN(rerr[0]) GT 2 THEN BEGIN
+ reterr=rerr[0] & RETURN
+ ENDIF
+ ;Call the procedure to set up the data structure and assign arrays for VAR_NAME and VAR_UNITS
+ IF inf EQ 0 THEN SET_UP_STRUCTURE,sds,inf ELSE SET_UP_STRUCTURE,sds[ndf],inf
+ IF STRLEN(rerr[0]) GT 2 THEN BEGIN
+ reterr=rerr[0] & RETURN
+ ENDIF ;Call the procedure to read the data
+ IF inf EQ 0 THEN READ_DATA,sds,inf ELSE READ_DATA,sds[ndf],inf
+ IF STRLEN(rerr[0]) GT 2 THEN BEGIN
+ reterr=rerr[0] & RETURN
+ ENDIF
+ ;Call the Procedure to construct the HDF output filename based on input metadata.
+ IF STRPOS(o3[0],'H5') NE -1 THEN hdffilename='.h5' $
+ ELSE IF o3[0] EQ 'NC' THEN hdffilename='.nc' $
+ ELSE hdffilename='.hdf'
+ FIND_HDF_FILENAME,hdffilename
+ IF STRLEN(rerr[0]) GT 2 THEN BEGIN
+ reterr=rerr[0] & RETURN
+ ENDIF
+
+ ;Create HDF file if the program is not performing QA and NOHDF keyword not selected
+ IF (~qa_yes) AND (o3[4] EQ '0') THEN BEGIN
+ ;Check to see if file exists - if so delete (o/w will append to existing file)
+ IF FILE_TEST(hdffilename) EQ 1 THEN FILE_DELETE,hdffilename
+
+ ;Call the procedure to write the Global Attributes, Variable Attributes and Data
+ ;to an HDF4 or HDF5 file
+ geoms_output=o3[0]
+ AVDC_HDF_WRITE,hdffilename,geoms_output
+ FILE_MOVE, hdffilename, outdir+hdffilename,/Allow_Same,/Overwrite
+ hdffilename=outdir+hdffilename
+
+ IF o3[3] EQ '' THEN $
+ WIDGET_CONTROL,wtxt,set_value=hdffilename+' successfully created!',/Append,Set_text_top_line=lineno
+ FOR i=dux[0],dux[1],dux[2] DO BEGIN
+ IF i EQ -1 THEN PRINT,hdffilename+' successfully created!' $
+ ELSE PRINTF,i,hdffilename+' successfully created!'
+ ENDFOR
+ ENDIF
+ ;Free up memory by destroying the heap variables pointed at by its pointer arguments
+ PTR_FREE,ds.data
+ENDFOR
+
+IF qa_yes THEN endtxt=dftxt+' QA function ' $
+ELSE IF o3[4] EQ '0' THEN endtxt=dftxt+' file creation ' $
+ELSE endtxt='GEOMS file testing '
+
+FOR i=dux[0],dux[1],dux[2] DO BEGIN
+ IF i EQ -1 THEN BEGIN
+ PRINT,'' & PRINT,endtxt+'completed - Program Ended on '+SYSTIME(0)
+ ENDIF ELSE IF (i EQ dux[0]) OR ((i EQ dux[1]) AND (STRPOS(o3[2],'idlcr8hdf.log') NE -1)) THEN BEGIN
+ PRINTF,i,'' & PRINTF,i,endtxt+'completed - Program Ended on '+SYSTIME(0)
+ ENDIF
+ENDFOR
+IF dux[1] NE dux[0] THEN FREE_LUN,dux[1]
+
+ri=WHERE(rerr EQ '',rcnt)
+IF (rcnt EQ N_ELEMENTS(rerr)) AND (~qa_yes) AND (o3[4] EQ '0') THEN BEGIN
+ IF nfile EQ 1 THEN reterr=hdffilename+' successfully created' $
+ ELSE reterr=dftxt+' files successfully created'
+ENDIF ELSE IF (rerr[1] NE 'NA') AND (rerr[1] NE '') THEN reterr=rerr[1]
+
+IF o3[3] EQ '' THEN BEGIN
+ WIDGET_CONTROL,b1,Sensitive=1,/Input_Focus
+ WIDGET_CONTROL,wtxt,set_value='',/Append,Set_text_top_line=lineno+2L
+ WIDGET_CONTROL,wtxt,set_value=endtxt+'completed - hit <Finish> to close program',/Append
+ XMANAGER,'idlcr8hdf',base
+ENDIF ELSE IF intype LT 0 THEN BEGIN ;Create Finish Dialog Box
+ res=DIALOG_MESSAGE(endtxt+'completed successfully!',/Information,Title='EVDC IDLcr8HDF')
+ENDIF
+
+END ;Procedure IDLcr8HDF
--
GitLab