Commit dab9306c authored by Ian Boyd's avatar Ian Boyd
Browse files

Replace idlcr8ascii.pro

parent e7e6271b
;Main Program Version: idlcr8ascii.pro v4.0b21, 20190514
;Main Program Version: idlcr8ascii.pro v4.0b22, 20190806
; Written by Ian Boyd for the AVDC - iboyd@astro.umass.edu
;
;Sub-versions:
......@@ -76,6 +76,12 @@
; v4.0b21 20190514 - Fix bug that occurred when GEOMS variable attribute information is missing from an
; HDF4 file but written to the heap structure based on the contents of the file. Previously
; the variable attribute label was not written correctly.
; v4.0b22 20190806 - Provide support for reading netCDF4 files. Currently uses the HDF5 routines to do this
; so need to account for features unique to netCDF4 such as; the inclusion of dimension
; names; standard netCDF attribute names; empty attribute values that do not include a
; null termination character (which causes an error in the H5A_Read routine). New
; sub-routines include is_a_number_ascii, alpha_numeric_underscore, nc_dimension_chk,
; file_format_a.
PRO idlcr8ascii_common
;Procedure to define the data COMMON block WIDGET_WIN_A, containing common variables
......@@ -195,7 +201,7 @@ COMMON WIDGET_WIN_A
;procedure which provides an introduction message before starting the program.
nhdr=46 & errtxt=STRARR(nhdr)
vertxt=['idlcr8ascii-v4.0_Readme.pdf','v4.0b21 May 2019']
vertxt=['idlcr8ascii-v4.0_Readme.pdf','v4.0b22 August 2019']
errtxt[1]='Welcome to IDLcr8ASCII. This program reads GEOMS compliant HDF4, HDF5 and netCDF files and'
errtxt[2]='saves contents to either session memory, an output window (summary only) or to ASCII or formatted'
errtxt[3]='files (also refer to '+vertxt[0]+').'
......@@ -214,25 +220,25 @@ errtxt[17]=' /P or /POPUP - sends log input/output information to a Pop-up Dial
errtxt[18]=' /L or /LOG - sends log input/output information to a file named idlcr8ascii.log'
errtxt[19]=' /H4 - outputs contents of the input file as an HDF4 file'
errtxt[20]=' /H5 - outputs contents of the input file as an HDF5 file'
errtxt[21]=' /NC - outputs contents of the input file as a netCDF file'
errtxt[23]='Example of command line input: idlcr8ascii,DFSpec[,/F][,/D][,/C][,/P][,/L][,/H4][,/H5][,/NC]'
errtxt[21]=' /NC3 - outputs contents of the input file as a netCDF3 file'
errtxt[23]='Example of command line input: idlcr8ascii,DFSpec[,/F][,/D][,/C][,/P][,/L][,/H4][,/H5][,/NC3]'
errtxt[24]=' where ''DFSpec'' can either be a string array containing filenames or a scalar string containing a'
errtxt[25]=' filespec or a single file name.'
errtxt[26]='If running the full version of IDL, and input is a single HDF file, output can also be saved'
errtxt[27]=' to session memory. This can be done by calling idlcr8ascii with the following command line:'
errtxt[28]=' idlcr8ascii,DFFile,GA,SDS[,/F][,/D][,/C][,/P][,/L][,/H4][,/H5][,/NC].'
errtxt[28]=' idlcr8ascii,DFFile,GA,SDS[,/F][,/D][,/C][,/P][,/L][,/H4][,/H5][,/NC3].'
errtxt[29]='DFFile can either be the name of an input file or '''' (in which case a DIALOG_BOX will open'
errtxt[30]=' and prompt for the input filename). ''GA'' is a returned string array containing the Global'
errtxt[31]=' Attributes, and ''SDS'' is a returned structure using pointers containing the Variable'
errtxt[32]=' Attribute Labels (sds.va_l), Variable Attribute Values (sds.va_v), and the Data (sds.data).
errtxt[34]='Contacts -'
errtxt[35]=' Ian Boyd, University of Massachusetts (iboyd@astro.umass.edu)'
errtxt[36]=' Department of Astronomy, 619 Lederle GRC, University of Massachusetts'
errtxt[37]=' 710 North Pleasant St, Amherst, MA 01002, USA'
errtxt[39]=' Ghassan Taha, AVDC Project Coordinator (ghassan.taha@nasa.gov)'
errtxt[40]=' NASA Goddard Space Flight Center, Code 613.3'
errtxt[41]=' Greenbelt, MD 20771, USA'
errtxt[43]='AVDC Website: Tools and documentation available from http://avdc.gsfc.nasa.gov/Overview/index.html'
errtxt[35]=' Ian Boyd, BC Scientific Consulting LLC (iboyd@astro.umass.edu)'
errtxt[36]=' 26 Campbells Rd'
errtxt[37]=' Pine Hill, Dunedin 9010, New Zealand'
errtxt[39]=' Ann Mari Fjaeraa, EVDC Project Manager (amf@nilu.no)'
errtxt[40]=' Norwegian Institute for Air Research, Instituttveien 18'
errtxt[41]=' Postbox 100, N-2027 KJELLER, NORWAY'
errtxt[43]='EVDC Website: Tools and documentation available from http://evdc.esa.int/'
errtxt[45]='To continue, please choose from the options below.'
errtxt=' '+errtxt
......@@ -240,7 +246,7 @@ errtxt=' '+errtxt
demomode=LMGR(/DEMO) & optsens=1
IF intype EQ -3 THEN xtxt=' - Command Line Input Error' ELSE xtxt=''
base=WIDGET_BASE(Title='idlcr8ascii '+vertxt[1]+xtxt,Tlb_Frame_Attr=1,/Column) ;,Tab_Mode=1)
wtxt=WIDGET_TEXT(base,xsize=88,ysize=25,/Scroll)
wtxt=WIDGET_TEXT(base,xsize=90,ysize=25,/Scroll)
tip='Left Mouse Click or Tab to entry and hit <Spacebar>'
base2=WIDGET_BASE(base,/Nonexclusive)
......@@ -252,8 +258,8 @@ IF demomode THEN BEGIN
ENDIF ELSE logtext=logtext+'(will create the file IF it doesn''t exist)'
h4text='Convert input file(s) to HDF4'
h5text='Convert input file(s) to HDF5'
nctext='Convert input file(s) to netCDF'
fctext1='For the following options, any existing files will be overwritten, IDLcr8HDF must be in the IDL Search Path,'
nctext='Convert input file(s) to netCDF3'
fctext1='For the following 3 options, any existing files will be overwritten, IDLcr8HDF must be in the IDL Search Path,'
fctext2='and a valid Table Attribute Values (TAV) file must be in the same directory as the Input file(s).'
b1=WIDGET_BUTTON(base2,value=cattxt,uvalue='C',frame=3)
b2=WIDGET_BUTTON(base2,value=poptext,uvalue='P',frame=3)
......@@ -382,6 +388,54 @@ END ;Procedure Stop_With_Error_A
FUNCTION is_a_number_ascii, value
ON_IOERROR, ConversionError
IF STRTRIM(value,2) EQ '' THEN RETURN, 0B
n=DOUBLE(value)
RETURN, 1B
ConversionError:
RETURN, 0B
END
FUNCTION alpha_numeric_underscore, str_value
;Function that returns the input string with non-alphanumeric characters replaced with '_'
; ----------
;Written by Ian Boyd for the EVDC - iboyd@astro.umass.edu
;
; History:
; 20190806: Introduced - Version 4.0b22
;
; Input: str_value: string
;
; Returns: string with non-alphanumeric values replaced with '_'
;
; Called by: Read_HDF_SDS
;
ON_IOERROR, ConversionError
numb=BINDGEN(10)+48B ;0-9
ucb=BINDGEN(26)+65B ;A-Z
lcb=BINDGEN(26)+97B ;a-z
str_value=STRING(str_value)
new_str_value=str_value
n_char=STRLEN(str_value)
FOR i=0L,n_char-1L DO BEGIN
byt_value=BYTE(STRMID(new_str_value,i,1))
testn=(byt_value GE numb[0]) AND (byt_value LE numb[9])
testu=(byt_value GE ucb[0]) AND (byt_value LE ucb[25])
testl=(byt_value GE lcb[0]) AND (byt_value LE lcb[25])
IF (~testn) AND (~testu) AND (~testl) THEN $
STRPUT,new_str_value,'_',i
ENDFOR
RETURN, new_str_value
ConversionError:
RETURN, 'io_error'
END ;Function alpha_numeric_underscore
PRO infotxt_output_a, in0, in1
;Procedure called to report information relevant to the reading of the HDF file.
;This information can be reported to a Pop-up logging window, IDLDE output log window, and/or
......@@ -415,10 +469,10 @@ CASE 1 OF
infotxt=STRARR(3)
IF LONG(in1[0]) EQ 0L THEN BEGIN
infotxt[0]=' INFORMATION: Global Attribute DATA_VARIABLES not found or has no values.'
infotxt[1]=' Dataset listing order matches the order that the dataset is read in the HDF file.'
infotxt[1]=' Dataset listing order matches the order that the dataset is read in the GEOMS file.'
ENDIF ELSE BEGIN
infotxt[0]=' INFORMATION: Number of Global Attribute DATA_VARIABLES values ('+in1[0]+')'
infotxt[1]=' does not match the number of datasets saved to the HDF file ('+in1[1]+')'
infotxt[1]=' does not match the number of datasets saved to the GEOMS file ('+in1[1]+')'
ENDELSE
infotxt[2]=' Number of datasets determined from '+in1[2]+' call'
END
......@@ -737,6 +791,142 @@ END ;Function Julian_Date
PRO nc_dimension_chk, ftype, sd_id, di, n_sds, dgi, n_sdsg
;Procedure to check for netCDF dimension datasets. These will be quietly ignored during the
;read process
; ----------
;Written by Ian Boyd for the EVDC - iboyd@astro.umass.edu
;
; History:
; 20190806: Introduced - Version 4.0b22
;
; Inputs: ftype - format type, HDF5 or netCDF
; sd_id - GEOMS file identifier number
; di - index values for datasets
; n_sds - number of datasets determined by H5G_GET_NMEMBERS call
;
; Outputs: dgi - index values for GEOMS datasets (not generated by netCDF libraries)
; n_sdsg - number of GEOMS datasets
;
; Called by: Read_HDF_SDS
;
dimen_names=['constant','independent_','_strlen']
n_dn=N_ELEMENTS(dimen_names)
ds_names=STRARR(n_sds) & gdi=INTARR(n_sds)-1
FOR i=0,n_sds-1 DO BEGIN
IF ftype EQ 'H5' THEN sds_name=H5G_GET_MEMBER_NAME(sd_id,'/',di[i]) $
ELSE BEGIN ;netCDF file
varstruct=NCDF_VARINQ(sd_id,i)
sds_name=varstruct.name
ENDELSE
ds_names[i]=STRTRIM(STRLOWCASE(sds_name),2)
ENDFOR
FOR i=0,n_sds-1 DO BEGIN
dnicnt=0 & j=0
WHILE (dnicnt EQ 0) AND (j LE n_dn-1) DO $
IF STRPOS(ds_names[i],dimen_names[j]) NE -1 THEN dnicnt++ ELSE j++
IF dnicnt NE 0 THEN BEGIN
;Check that names are actually netCDF dimension names o/w will treat as a dataset name
CASE j OF
0: IF ds_names[i] NE dimen_names[j] THEN dnicnt=0 ;i.e. not a netCDF dimension name
1: BEGIN
;check that first part of the name is INDEPENDENT_ and the second part is a number
IF STRMID(ds_names[i],0,STRLEN(dimen_names[j])) NE dimen_names[j] $
THEN dnicnt=0 $
ELSE BEGIN
res=STRSPLIT(ds_names[i],'_',/EXTRACT,COUNT=cres)
res=[res,''] ;to ensure at least 2 extracted values in ds_names[i]
if ~IS_A_NUMBER_ASCII(res[1]) THEN dnicnt=0 ; i.e. not a netCDF dimension name
ENDELSE
END
2: BEGIN
;check that first part of the name is a named dataset
res=STRSPLIT(ds_names[i],'_',/EXTRACT)
ni=WHERE(res[0] EQ ds_names,ncnt)
IF ncnt EQ 0 THEN dnicnt=0 ;i.e. not a netCDF dimension name
END
ENDCASE
ENDIF
IF dnicnt EQ 0 THEN gdi[i]=i ;i.e. this is a valid dataset name and not a netCDF dimension name
ENDFOR
dgi=WHERE(gdi NE -1,n_sdsg) ;dgi and n_sdsg are returned by the procedure
END ;Procedure nc_dimension_chk
FUNCTION file_format_a, infile
;Function to identify the format of the input file - tests for HDF4, HDF5/netCDF4 and netCDF3.
;netCDF4 files will be identified as HDF5 and read in using the HDF5 library
; ----------
;Written by Ian Boyd for the EVDC - iboyd@astro.umass.edu
;
; History:
; 20190806: Introduced - Version 4.0b22
;
; Inputs: infile - Input GEOMS compliant file
;
; Outputs: string identifying the type of file (H4, H5, NC) or XX for unidentified file
;
; Called by: IDLCR8ASCII
;
;Markers
;HDF4: 14 3 19 (1 0 16 0 1 95 98 0 30)
;HDF5: 137 72 68 70 (13 10 26 10 0 0 0 0)
;nc3: 67 68 70 (1 0 0 0 0 0 0 0 10)
;nc4: 137 72 68 70 (13 10 26 10 (2 8 8 0))
h4id=[14B, 3B, 19B]
h5id=[137B, 72B, 68B, 70B]
nc3id=[67B, 68B, 70B]
;First test using start of file byte markers
fid=BYTARR(4)
openr,fu,infile,/GET_LUN
readu,fu,fid
FREE_LUN,fu
idfound='XX'
CASE 1 OF
ARRAY_EQUAL(fid[0:2],h4id): idfound='H4'
ARRAY_EQUAL(fid,h5id): idfound='H5'
ARRAY_EQUAL(fid[0:2],nc3id): idfound='NC'
ELSE:
ENDCASE
;Second test, if required, using IDL commands
IF idfound EQ 'XX' THEN BEGIN
IF HDF_ISHDF(infile) EQ 1 THEN idfound='H4' $
ELSE IF FLOAT(!Version.Release) GE 5.6 THEN BEGIN
IF H5F_IS_HDF5(infile) EQ 1 THEN idfound='H5' ;netCDF4 or HDF5
ENDIF
IF idfound EQ 'XX' THEN BEGIN
;final test for netCDF3
validname=1
CATCH, ncdferror ;To catch the error when attempting to open the file as a netCDF
;Error Handler
IF ncdferror NE 0 THEN BEGIN ;not a netCDF file
validname=0
CATCH, /CANCEL
ENDIF
IF validname EQ 1 THEN BEGIN
fileid=NCDF_OPEN(infile)
;If get to here then file is netCDF
CATCH, /CANCEL
NCDF_CLOSE,fileid
idfound='NC'
ENDIF
ENDIF
ENDIF
RETURN, idfound
END ;File_Format_A
PRO test_dim_order, va_name, va_value, va_type, sds_dim, rev_vd_vs
;Procedure to test the dimension ordering of any multi-dimensional datasets in the input DF file, and
;return correct ordering code (dimension ordering for idlcr8ascii uses IDL/Fortran convention)
......@@ -752,7 +942,6 @@ PRO test_dim_order, va_name, va_value, va_type, sds_dim, rev_vd_vs
; 20141110: Fix bug that caused multi-dimensional array ordering to not be correctly identified
; if the first dataset to be checked in the file had the same number of elements in
; the array (e.g. was a set of Averaging Kernels) - Version 4.0b9
;
; Inputs: va_name - An abbreviated version of the Variable Name; either 'VD' (VAR_DEPEND) or 'VS'
; (VAR_SIZE)
......@@ -893,6 +1082,11 @@ PRO read_hdf_sds, ifile, ga, sds, catinfo
; 20190514 Fix bug that occurred when GEOMS variable attribute information is missing from an
; HDF4 file but written to the heap structure based on the contents of the file. Previously
; the variable attribute label was not written correctly - Version 4.0b21
; 20190806 Provide support for reading netCDF4 files. Currently uses the HDF5 routines to do this
; so need to account for features unique to netCDF4 such as; the inclusion of dimension
; names; standard netCDF attribute names; empty attribute values that do not include a
; null termination character (which causes an error in the H5A_Read routine)
; - Version 4.0b22
;
; Inputs: ifile - a string containing the name of the input file to be read in.
; catinfo - a string array identifying the type of input file ('H4','H5','NC')
......@@ -1264,10 +1458,18 @@ IF catinfo[0,0] EQ 'H4' THEN BEGIN
!QUIET=0 ;Allow system error and information messages
;The HDF_SD_END function closes the SD interface to an HDF file.
HDF_SD_END,sd_id
ENDIF ELSE IF catinfo[0,0] EQ 'H5' THEN BEGIN ;HDF5 format file
ENDIF ELSE IF catinfo[0,0] EQ 'H5' THEN BEGIN ;HDF5 or netCDF4 format file
stdfm=1 ;Boolean indicating standard AVDC/EVDC/NDACC H5 format or not
dv_order=1 ;Boolean indicating whether Metadata Variable listing order matches DATA_VARIABLES values
n_vn=0L ;Initialize the number of Variable Names in the standard format h5 file
;labels to quietly ignore if the file is netCDF4
nc4_ignore=['reference_list','dimension_list','class','name','_netcdf4dimid']
;Do bulk read so errors generated by netCDF4 inconsistencies can be identified without
;crashing the program e.g. null string attribute values
h5p=H5_PARSE(ifile)
hdf_file_id=H5F_OPEN(ifile)
sd_id=H5G_OPEN(hdf_file_id,'/')
......@@ -1302,7 +1504,14 @@ ENDIF ELSE IF catinfo[0,0] EQ 'H5' THEN BEGIN ;HDF5 format file
FOR i=0L,n_ga-1L DO BEGIN
ga_id=H5A_OPEN_IDX(sd_id,i)
ga_name=H5A_GET_NAME(ga_id) & ga_name=STRTRIM(ga_name,2)
ga_data=H5A_READ(ga_id) & ga_data=STRTRIM(ga_data,2)
;test for valid attribute value - can be invalid if it is netCDF4 and equal to ''
gni=WHERE(TAG_NAMES(h5p) EQ ga_name)
attest=h5p.(gni[0])
IF attest._data EQ '<read error>' THEN ga_data='' $
ELSE BEGIN
ga_data=H5A_READ(ga_id) & ga_data=STRTRIM(ga_data,2)
ENDELSE
;Check for _GLOSDS suffix and strip (added for compatibility with H4toH5 utility)
IF STRMID(STRUPCASE(ga_name),STRLEN(ga_name)-7) EQ '_GLOSDS' THEN $
ga_name=STRMID(ga_name,0,STRPOS(ga_name,'_',/Reverse_Search))
......@@ -1318,6 +1527,8 @@ ENDIF ELSE IF catinfo[0,0] EQ 'H5' THEN BEGIN ;HDF5 format file
ENDIF
IF n_sds NE 0L THEN BEGIN
NC_DIMENSION_CHK,'H5',sd_id,di,n_sds,dgi,n_sdsg ;check for netCDF dimension names (will quietly ignore)
di=dgi & n_sds=n_sdsg
IF n_vn NE n_sds THEN $
INFOTXT_OUTPUT_A,[0],[STRTRIM(n_vn,2),STRTRIM(n_sds,2),'H5G_GET_NMEMBERS']
;DATA_VARIABLES values not read successfully, or number of datasets given under DATA_VARIABLES
......@@ -1330,6 +1541,9 @@ ENDIF ELSE IF catinfo[0,0] EQ 'H5' THEN BEGIN ;HDF5 format file
FOR i=0,n_sds-1 DO BEGIN
sds_name=H5G_GET_MEMBER_NAME(sd_id,'/',di[i])
ds_id=H5D_OPEN(sd_id,sds_name)
;need to swap non-alphanumeric characters to '_' to match names from h5_parse for attribute values test
h5p_name=ALPHA_NUMERIC_UNDERSCORE(STRUPCASE(sds_name))
IF sds_name EQ '' THEN sds_name='N/A'
datasize=H5D_READ(ds_id)
sds_dim=SIZE(datasize,/DIMENSIONS) & n_sds_dim=N_ELEMENTS(sds_dim)
......@@ -1343,46 +1557,58 @@ ENDIF ELSE IF catinfo[0,0] EQ 'H5' THEN BEGIN ;HDF5 format file
vnf=0 & lcnt=0 & vnv=''
ci=STRARR(4) ;array containing catalog attributes
;Read in Dataset Attributes
jj=0 ;use this index in the event of invalid attribute names (e.g. dimension_list from netCDF4 file)
FOR j=0L,sds_natts-1L DO BEGIN
da_id=H5A_OPEN_IDX(ds_id,j)
da_name=H5A_GET_NAME(da_id) & da_name=STRTRIM(da_name,2)
va_value=H5A_READ(da_id)
IF SIZE(va_value,/TYPE) EQ 7 THEN va_type='STRING' ELSE va_type='NON_STRING'
schk=WHERE(STRUPCASE(da_name) EQ slabel,schkcnt)
IF ((sds_type EQ 7) AND (schkcnt EQ 0)) OR ((sds_type NE 7) AND (va_type EQ 'STRING')) THEN $
va_value=STRTRIM(va_value,2)
CASE 1 OF
(STRUPCASE(da_name) EQ 'VAR_NAME') AND (vnf EQ 0):BEGIN
vnf=1
;check that the dataset name matches the VAR_NAME
IF STRUPCASE(va_value) NE STRUPCASE(sds_name) THEN BEGIN
;Try and determine which is wrong - Dataset name or VAR_NAME - default is Dataset name
li=WHERE((STRUPCASE(sds_name) EQ STRUPCASE(vn)) AND (vn[0] NE ''),lcnt)
;If lcnt NE 0 then Dataset name matches DATA_VARIABLES value so assume that VAR_NAME is wrong
IF lcnt NE 0 THEN usesds=1 ELSE usesds=0
INFOTXT_OUTPUT_A,[3,5,usesds],[sds_name,va_value]
IF usesds EQ 0 THEN sds_name=va_value ELSE va_value=sds_name
ENDIF
;Determine actual list order from DATA_VARIABLE listing (H5 lists datasets alphabetically)
li=WHERE(STRUPCASE(va_value[0]) EQ STRUPCASE(vn),lcnt)
ci[0]=va_value & vnv=va_value
END
STRUPCASE(da_name) EQ 'VAR_DATA_TYPE': BEGIN
ci[1]=va_value
vi=WHERE(STRUPCASE(va_value[0]) EQ avdt,vcnt)
IF vcnt EQ 0 THEN INFOTXT_OUTPUT_A,[2],[sds_name,va_value]
END
STRUPCASE(da_name) EQ 'VAR_DEPEND': BEGIN
TEST_DIM_ORDER,'VD',va_value,va_type,sds_dim,rev_vd_vs
END
STRUPCASE(da_name) EQ 'VAR_SIZE': BEGIN
TEST_DIM_ORDER,'VS',va_value,va_type,sds_dim,rev_vd_vs
END
ELSE:
ENDCASE
nc4i=WHERE(STRLOWCASE(da_name) EQ nc4_ignore,nc4icnt)
IF nc4icnt EQ 0 THEN BEGIN
;test for valid attribute value - can be invalid if it is netCDF4 and equal to ''
gni=WHERE(TAG_NAMES(h5p) EQ h5p_name) ;Tag index from h5_parse read
attest=h5p.(gni[0])
gni=WHERE(TAG_NAMES(attest) EQ da_name)
attestx=attest.(gni[0])
IF STRTRIM(attestx._data,2) EQ '<read error>' THEN va_value='' $
ELSE va_value=H5A_READ(da_id)
IF SIZE(va_value,/TYPE) EQ 7 THEN va_type='STRING' ELSE va_type='NON_STRING'
schk=WHERE(STRUPCASE(da_name) EQ slabel,schkcnt)
IF ((sds_type EQ 7) AND (schkcnt EQ 0)) OR ((sds_type NE 7) AND (va_type EQ 'STRING')) THEN $
va_value=STRTRIM(va_value,2)
CASE 1 OF
(STRUPCASE(da_name) EQ 'VAR_NAME') AND (vnf EQ 0):BEGIN
vnf=1
;check that the dataset name matches the VAR_NAME
IF STRUPCASE(va_value) NE STRUPCASE(sds_name) THEN BEGIN
;Try and determine which is wrong - Dataset name or VAR_NAME - default is Dataset name
li=WHERE((STRUPCASE(sds_name) EQ STRUPCASE(vn)) AND (vn[0] NE ''),lcnt)
;If lcnt NE 0 then Dataset name matches DATA_VARIABLES value so assume that VAR_NAME is wrong
IF lcnt NE 0 THEN usesds=1 ELSE usesds=0
INFOTXT_OUTPUT_A,[3,5,usesds],[sds_name,va_value]
IF usesds EQ 0 THEN sds_name=va_value ELSE va_value=sds_name
ENDIF
;Determine actual list order from DATA_VARIABLE listing (H5 lists datasets alphabetically)
li=WHERE(STRUPCASE(va_value[0]) EQ STRUPCASE(vn),lcnt)
ci[0]=va_value & vnv=va_value
END
STRUPCASE(da_name) EQ 'VAR_DATA_TYPE': BEGIN
ci[1]=va_value
vi=WHERE(STRUPCASE(va_value[0]) EQ avdt,vcnt)
IF vcnt EQ 0 THEN INFOTXT_OUTPUT_A,[2],[sds_name,va_value]
END
STRUPCASE(da_name) EQ 'VAR_DEPEND': BEGIN
TEST_DIM_ORDER,'VD',va_value,va_type,sds_dim,rev_vd_vs
END
STRUPCASE(da_name) EQ 'VAR_SIZE': BEGIN
TEST_DIM_ORDER,'VS',va_value,va_type,sds_dim,rev_vd_vs
END
ELSE:
ENDCASE
jj++
ENDIF ;ELSE IF nc4i[0] EQ 4 THEN catinfo[0,0]='H5N' ;netCDF4 file - not implemented yet
H5A_CLOSE,da_id
ENDFOR
ci[3]=STRTRIM(sds_natts,2)
ci[3]=STRTRIM(jj,2)
IF ci[0] EQ '' THEN ci[0]=sds_name
ENDIF ELSE BEGIN ;No Attributes, so can only write dataset name to Metadata
ci=[STRUPCASE(sds_name),'','','1'] & vnv=sds_name
......@@ -1426,6 +1652,9 @@ ENDIF ELSE IF catinfo[0,0] EQ 'H5' THEN BEGIN ;HDF5 format file
notranspose=1 ;0/1 will change to 0 if the dataset needs to be transposed
sds_name=H5G_GET_MEMBER_NAME(sd_id,'/',di[i])
ds_id=H5D_OPEN(sd_id,sds_name)
;need to swap non-alphanumeric characters to '_' to match names from h5_parse for attribute values test
h5p_name=ALPHA_NUMERIC_UNDERSCORE(STRUPCASE(sds_name))
IF sds_name EQ '' THEN sds_name='N/A'
datasize=H5D_READ(ds_id)
sds_type=SIZE(datasize,/TYPE)
......@@ -1434,38 +1663,50 @@ ENDIF ELSE IF catinfo[0,0] EQ 'H5' THEN BEGIN ;HDF5 format file
IF sds_natts NE 0L THEN BEGIN
;Read in Dataset Attributes
jj=0 ;use this index in the event of invalid attribute names (e.g. dimension_list fro netCDF4 file)
FOR j=0L,sds_natts-1L DO BEGIN
da_id=H5A_OPEN_IDX(ds_id,j)
da_name=H5A_GET_NAME(da_id) & da_name=STRTRIM(da_name,2)
va_value=H5A_READ(da_id)
IF SIZE(va_value,/TYPE) EQ 7 THEN va_type='STRING' ELSE va_type='NON_STRING'
schk=WHERE(STRUPCASE(da_name) EQ slabel,schkcnt)
IF ((sds_type EQ 7) AND (schkcnt EQ 0)) OR ((sds_type NE 7) AND (va_type EQ 'STRING')) THEN $
va_value=STRTRIM(va_value,2)
H5A_CLOSE,da_id
nc4i=WHERE(STRLOWCASE(da_name) EQ nc4_ignore,nc4icnt)
IF nc4icnt EQ 0 THEN BEGIN
;test for valid attribute value - can be invalid if it is netCDF4 and equal to ''
gni=WHERE(TAG_NAMES(h5p) EQ h5p_name) ;Tag index from h5_parse read
attest=h5p.(gni[0])
gni=WHERE(TAG_NAMES(attest) EQ da_name)
attestx=attest.(gni[0])
IF STRTRIM(attestx._data,2) EQ '<read error>' THEN va_value='' $
ELSE va_value=H5A_READ(da_id)
IF ((STRUPCASE(da_name) EQ 'VAR_DEPEND') OR (STRUPCASE(da_name) EQ 'VAR_SIZE')) AND $
((rev_vd_vs MOD 10 EQ 1) OR (rev_vd_vs EQ 20)) THEN BEGIN ;need to reverse the va_values
IF SIZE(va_value,/TYPE) EQ 7 THEN va_type='STRING' ELSE va_type='NON_STRING'
schk=WHERE(STRUPCASE(da_name) EQ slabel,schkcnt)
IF ((sds_type EQ 7) AND (schkcnt EQ 0)) OR ((sds_type NE 7) AND (va_type EQ 'STRING')) THEN $
va_value=STRTRIM(va_value,2)
H5A_CLOSE,da_id
IF va_type NE 'STRING' THEN BEGIN ;VAR_SIZE contains numeric values instead of in the form of a string
vavhold='' & n_vav=N_ELEMENTS(va_value)
FOR k=0,n_vav-1 DO BEGIN
IF k EQ n_vav-1 THEN vtxt='' ELSE vtxt=';'
vavhold=vavhold+STRTRIM(va_value[k],2)+vtxt
ENDFOR
ENDIF ELSE vavhold=va_value
IF ((STRUPCASE(da_name) EQ 'VAR_DEPEND') OR (STRUPCASE(da_name) EQ 'VAR_SIZE')) AND $
((rev_vd_vs MOD 10 EQ 1) OR (rev_vd_vs EQ 20)) THEN BEGIN ;need to reverse the va_values
vs_v=STRCOMPRESS(STRSPLIT(vavhold,';',/EXTRACT,COUNT=rcnt),/REMOVE_ALL)
IF rcnt GT 1 THEN BEGIN ;multi-dimensions
IF (rev_vd_vs EQ 20) THEN $ ;AND (STRPOS(STRUPCASE(va_value),'DATETIME') NE -1) THEN $
notranspose=0 ;need to transpose dataset as well
vs_v=REVERSE(vs_v)
FOR k=0,rcnt-1 DO $
IF k EQ 0 THEN va_value=vs_v[k] ELSE va_value=va_value+';'+vs_v[k]
IF va_type NE 'STRING' THEN BEGIN ;VAR_SIZE contains numeric values instead of in the form of a string
vavhold='' & n_vav=N_ELEMENTS(va_value)
FOR k=0,n_vav-1 DO BEGIN
IF k EQ n_vav-1 THEN vtxt='' ELSE vtxt=';'
vavhold=vavhold+STRTRIM(va_value[k],2)+vtxt
ENDFOR
ENDIF ELSE vavhold=va_value
vs_v=STRCOMPRESS(STRSPLIT(vavhold,';',/EXTRACT,COUNT=rcnt),/REMOVE_ALL)
IF rcnt GT 1 THEN BEGIN ;multi-dimensions
IF (rev_vd_vs EQ 20) THEN $ ;AND (STRPOS(STRUPCASE(va_value),'DATETIME') NE -1) THEN $
notranspose=0 ;need to transpose dataset as well
vs_v=REVERSE(vs_v)
FOR k=0,rcnt-1 DO $
IF k EQ 0 THEN va_value=vs_v[k] ELSE va_value=va_value+';'+vs_v[k]
ENDIF
ENDIF
sds[oi[i],jj].va_l=PTR_NEW(da_name)
sds[oi[i],jj].va_v=PTR_NEW(va_value)
jj++
ENDIF
sds[oi[i],j].va_l=PTR_NEW(da_name)
sds[oi[i],j].va_v=PTR_NEW(va_value)
ENDFOR
ENDIF ELSE BEGIN ;No Attributes, so can only write dataset name to structure
val='VAR_NAME'
......@@ -1481,12 +1722,13 @@ ENDIF ELSE IF catinfo[0,0] EQ 'H5' THEN BEGIN ;HDF5 format file
H5G_CLOSE,sd_id
H5F_CLOSE,hdf_file_id
ENDIF ELSE BEGIN ;netCDF file
ENDIF ELSE BEGIN ;netCDF3 file
ncdf_desc=['long_name']
ncdf_std=['units','valid_range','_fillvalue']
ncdf_dimen=['constant','independent']
dv_order=1 ;Boolean indicating whether Metadata Variable listing order matches DATA_VARIABLES values
n_vn=0L ;Initialize the number of Variable Names in the standard format h5 file
n_vn=0L ;Initialize the number of Variable Names in the standard format netCDF file
fileid=NCDF_OPEN(ifile)
;Get number of global atts and variables
......@@ -1523,6 +1765,10 @@ ENDIF ELSE BEGIN ;netCDF file
ENDFOR
ENDIF ELSE mv_dim=['']
di=INDGEN(n_sds)
NC_DIMENSION_CHK,'NC',fileid,di,n_sds,dgi,n_sdsg
di=dgi & n_sds=n_sdsg ;di contains index values of valid sds datasets
IF n_vn NE n_sds THEN $
INFOTXT_OUTPUT_A,[0],[STRTRIM(n_vn,2),STRTRIM(n_sds,2),'NCDF_INQUIRE']
;DATA_VARIABLES values not read successfully, or number of datasets given under DATA_VARIABLES
......@@ -1533,7 +1779,7 @@ ENDIF ELSE BEGIN ;netCDF file
catinfo=STRARR(n_sds,4) ;output info for catalog output
FOR j=0,n_sds-1 DO BEGIN
varstruct=NCDF_VARINQ(fileid,j)
varstruct=NCDF_VARINQ(fileid,di[j])
sds_natts=varstruct.natts
IF sds_natts GT max_atts THEN max_atts=sds_natts
sds_name=varstruct.name
......@@ -1554,8 +1800,8 @@ ENDIF ELSE BEGIN ;netCDF file
std_fnd=INTARR(n_aad)
;Read in Dataset Attributes
FOR k=0,sds_natts-1 DO BEGIN
varattsname=NCDF_ATTNAME(fileid,j,k)
NCDF_ATTGET,fileid,j,varattsname,va_value
varattsname=NCDF_ATTNAME(fileid,di[j],k)