Replace cds_tools.pro

cds_tools.pro

-;Main Program Version: cds_tools.pro Release v2.0b62, 20191112
+;Main Program Version: cds_tools.pro Release v2.0b63, 20200311
 ;  Written by Ian Boyd for the AVDC/EVDC - iboyd@astro.umass.edu
 ;Sub-versions:
 ;  20090828: Initial Release for testing - v0.1
@@ -163,6 +163,8 @@
 ;  20191112, Add support for v6 SHADOZ measurements. Include checks to differentiate
 ;            between the 3 different Costa Rica SHADOZ sites and make latitude and longitude
 ;            dependent on DATETIME rather than constant - v2.0b62
+;  20200311, Update due to modifications and fixes to idlcr8hdf - v2.0b63
+
 
 ;Program to read in a file or string array created by CDS_Match (or equivalent) and,
 ;based on the input information, convert NDACC NASA/AMES, WOUDC CSV, SHADOZ, NOAA-ESRL or
@@ -1593,7 +1595,7 @@ IF gcnt NE 0 THEN BEGIN
     lati=WHERE(STRMID(STRUPCASE(ndacc),0,8) EQ 'LATITUDE')
     loni=WHERE(STRMID(STRUPCASE(ndacc),0,9) EQ 'LONGITUDE')
     res=STRSPLIT(ndacc[lati[0]],' ',/EXTRACT,COUNT=cres)
-    slat=FLOAT(res[cres-1])   
+    slat=FLOAT(res[cres-1])
     res=STRSPLIT(ndacc[loni[0]],' ',/EXTRACT,COUNT=cres)
     slon=FLOAT(res[cres-1])
     latdiff=FLTARR(scnt)
@@ -2431,18 +2433,27 @@ IF ((nw EQ 'S') OR (nw EQ 'W') OR (nw EQ 'E')) AND (STRUPCASE(res[0]) EQ 'DATETI
     ENDIF
 
     ;Check for isolated out-of-order time values and convert to missing values (will be interpolated below)
+    maxo=5 ;maximum number of out-of-order values allowed
     IF mcnt GT 2L THEN BEGIN
       dsizhold=dsizx
       FOR i=0L,mcnt-2L DO BEGIN
-        bi=WHERE((dsizx[i] GT dsizx[i+1L:mcnt-1L]) AND (dsizx[i+1L:mcnt-1L] NE fv),bcnt)
-        IF bcnt NE 0 THEN dsizhold[bi+i+1L]=fv
+        ;This should find instances where timestamps are > 1000 secs different to neighbouring values
+        ;e.g. Hilo SHADOZ 2009 to 2015
+        nfvi=WHERE(dsizx[i+1L:mcnt-1L] NE fv,nfvcnt)
+        IF nfvcnt NE 0 THEN BEGIN
+          bi=WHERE(dsizx[i]-dsizx[nfvi+i+1L] GT 1000.d,bcnt)
+          IF bcnt NE 0 THEN BEGIN
+            IF nfvcnt-bcnt LE maxo THEN dsizhold[i]=fv
+          ENDIF
+        ENDIF
       ENDFOR
       ti=WHERE(dsizhold NE fv, ordchk) & ordchk=mcnt-ordchk
-      IF (ordchk LE CEIL(ndsize*0.01)) AND (ordchk LT 5) THEN BEGIN
-        ;will convert to missing if no more than of 5 out-of-order values or 1% of ndsize
+      ;IF (ordchk LE CEIL(ndsize*0.01)) AND (ordchk LT 5) THEN BEGIN
+      IF ordchk LE maxo THEN BEGIN
+        ;will convert to missing if no more than of 5 out-of-order values
         dsizx=dsizhold & dsize=dsizhold
       ENDIF ELSE IF (ordchk NE 0L) AND (nw EQ 'S') AND (dsd GT 2192.D) AND (dsd LT 2221.D) THEN BEGIN
-        ;check for Hanoi January 1996 SHADOZ files
+        ;check for Hanoi January 2006 SHADOZ files
         sti=WHERE(STRMID(STRUPCASE(ndacc),0,7) EQ 'STATION')
         IF STRPOS(STRUPCASE(ndacc[sti[0]]),'VIETNAM') NE -1 THEN BEGIN
           ;interpolate all values except for first and last
@@ -2524,7 +2535,7 @@ IF (nw EQ 'N') AND (STRUPCASE(res[0]) EQ 'DATETIME') AND (N_ELEMENTS(dsizx) GT 2
       IF dsizhold[i] NE fv THEN begin
         ai=WHERE((dsizhold[i] GT dsizhold[i+1L:ndsize-1L]) AND (dsizhold[i+1L:ndsize-1L] NE fv),acnt)
         IF acnt NE 0 THEN BEGIN
-          IF acnt NE 1 THEN print,dsizhold[i] ELSE print,dsizhold[ai+i+1L]
+          ;IF acnt NE 1 THEN PRINT,dsizhold[i] ELSE PRINT,dsizhold[ai+i+1L]
           IF acnt NE 1 THEN dsizhold[i]=fv ELSE dsizhold[ai+i+1L]=fv
           ordchk=ordchk+1L
         ENDIF
@@ -2565,12 +2576,6 @@ IF STRPOS(STRUPCASE(res[0]),'WIND.DIRECTION') NE -1 THEN BEGIN
   ENDIF
 ENDIF
 
-IF (nw EQ 'S') AND (res[0] EQ 'CURRENT.RAW.INSTRUMENT') THEN BEGIN
-  ;check for negative current values and change to fill value
-  fi=WHERE(dsize LT 0.0,fcnt)
-  IF fcnt NE 0 THEN dsize[fi]=fv
-ENDIF
-
 ;Apply conversions or scale factor to the NDACC/WOUDC/SHADOZ/NOAA data
 IF STRPOS(res[3],'MJD2000') NE -1 THEN BEGIN
   mjd2000=0.d
@@ -2651,7 +2656,7 @@ ENDIF ELSE BEGIN
 ENDELSE
 
 ;Check for out-of-range values - if more than 10% outside of VAR_VALID_MIN/MAX then change to fill values
-;Otherwise leave unchanged (will then generate error). This is designed to account for undocumented fill
+;Otherwise leave unchanged (will then generate error). This is designed to account for incorrect fill
 ;values being present in the data - especially SHADOZ and NOAA. Do not do for DATETIME type variables
 IF (STRPOS(res[0],'DATETIME') EQ -1) AND (mcnt NE 0) THEN BEGIN
   minmaxfill=DBLARR(2) ;to hold Min, Max values
@@ -2660,7 +2665,10 @@ IF (STRPOS(res[0],'DATETIME') EQ -1) AND (mcnt NE 0) THEN BEGIN
          (res[0] EQ 'O3.MIXING.RATIO_INSITU') OR (res[0] EQ 'O3.NUMBER.DENSITY_INSITU')
   ;always convert to fill values for out-of-range values for following
   oorchk=(res[0] EQ 'WIND.DIRECTION_INSITU') OR (res[0] EQ 'PRESSURE_INSITU') OR $
-         (res[0] EQ 'HUMIDITY.RELATIVE_INSITU')
+         (res[0] EQ 'HUMIDITY.RELATIVE_INSITU') OR (res[0] EQ 'CURRENT.RAW.INSTRUMENT') OR $
+         (res[0] EQ 'ALTITUDE_GPS') OR (res[0] EQ 'WIND.SPEED_INSITU') OR $
+         (res[0] EQ 'LONGITUDE') OR (res[0] EQ 'TEMPERATURE_INSITU') OR $
+         (res[0] EQ 'INTERNAL.BOX.TEMPERATURE_INSITU') OR (res[0] EQ 'LATITUDE')
   attchk=['VAR_VALID_MIN','VAR_VALID_MAX']
   FOR i=0,1 DO BEGIN
     vi=WHERE(STRMID(vat,0,STRLEN(attchk[i])) EQ attchk[i])
@@ -3191,7 +3199,7 @@ IF metaok EQ '' THEN BEGIN
             ENDIF
             ;Determine location of required dataset (divide into Section Header and Column Name components)
             lres=STRSPLIT(res[1],'_',/EXTRACT,COUNT=nlres) & lres=STRUPCASE(STRTRIM(lres,2))
-            IF (nlres GT 2) AND (nw NE 'S') THEN BEGIN 
+            IF (nlres GT 2) AND (nw NE 'S') THEN BEGIN
               ;need to append all but the last sub-value to the first value
               FOR m=1,nlres-2 DO lres[0]=lres[0]+'_'+lres[m]
               lres=[lres[0],lres[nlres-1]] & nlres=2
@@ -3218,9 +3226,9 @@ IF metaok EQ '' THEN BEGIN
                 ltlni=WHERE(STRMID(STRUPCASE(ndacc),0,STRLEN(vres[0])) EQ vres[0])
                 vres=STRSPLIT(ndacc[ltlni[0]],' ',/EXTRACT,COUNT=cres)
                 sltln=FLOAT(vres[cres-1])
-              ENDELSE              
+              ENDELSE
             ENDIF
-                        
+
             IF ((nw EQ 'S') OR (nw EQ 'E')) AND (lres[0] EQ '#PROFILE') THEN scnt=1 $
             ELSE si=WHERE(STRMID(STRUPCASE(ndacc),0,STRLEN(lres[0])) EQ lres[0],scnt)
             IF (scnt NE 0) OR ((lc EQ sccnt) AND (sccnt GT 1)) OR (aplabel[0] NE '-1') THEN BEGIN
@@ -3272,7 +3280,7 @@ IF metaok EQ '' THEN BEGIN
                       IF STRUPCASE(STRMID(aplabel[lc,0],0,29)) EQ 'CORRECTION FACTOR (COL2/COL1)' THEN $
                         ;special case for correction factor - change to -9 to indicate no correction applied
                         aplabel[lc,1]='-9.0000'
-                    ENDIF  
+                    ENDIF
                   ENDIF
                 ENDELSE
                 IF N_ELEMENTS(resy) EQ 1 THEN BEGIN ;No accompanying info to the header so add values
@@ -3352,13 +3360,13 @@ IF metaok EQ '' THEN BEGIN
                 ENDFOR
               ENDIF
             ENDIF
-            
+
             ;Add station latitude and longitude if no GPS values are available for SHADOZ
             IF ((res[0] EQ 'LATITUDE') OR (res[0] EQ 'LONGITUDE')) AND (nw EQ 'S') THEN BEGIN
               IF (FLOAT(holddata[0,j]) EQ FLOAT(mv)) AND (sltln NE -999.) THEN $
                 holddata[0,j]=STRTRIM(sltln,2)
             ENDIF
-            
+
             ;Do check for missing DATETIME values - if so add comment to VAR_NOTES
             IF res[0] EQ 'DATETIME' THEN BEGIN
               IF nw EQ 'S' THEN fi=WHERE(DOUBLE(holddata[*,j]) EQ DOUBLE(mv),fcnt) $
@@ -3761,7 +3769,7 @@ FUNCTION cds_tools, listf, tav, outdir, dataformat, DATETIME=o1
 
 ;Initialize statusflag, and exit_code
 statusflag='' & dum='' & exit_code=3
-cdver='cds_tools version 2.0b62, 2019-11-12'
+cdver='cds_tools version 2.0b63, 2020-03-11'
 
 ;Check input parameters
 IF N_PARAMS() LT 3 THEN $