Commit 937dfbae authored by Ian Boyd's avatar Ian Boyd

Replace cds_tools.pro

parent e12b07fb
;Main Program Version: cds_tools.pro Release v2.0b62, 20191112
;Main Program Version: cds_tools.pro Release v2.0b63, 20200311
; Written by Ian Boyd for the AVDC/EVDC - iboyd@astro.umass.edu
;Sub-versions:
; 20090828: Initial Release for testing - v0.1
......@@ -163,6 +163,8 @@
; 20191112, Add support for v6 SHADOZ measurements. Include checks to differentiate
; between the 3 different Costa Rica SHADOZ sites and make latitude and longitude
; dependent on DATETIME rather than constant - v2.0b62
; 20200311, Update due to modifications and fixes to idlcr8hdf - v2.0b63
;Program to read in a file or string array created by CDS_Match (or equivalent) and,
;based on the input information, convert NDACC NASA/AMES, WOUDC CSV, SHADOZ, NOAA-ESRL or
......@@ -1593,7 +1595,7 @@ IF gcnt NE 0 THEN BEGIN
lati=WHERE(STRMID(STRUPCASE(ndacc),0,8) EQ 'LATITUDE')
loni=WHERE(STRMID(STRUPCASE(ndacc),0,9) EQ 'LONGITUDE')
res=STRSPLIT(ndacc[lati[0]],' ',/EXTRACT,COUNT=cres)
slat=FLOAT(res[cres-1])
slat=FLOAT(res[cres-1])
res=STRSPLIT(ndacc[loni[0]],' ',/EXTRACT,COUNT=cres)
slon=FLOAT(res[cres-1])
latdiff=FLTARR(scnt)
......@@ -2431,18 +2433,27 @@ IF ((nw EQ 'S') OR (nw EQ 'W') OR (nw EQ 'E')) AND (STRUPCASE(res[0]) EQ 'DATETI
ENDIF
;Check for isolated out-of-order time values and convert to missing values (will be interpolated below)
maxo=5 ;maximum number of out-of-order values allowed
IF mcnt GT 2L THEN BEGIN
dsizhold=dsizx
FOR i=0L,mcnt-2L DO BEGIN
bi=WHERE((dsizx[i] GT dsizx[i+1L:mcnt-1L]) AND (dsizx[i+1L:mcnt-1L] NE fv),bcnt)
IF bcnt NE 0 THEN dsizhold[bi+i+1L]=fv
;This should find instances where timestamps are > 1000 secs different to neighbouring values
;e.g. Hilo SHADOZ 2009 to 2015
nfvi=WHERE(dsizx[i+1L:mcnt-1L] NE fv,nfvcnt)
IF nfvcnt NE 0 THEN BEGIN
bi=WHERE(dsizx[i]-dsizx[nfvi+i+1L] GT 1000.d,bcnt)
IF bcnt NE 0 THEN BEGIN
IF nfvcnt-bcnt LE maxo THEN dsizhold[i]=fv
ENDIF
ENDIF
ENDFOR
ti=WHERE(dsizhold NE fv, ordchk) & ordchk=mcnt-ordchk
IF (ordchk LE CEIL(ndsize*0.01)) AND (ordchk LT 5) THEN BEGIN
;will convert to missing if no more than of 5 out-of-order values or 1% of ndsize
;IF (ordchk LE CEIL(ndsize*0.01)) AND (ordchk LT 5) THEN BEGIN
IF ordchk LE maxo THEN BEGIN
;will convert to missing if no more than of 5 out-of-order values
dsizx=dsizhold & dsize=dsizhold
ENDIF ELSE IF (ordchk NE 0L) AND (nw EQ 'S') AND (dsd GT 2192.D) AND (dsd LT 2221.D) THEN BEGIN
;check for Hanoi January 1996 SHADOZ files
;check for Hanoi January 2006 SHADOZ files
sti=WHERE(STRMID(STRUPCASE(ndacc),0,7) EQ 'STATION')
IF STRPOS(STRUPCASE(ndacc[sti[0]]),'VIETNAM') NE -1 THEN BEGIN
;interpolate all values except for first and last
......@@ -2524,7 +2535,7 @@ IF (nw EQ 'N') AND (STRUPCASE(res[0]) EQ 'DATETIME') AND (N_ELEMENTS(dsizx) GT 2
IF dsizhold[i] NE fv THEN begin
ai=WHERE((dsizhold[i] GT dsizhold[i+1L:ndsize-1L]) AND (dsizhold[i+1L:ndsize-1L] NE fv),acnt)
IF acnt NE 0 THEN BEGIN
IF acnt NE 1 THEN print,dsizhold[i] ELSE print,dsizhold[ai+i+1L]
;IF acnt NE 1 THEN PRINT,dsizhold[i] ELSE PRINT,dsizhold[ai+i+1L]
IF acnt NE 1 THEN dsizhold[i]=fv ELSE dsizhold[ai+i+1L]=fv
ordchk=ordchk+1L
ENDIF
......@@ -2565,12 +2576,6 @@ IF STRPOS(STRUPCASE(res[0]),'WIND.DIRECTION') NE -1 THEN BEGIN
ENDIF
ENDIF
IF (nw EQ 'S') AND (res[0] EQ 'CURRENT.RAW.INSTRUMENT') THEN BEGIN
;check for negative current values and change to fill value
fi=WHERE(dsize LT 0.0,fcnt)
IF fcnt NE 0 THEN dsize[fi]=fv
ENDIF
;Apply conversions or scale factor to the NDACC/WOUDC/SHADOZ/NOAA data
IF STRPOS(res[3],'MJD2000') NE -1 THEN BEGIN
mjd2000=0.d
......@@ -2651,7 +2656,7 @@ ENDIF ELSE BEGIN
ENDELSE
;Check for out-of-range values - if more than 10% outside of VAR_VALID_MIN/MAX then change to fill values
;Otherwise leave unchanged (will then generate error). This is designed to account for undocumented fill
;Otherwise leave unchanged (will then generate error). This is designed to account for incorrect fill
;values being present in the data - especially SHADOZ and NOAA. Do not do for DATETIME type variables
IF (STRPOS(res[0],'DATETIME') EQ -1) AND (mcnt NE 0) THEN BEGIN
minmaxfill=DBLARR(2) ;to hold Min, Max values
......@@ -2660,7 +2665,10 @@ IF (STRPOS(res[0],'DATETIME') EQ -1) AND (mcnt NE 0) THEN BEGIN
(res[0] EQ 'O3.MIXING.RATIO_INSITU') OR (res[0] EQ 'O3.NUMBER.DENSITY_INSITU')
;always convert to fill values for out-of-range values for following
oorchk=(res[0] EQ 'WIND.DIRECTION_INSITU') OR (res[0] EQ 'PRESSURE_INSITU') OR $
(res[0] EQ 'HUMIDITY.RELATIVE_INSITU')
(res[0] EQ 'HUMIDITY.RELATIVE_INSITU') OR (res[0] EQ 'CURRENT.RAW.INSTRUMENT') OR $
(res[0] EQ 'ALTITUDE_GPS') OR (res[0] EQ 'WIND.SPEED_INSITU') OR $
(res[0] EQ 'LONGITUDE') OR (res[0] EQ 'TEMPERATURE_INSITU') OR $
(res[0] EQ 'INTERNAL.BOX.TEMPERATURE_INSITU') OR (res[0] EQ 'LATITUDE')
attchk=['VAR_VALID_MIN','VAR_VALID_MAX']
FOR i=0,1 DO BEGIN
vi=WHERE(STRMID(vat,0,STRLEN(attchk[i])) EQ attchk[i])
......@@ -3191,7 +3199,7 @@ IF metaok EQ '' THEN BEGIN
ENDIF
;Determine location of required dataset (divide into Section Header and Column Name components)
lres=STRSPLIT(res[1],'_',/EXTRACT,COUNT=nlres) & lres=STRUPCASE(STRTRIM(lres,2))
IF (nlres GT 2) AND (nw NE 'S') THEN BEGIN
IF (nlres GT 2) AND (nw NE 'S') THEN BEGIN
;need to append all but the last sub-value to the first value
FOR m=1,nlres-2 DO lres[0]=lres[0]+'_'+lres[m]
lres=[lres[0],lres[nlres-1]] & nlres=2
......@@ -3218,9 +3226,9 @@ IF metaok EQ '' THEN BEGIN
ltlni=WHERE(STRMID(STRUPCASE(ndacc),0,STRLEN(vres[0])) EQ vres[0])
vres=STRSPLIT(ndacc[ltlni[0]],' ',/EXTRACT,COUNT=cres)
sltln=FLOAT(vres[cres-1])
ENDELSE
ENDELSE
ENDIF
IF ((nw EQ 'S') OR (nw EQ 'E')) AND (lres[0] EQ '#PROFILE') THEN scnt=1 $
ELSE si=WHERE(STRMID(STRUPCASE(ndacc),0,STRLEN(lres[0])) EQ lres[0],scnt)
IF (scnt NE 0) OR ((lc EQ sccnt) AND (sccnt GT 1)) OR (aplabel[0] NE '-1') THEN BEGIN
......@@ -3272,7 +3280,7 @@ IF metaok EQ '' THEN BEGIN
IF STRUPCASE(STRMID(aplabel[lc,0],0,29)) EQ 'CORRECTION FACTOR (COL2/COL1)' THEN $
;special case for correction factor - change to -9 to indicate no correction applied
aplabel[lc,1]='-9.0000'
ENDIF
ENDIF
ENDIF
ENDELSE
IF N_ELEMENTS(resy) EQ 1 THEN BEGIN ;No accompanying info to the header so add values
......@@ -3352,13 +3360,13 @@ IF metaok EQ '' THEN BEGIN
ENDFOR
ENDIF
ENDIF
;Add station latitude and longitude if no GPS values are available for SHADOZ
IF ((res[0] EQ 'LATITUDE') OR (res[0] EQ 'LONGITUDE')) AND (nw EQ 'S') THEN BEGIN
IF (FLOAT(holddata[0,j]) EQ FLOAT(mv)) AND (sltln NE -999.) THEN $
holddata[0,j]=STRTRIM(sltln,2)
ENDIF
;Do check for missing DATETIME values - if so add comment to VAR_NOTES
IF res[0] EQ 'DATETIME' THEN BEGIN
IF nw EQ 'S' THEN fi=WHERE(DOUBLE(holddata[*,j]) EQ DOUBLE(mv),fcnt) $
......@@ -3761,7 +3769,7 @@ FUNCTION cds_tools, listf, tav, outdir, dataformat, DATETIME=o1
;Initialize statusflag, and exit_code
statusflag='' & dum='' & exit_code=3
cdver='cds_tools version 2.0b62, 2019-11-12'
cdver='cds_tools version 2.0b63, 2020-03-11'
;Check input parameters
IF N_PARAMS() LT 3 THEN $
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment