idlcr8hdf.pro 307 KB
Newer Older
Ian Boyd's avatar
Ian Boyd committed
1
;Main Program Version: idlcr8hdf.pro v4.0b54, 20201103
Ian Boyd's avatar
Ian Boyd committed
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
;  Written by Ian Boyd for the AVDC - iboyd@astro.umass.edu
;
;Sub-versions (refer to idlcr8hdf-v4.0_Readme.pdf for full history)

PRO idlcr8hdf_common
;Procedure to define the COMMON blocks for this program
; ----------
;Written by Ian Boyd for the AVDC - iboyd@astro.umass.edu
;
;History:
;  20050909: Introduced to IDLCR8HDF - Version 1.1
;  20061012: Variable 'ncsa' added to METADATA; 'iarr', 'larr', 'rarr', 'darr' holding Arrays
;            removed from DATA and replaced with Structure 'ds'; Variable 'vfv' added to DATA;
;            WIDGET_WIN added for common variables associated with the Graphical User Interface
;            - Version 2.0
;  20080302: tab_type integer added to TABLEDATA - Version 3.0
;  20100205: rerr string added to WIDGET_WIN - Version 3.09
;  20110401: vnchange added to DATA; mv_lng and mv_dbl added to METADATA; vserror added to DATA;
;            vfv removed from DATA; ncsa removed from METADATA - Version 4.0b1
;  20150127: mv_str added to METADATA to hold maximum string length of string dataset entries -
;            Version 4.0b25
;  20151012: Rename free_attr to attr_free and add qa_yes - Version 4.0b31
;
;Input: Nil
;
;Output: Nil
;
;Called by: N/A
;
;Subroutines Called: None

COMMON TABLEDATA, tab_arr,tab_ver,tab_type
COMMON METADATA, meta_arr,attr_arr_glob,attr_arr_data,attr_free,mv_lng,mv_dbl,mv_str
COMMON DATA, ds,ndm,nvn,vn,vu,vnchange,vserror,qa_yes
Ian Boyd's avatar
Ian Boyd committed
36
COMMON WIDGET_WIN, wtxt,b1,lineno,base,o3,dux,rerr,lvals
Ian Boyd's avatar
Ian Boyd committed
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68

END ;Procedure idlcr8hdf_common



PRO intro_event, ev
;Procedure to define how Events from the Start-up Introduction Window are handled
; ----------
;Written by Ian Boyd for the AVDC - iboyd@astro.umass.edu
;
;History:
;  20061012: Introduced to IDLCR8HDF - Version 2.0
;
;Input: ev - Selected widget event structure
;
;Output: o3 - Common string array defining the various options for program output
;
;Called by: XMANAGER in INTRO
;
;Subroutines Called: None

COMMON WIDGET_WIN

;The uservalue is retrieved from a widget when an event occurs
WIDGET_CONTROL,ev.id,GET_UVALUE=uv
;Assign/Remove AVK button event to/from a variable name
IF uv EQ 'AVK' THEN IF o3[1] EQ uv THEN o3[1]='0' ELSE o3[1]=uv $
;Assign/Remove Log Output button event to/from a variable name
ELSE IF uv EQ 'idlcr8hdf.log' THEN IF o3[2] EQ uv THEN o3[2]='0' ELSE o3[2]=uv $
;Assign/Remove Pop-up window for Log Output button event to/from a variable name
ELSE IF uv EQ 'Pop' THEN IF o3[3] EQ uv THEN o3[3]='0' ELSE o3[3]=uv $
;Assign button event to a variable name
Ian Boyd's avatar
Ian Boyd committed
69
70
71
72
73
74
75
ELSE IF (uv EQ 'H4') OR (uv EQ 'H5') OR (uv EQ 'NC') THEN BEGIN
  o3[0]=uv
  IF uv EQ 'H5' THEN BEGIN
    WIDGET_CONTROL,ev.id+1,Sensitive=1
    WIDGET_CONTROL,ev.id+2,Sensitive=1,Set_Combobox_Select=0
  ENDIF
ENDIF ELSE IF uv EQ '0' THEN IF ev.str EQ 'None' THEN o3[0]='H5' ELSE o3[0]='H5_'+STRTRIM(ev.str,2) $
Ian Boyd's avatar
Ian Boyd committed
76
ELSE o3[0]='0' ;Cancel button chosen
Ian Boyd's avatar
Ian Boyd committed
77
IF (uv NE 'AVK') AND (uv NE 'idlcr8hdf.log') AND (uv NE 'Pop') AND (uv NE 'H5') THEN WIDGET_CONTROL,ev.top,/DESTROY
Ian Boyd's avatar
Ian Boyd committed
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112

END ;Intro_Event



PRO intro, intype
;Procedure which creates an Introduction Window at start-up when IDLCR8HDF is called without parameters,
;or invalid parameters, or is called by IDL Virtual Machine. The user has a choice of continuing with
;the program after selecting input options, or stopping the program.
; ----------
;Written by Ian Boyd for the AVDC - iboyd@astro.umass.edu
;
;History:
;  20061012: Introduced to IDLCR8HDF - Version 2.0
;  20080302: Swapped the order of the option windows, so that the options indicated by the check
;            boxes appear above the option boxes that close the Introduction Window and continue
;            with the program - Version 3.0
;  20100205: Include text regarding the RETERR argument which can be accepted as an input
;            parameter when using the full version of idlcr8hdf - Version 3.09
;  20110401: Add vertxt string array to hold text which changes between versions of the code;
;            Change text to reference GEOMS compliant files instead of AVDC/EVDC/NDACC; Change
;            text regarding the format of the structure required for input - Version 4.0b1
;  20111220: Change text and options to include netCDF; change contact details - Version 4.0b7
;  20130114: Make insensitive /LOG and /NC options if the program is called in IDL DEMO mode
;            -Version 4.0b14
;
;Input: intype - Integer set to -1 or -2: -1 indicates normal state; -2 indicates that input
;                parameters at the IDLCR8HDF call were invalid.
;
;Output: Nil
;
;Called by: IDLCR8HDF
;
;Subroutines Called: INTRO_EVENT (via XMANAGER)

Ian Boyd's avatar
Ian Boyd committed
113
114
115
COMMON WIDGET_WIN

nhdr=44 & errtxt=STRARR(nhdr)
Ian Boyd's avatar
Ian Boyd committed
116
vertxt=['idlcr8hdf-v4.0_Readme.pdf','v4.0b54 March 2020']
Ian Boyd's avatar
Ian Boyd committed
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
errtxt[1]='Welcome to IDLcr8HDF.  This program creates GEOMS compliant HDF4, HDF5 and netCDF files'
errtxt[2]='(also refer to '+vertxt[0]+').'
errtxt[4]='Inputs to the program (IDL Virtual Machine (VM) and IDL Licensed (LIC) Versions):'
errtxt[5]='  METADATA TEMPLATE FILE - Containing the Global and Variable Attribute information for the site.'
errtxt[6]='  DATA FILE(s) - Multiple selection permitted. The file(s) must show the datasets in the single column'
errtxt[7]='    format described in the documentation.'
errtxt[8]='  TAV FILE - the current non-encrypted Table Attribute Values file.'
errtxt[9]='  OUTPUT DIRECTORY - Directory for any HDF, netCDF and log files created. Shortcut values of ''M'' or ''D'''
errtxt[10]='   will output files to the Metadata or Data file directories respectively.'
errtxt[12]='Alternative Input Option for IDL LIC Versions:'
errtxt[13]='  GLOBAL ATTRIBUTES ARRAY (GA) - a string array containing the Global Attributes in the form'
errtxt[14]='    ''label=value''.'
errtxt[15]='  DATA STRUCTURE (DS) - a heap structure using pointers containing the Variable Attribute Labels'
errtxt[16]='    (DS.VA_L), the Variable Attribute Values (DS.VA_V), and the Data (DS.Data), for a single output file.'
errtxt[17]='  TAV FILE - the current non-encrypted Table Attribute Values file.'
errtxt[18]='  OUTPUT DIRECTORY - Directory for any HDF, netCDF and log files created.'
errtxt[19]='  RETERR - String variable to which any error(s) are written.'
errtxt[21]='For IDL VM, input is by ''DIALOG_BOXES''. For IDL LIC, input can be by ''DIALOG_BOXES'' or passed'
errtxt[22]='by calling the program with one of the following command line options:'
errtxt[23]='  1. idlcr8hdf  (Opens this box, and allows the user the option to continue with file inputs).'
Ian Boyd's avatar
Ian Boyd committed
137
138
139
140
errtxt[24]='  2. idlcr8hdf,METAFILE,DATAFILE(s),TAVFILE,OUTDIR[,RETERR][,/H5][,/Cn][,/NC][,/AVK][,/Log][,/Popup]  or'
errtxt[25]='    idlcr8hdf,'''','''','''',''''[,/H5][,/Cn][,/NC][,/AVK][,/Log][,/Popup]  (For null string, DIALOG_BOXES will prompt for input).'
errtxt[26]='  3. idlcr8hdf,GA,DS,TAVFILE,OUTDIR[,RETERR][,/H5][,/Cn][,/NC][,/AVK][,/Log][,/Popup] or
errtxt[27]='    idlcr8hdf,GA,DS[,/H5][,/Cn][,/NC][,/AVK][,/Log][,/Popup]  (Inputs are from session memory, DIALOG_BOX(s) will'
Ian Boyd's avatar
Ian Boyd committed
141
errtxt[28]='    prompt for input if TAVFILE or OUTDIR are not included).'
Ian Boyd's avatar
Ian Boyd committed
142
143
144
errtxt[29]='    /H5, /Cn, /NC, /AVK, /Log and /Popup keywords are used in place of the options given below (HDF4 is default).'
errtxt[30]='The /Cn keyword enables compression and shuffling of HDF5 files only (C1=low, C9=high). Default is no compression.'
errtxt[32]='Contacts -'
Ian Boyd's avatar
Ian Boyd committed
145
146
147
errtxt[33]='  Ian Boyd, BC Scientific Consulting LLC (iboyd@astro.umass.edu)'
errtxt[34]='  26 Campbells Rd'
errtxt[35]='  Pine Hill, Dunedin 9010, New Zealand'
Ian Boyd's avatar
Ian Boyd committed
148
149
150
151
152
errtxt[37]='  Ann Mari Fjaeraa, EVDC Project Manager (amf@nilu.no)'
errtxt[38]='  Norwegian Institute for Air Research, Instituttveien 18'
errtxt[39]='  Postbox 100, N-2027 KJELLER, NORWAY'
errtxt[41]='EVDC Website: Tools and documentation available from http://evdc.esa.int/'
errtxt[43]='To continue, please choose from the options below (Note: HDF5 only available on IDL6.2 or greater).'
Ian Boyd's avatar
Ian Boyd committed
153
154
155
156
157
158
errtxt='      '+errtxt

;Set-up text display widget
IF intype EQ -2 THEN xtxt=' - Command Line Input Error' ELSE xtxt=''
IF intype EQ -3 THEN optsens=0 ELSE optsens=1
base=WIDGET_BASE(Title='idlcr8hdf '+vertxt[1]+xtxt,Tlb_Frame_Attr=1,/Column) ;,Tab_Mode=1)
Ian Boyd's avatar
Ian Boyd committed
159
wtxt=WIDGET_TEXT(base,xsize=102,ysize=25,/Scroll)
Ian Boyd's avatar
Ian Boyd committed
160
161
162
163
164
165
166
167
168
169
170
171
172
173
base3=WIDGET_BASE(base,/Nonexclusive)
logtext='Append log output to the file ''idlcr8hdf.log'' '
IF intype EQ -3 THEN logtext=logtext+'(No log or netCDF file output permitted in IDL DEMO Mode)' $
ELSE logtext=logtext+'(will create the file IF it doesn''t exist)'
poptext='Open a Pop-Up Window to display log output'
avktext='If Avg. Kernel data are present, append sentence to VAR_NOTES giving first three values of '
avktext=avktext+'the first kernel'
avktip='Sentence reads ''First three values of the first kernel are: x.xx x.xx x.xx'''
b4=WIDGET_BUTTON(base3,value=logtext,uvalue='idlcr8hdf.log',frame=3,SENSITIVE=optsens)
b5=WIDGET_BUTTON(base3,value=poptext,uvalue='Pop',frame=3)
b6=WIDGET_BUTTON(base3,value=avktext,uvalue='AVK',frame=3) ;,Tooltip=AVKTip)
base2=WIDGET_BASE(base,/Row)
tip='Left Mouse Click or Tab to entry and hit <Spacebar>'
b1=WIDGET_BUTTON(base2,value='HDF4',uvalue='H4',frame=3) ;,Tooltip=Tip)
Ian Boyd's avatar
Ian Boyd committed
174
b7=WIDGET_BUTTON(base2,value='netCDF3',uvalue='NC',frame=3,SENSITIVE=optsens)
Ian Boyd's avatar
Ian Boyd committed
175
176
177
IF FLOAT(!Version.Release) GE 6.2 THEN $
  b2=WIDGET_BUTTON(base2,value='HDF5',uvalue='H5',frame=3) $;,Tooltip=Tip) $
ELSE b2=WIDGET_BUTTON(base2,value='HDF5',Sensitive=0,frame=3)
Ian Boyd's avatar
Ian Boyd committed
178
179
180
b8=Widget_Label(base2,Value='  Compression ',Sensitive=0)
b9=Widget_Combobox(base2,Value=lvals,Sensitive=0,uvalue='0')

Ian Boyd's avatar
Ian Boyd committed
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
b3=WIDGET_BUTTON(base2,value='Stop',uvalue='CANCEL',frame=3) ;,ToolTip=Tip)
WIDGET_CONTROL,base,/Realize
WIDGET_CONTROL,b4,/Input_Focus
FOR i=0,N_ELEMENTS(errtxt)-1 DO $
  WIDGET_CONTROL,wtxt,set_value=errtxt[i],/Append
XMANAGER,'intro',base

END ;Intro



PRO idlcr8hdf_event, ev
;Procedure to close the pop-up logging window after user selects 'Finish'.
; ----------
;Written by Ian Boyd for the AVDC - iboyd@astro.umass.edu
;
;History:
;  20061012: Introduced to IDLCR8HDF - Version 2.0
;
;Input: Selected widget event structure
;
;Output: Nil
;
;Called by: XMANAGER in IDLCR8HDF and STOP_WITH_ERROR
;
;Subroutines Called: None

WIDGET_CONTROL,ev.top,/DESTROY
RETALL & HEAP_GC

END ;IDLcr8HDF_Event



Ian Boyd's avatar
Ian Boyd committed
215
FUNCTION is_a_number_hdf, value
Ian Boyd's avatar
Ian Boyd committed
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
  ON_IOERROR, ConversionError
  IF STRTRIM(value,2) EQ '' THEN RETURN, 0B
  n=DOUBLE(value)
  RETURN, 1B
  ConversionError:
  RETURN, 0B
END



PRO stop_with_error, txt1, txt2, lu
;Procedure called when an error in the program inputs is detected. An error message is displayed
;and the program stopped and reset. If necessary, open files are closed, and memory associated
;with a structure is freed. The error message is displayed in one or more of the following:
;a Pop-up dialog window; the Pop-up logging window; the Output Logging window in the IDLDE;
;a log file (idlcr8hdf.log)
; ----------
;Written by Ian Boyd for the AVDC - iboyd@astro.umass.edu
;
;History:
;  20050802: Original IDLCR8HDF Routine - Version 1.0
;  20061012: Set-up so that the error output is displayed in the output window(s) dependent on
;            the options chosen by the user, the point in the program that the error is detected,
;            and the method that IDLCR8HDF is called. If txt1 is preceeded by 'D_' or is null,
;            the error output is to a Pop-up Dialog window. Other error output options are
;            determined by the values in the (Common) dux array - Version 2.0
;  20100205: Allow routine to return to the calling routine, rather than stop the application, if
;            a 'reterr' argument is included in the call to idlcr8hdf - Version 3.09
;  20110401: Change end text dependent on whether the program is creating a GEOMS file or doing QA
;            on a GEOMS file - Version 4.0b1
;  20111220: Change text referring to HDF to GEOMS - Version 4.0b7
;  20171121: Add comment advising that program stopped before all checks were completed if doing
;            QA; Add COMMON DATA to routine and remove ds from the list of input parameters
;            - Version 4.0b43
;
;Inputs: txt1 - the initial text line of the error message. Generally contains the name of the routine
;               where the error was detected.
;        txt2 - the second text line which generally describes the error state.
;        lu - Where applicable, the file unit that needs to be closed at the termination of the program,
;             otherwise set to -1.
;        ds - Where applicable, the name of the data structure set-up by the program (dependent on
;             the point in the program that the error is detected), so that memory associated with the
;             structure can be freed (COMMON variable).
;        dux - an integer array used to determine the output options for the error message (COMMON) variable).
;        qa_yes - a boolean to indicate whether idlcr8hdf called in QA mode or not (COMMON variable).
;
;Output: Error message displayed/output dependent on the requested output options.
;
;Called by: The routine in which the error was detected. The following routines call STOP_WITH_ERROR:
;           READ_TABLEFILE; TEST_FILE_INPUT; READ_METADATA; EXTRACT_AND_TEST; CHECK_METADATA;
;           SET_UP_STRUCTURE; CHECK_MIN_MAX_FILL; EXTRACT_DATA; FIND_HDF_FILENAME; IDLCR8HDF.
;
;Subroutines Called: IDLCR8HDF_EVENT (via XMANAGER)

COMMON DATA
COMMON WIDGET_WIN

IF lu NE -1L THEN FREE_LUN,lu

IF txt1 EQ '' THEN BEGIN ;<cancel> chosen on Intro box
  res=DIALOG_MESSAGE('IDLcr8HDF Stopped!',/Information,Title='AVDC IDLcr8HDF')
ENDIF ELSE BEGIN
  ;If necessary free up memory by destroying the heap variables pointed at by its pointer arguments
  IF N_ELEMENTS(ds) NE 0 THEN PTR_FREE,ds.data
  ;Write error to file and/or IDL output window
  IF STRMID(txt1,0,2) EQ 'D_' THEN txtx=STRMID(txt1,2) ELSE txtx=txt1
  FOR i=dux[0],dux[1],dux[2] DO BEGIN
    IF i EQ -1 THEN BEGIN
      PRINT,'  ERROR in '+txtx & PRINT,'    '+txt2
      PRINT,'' & PRINT,'IDLcr8HDF stopped - Program Ended on '+SYSTIME(0)
    ENDIF ELSE BEGIN
      PRINTF,i,'  ERROR in '+txtx & PRINTF,i,'    '+txt2
      IF (i EQ dux[0]) OR ((i EQ dux[1]) AND (STRPOS(o3[2],'idlcr8hdf.log') NE -1)) THEN BEGIN
        PRINTF,i,'' & PRINTF,i,'IDLcr8HDF stopped - Program Ended on '+SYSTIME(0)
      ENDIF ELSE IF qa_yes THEN BEGIN
        PRINTF,i,'' & PRINTF,i,'  INFORMATION: QA stopped before all checks could be completed'
      ENDIF
    ENDELSE
  ENDFOR
  IF dux[1] NE dux[0] THEN FREE_LUN,dux[1]
  IF (STRMID(txt1,0,2) EQ 'D_') AND (rerr[0] EQ 'NA') THEN BEGIN
    ;write error to DIALOG Box instead of Popup window
    errtxt2=STRARR(4)
    errtxt2[0]=STRMID(txt1,2) & errtxt2[1]=txt2 & errtxt2[3]='IDLcr8HDF Stopped!'
    res=DIALOG_MESSAGE(errtxt2,/Error,Title='AVDC IDLcr8HDF Error')
  ENDIF ELSE IF rerr[0] EQ 'NA' THEN BEGIN ;write error to Popup window
    IF o3[4] EQ '0' THEN endtxt='GEOMS file creation ' $
    ELSE endtxt='GEOMS file testing '
    lineno=lineno+4L
    WIDGET_CONTROL,wtxt,set_value='  ERROR in '+txt1,/Append
    WIDGET_CONTROL,wtxt,set_value='    '+txt2,/Append
    WIDGET_CONTROL,wtxt,set_value='',/Append,Set_text_top_line=lineno
    WIDGET_CONTROL,wtxt,set_value=endtxt+'stopped - hit <Finish> to close program',/Append
    WIDGET_CONTROL,b1,Sensitive=1,/Input_Focus
    XMANAGER,'stop_with_error',base,Event_Handler='idlcr8hdf_event'
  ENDIF
ENDELSE
HEAP_GC
IF rerr[0] EQ 'NA' THEN RETALL $
ELSE BEGIN
  IF o3[4] EQ '0' THEN endtxt='create GEOMS file' $
  ELSE endtxt='complete GEOMS file test'
  rerr[0]='Unable to '+endtxt+' - '+txtx+txt2
ENDELSE

END ;Procedure Stop_With_Error



PRO infotxt_output, infotxt
;Procedure called when the program makes a change to the input meta data or reports information
;relevant to the creation of the HDF/netCDF file. This information can be reported to a Pop-up logging
;window, IDLDE output log window, and/or the log file. Code for this output was originally
;written directly in the affected procedures
; ----------
;Written by Ian Boyd for the AVDC - iboyd@astro.umass.edu
;
;History:
;  20090311: Introduced to IDLCR8HDF - Version 3.06
;  20110401: infotxt INFORMATION header changed to a number, to allow changes to the comment
;            header depending on how idlcr8hdf has been called, as follows: 0 = INFORMATION;
;            1 = NON-STANDARD COMPLIANCE NOTIFICATION (QA) o/w INFORMATION; 2 = ERROR (QA)
;            o/w INFORMATION; 3 = ERROR; 4 = DEBUG - Version 4.01
;
;Inputs: infotxt - the text line(s) of the information message. Can be either a scalar string
;                  or string array
;
;Output: Message displayed/output dependent on the requested output options
;
;Called by: The routine in which the reported change was made. The following routines call
;           INFOTXT_OUTPUT: READ_TABLEFILE; GEOMS_RULE_CHANGES; PRE_DEFINED_ATT_CHECKS;
;           READ_METADATA; EXTRACT_AND_TEST; CHECK_METADATA; EXTRACT_DATA; SET_UP_STRUCTURE;
;           CHECK_STRING_DATATYPE; CHECK_MIN_MAX_FILL; READ_DATA; FIND_HDF_FILENAME;
;           AVDC_HDF_WRITE; IDLCR8HDF
;
;Subroutines Called: None

COMMON DATA
COMMON WIDGET_WIN

dm=SIZE(infotxt,/N_ELEMENTS)
qaval=FIX(STRMID(infotxt[0],0,1))
write_rerr=0 ;Boolean to generate a return error message

IF qa_yes THEN BEGIN ;program called in QA mode
  ;Add correct message title
  CASE 1 OF
    qaval EQ 0: infotxt[0]='  INFORMATION:'+STRMID(infotxt[0],1)
    qaval EQ 1: infotxt[0]='  NON-STANDARD COMPLIANCE NOTIFICATION:'+STRMID(infotxt[0],1)
    qaval EQ 4: infotxt[0]='  DEBUG:'+STRMID(infotxt[0],1)
    ELSE: infotxt[0]='  ERROR:'+STRMID(infotxt[0],1)
  ENDCASE
  ;truncate message from the '|'
  bs_found=0
  FOR n=0,dm-1 DO BEGIN
    IF bs_found EQ 1 THEN infotxt[n]='' $
    ELSE BEGIN
      bspos=STRPOS(infotxt[n],'|')
      IF bspos NE -1 THEN BEGIN
        infotxt[n]=STRMID(infotxt[n],0,bspos)
        bs_found=1
      ENDIF
    ENDELSE
  ENDFOR
  ;recalculate number of good infotxt values
  gi=WHERE(infotxt NE '',dm)
ENDIF ELSE BEGIN ;program called in HDF file create mode
  CASE 1 OF
    qaval EQ 3: BEGIN
                  infotxt[0]='  ERROR:'+STRMID(infotxt[0],1)
                  IF rerr[1] NE 'NA' THEN BEGIN ;generate return error message
                    IF rerr[1] EQ '' THEN BEGIN
                      IF o3[4] EQ '0' THEN endtxt='create GEOMS file' $
                      ELSE endtxt='complete GEOMS file test'
                    ENDIF
                    write_rerr=1
                  ENDIF
                  o3[4]='NOHDF' ;Error in Input so do not create HDF file
                END
    qaval EQ 4: infotxt[0]='  DEBUG:'+STRMID(infotxt[0],1)
    ELSE: infotxt[0]='  INFORMATION:'+STRMID(infotxt[0],1)
  ENDCASE
  ;remove '|' from the message
  FOR n=0,dm-1 DO BEGIN
    bspos=STRPOS(infotxt[n],'|')
    IF bspos NE -1 THEN BEGIN
      infotxt[n]=STRMID(infotxt[n],0,bspos)+STRMID(infotxt[n],bspos+1)
    ENDIF
  ENDFOR
ENDELSE

lineno=lineno+dm
FOR n=0,dm-1 DO BEGIN
  IF o3[3] EQ '' THEN WIDGET_CONTROL,wtxt,set_value=infotxt[n],/Append,Set_text_top_line=lineno
  FOR m=dux[0],dux[1],dux[2] DO BEGIN
    IF m EQ -1 THEN PRINT,infotxt[n] ELSE PRINTF,m,infotxt[n] ;write out to log
  ENDFOR
ENDFOR

IF write_rerr EQ 1 THEN BEGIN
  IF rerr[1] EQ '' THEN rerr[1]='Unable to '+endtxt+' -'+STRMID(infotxt[0],8) $
  ELSE rerr[1]=rerr[1]+';'+STRMID(infotxt[0],8)
  IF dm GT 1 THEN FOR n=1,dm-1 DO rerr[1]=rerr[1]+STRMID(infotxt[n],3)
ENDIF

END ;Procedure InfoTxt_Output



FUNCTION jdf_2_datetime, jdf, MJD2000=mjd2000, SHORTISO8601=iso, LONGISO8601=isoms
;Computes the UT date/time from JDF/MJD2000.
; ----------
;   Bojan R. Bojkov
;   bojan.r.bojkov@nasa.gov
;   03/04/2004
;   03/11/2004 bug fix
;   05/27/2005 add ShortISO8601 and LongISO8601 switches,
;              and change seconds value to decimal seconds (Ian Boyd)
;
; References ----------
;   Explan. Supp. Astron. Almanac, p.604 (1992); through E. Celarier
;
; Caveats ----------
;   Valid for all YYYY >= -4712 (i.e. for all JD .ge. 0)
;   The true Gregorian calendar was only adopted on 15 October 1582,
;   so any dates before this are "virtual" Gregorian dates.
;
; Input ----------
;   jdf     : double precision value
;   mjd2000 : flag if input is MJD2000
;   iso     : flag if output is in ISO8601 (YYYYMMDDThhmmssZ)
;   isoms   : flag if output is in ISO8601ms (YYYYMMDDThhmmss.sssZ)
;
; Output ---------
;   floating point array [YYYY, MM, DD, hh, mn, ss.sss],
;   or ISO8601 string format
; External subroutines ---------
;   NONE

jdf=DOUBLE(jdf)

j0=2451544.5D
IF KEYWORD_SET(mjd2000) THEN jdhold=jdf+j0 ELSE jdhold=jdf
jdi=LONG(jdhold)
df=jdhold-jdi

;Determine hh, mm, ss, ms
hh=(df+0.5)*24.D
q=WHERE(hh GE 24.D)
IF q[0] NE -1 THEN BEGIN
  hh[q]=hh[q]-24.D
  jdi[q]=jdi[q]+1
ENDIF
t1=hh
hh=LONG(t1)
t2=(t1-hh)*60.D
mn=LONG(t2)
t3=(t2-mn)*60.D
ss=t3
;ss=LONG(t3)
;ms=LONG((t3-ss)*1.d3)

;Determine YYYY, MM, DD
t1=jdi+68569L
t2=(4*t1)/146097L
t1=t1-(146097L*t2+3L)/4L
t3=(4000L*(t1+1L))/1461001L
t1=t1-(1461L*t3)/4L + 31L
t4=(80L*t1)/2447L

dd=t1-(2447L*t4)/80L
t1=t4/11L
mm=t4+2L-12L*t1
yyyy=100L*(t2-49L)+t3+t1

dt=TRANSPOSE([[yyyy],[mm],[dd],[hh],[mn],[ss]])
IF (KEYWORD_SET(iso)) OR (KEYWORD_SET(isoms)) THEN BEGIN
  dts=STRARR(6)
  IF (KEYWORD_SET(iso)) AND (ss-FIX(ss) GE 0.5) THEN BEGIN
    ;recalculate to get correct seconds value
    jdhold=jdf+0.000008D ;add ~0.7 secs
    IF KEYWORD_SET(mjd2000) THEN jdhold=jdhold+j0
    CALDAT,jdhold,mm,dd,yyyy,hh,mn,ss
    dt=TRANSPOSE([[yyyy],[mm],[dd],[hh],[mn],[ss]])
  ENDIF
  dt=LONG(dt)
  FOR i=1,5 DO $
    IF dt[i] LT 10L THEN dts[i]='0'+STRTRIM(dt[i],2) ELSE dts[i]=STRTRIM(dt[i],2)
  IF KEYWORD_SET(isoms) THEN BEGIN
    ssd=STRING(format='(f6.3)',ss)
    IF FLOAT(ssd) LT 10.0 THEN dts[5]='0'+STRTRIM(ssd,2) ELSE dts[5]=STRTRIM(ssd,2)
  ENDIF
  RETURN,STRTRIM(dt[0],2)+dts[1]+dts[2]+'T'+dts[3]+dts[4]+dts[5]+'Z'
ENDIF ELSE RETURN, dt

END ;Function jdf_2_datetime



FUNCTION julian_date, date_ut, ISO8601=iso, MJD2000=mjd2000
;Computes the Julian date (jd) or date in MJD2000 format in double precision.
; ----------
;   Bojan R. Bojkov
;   bojan.r.bojkov@nasa.gov
;   06/03/2001
;   09/24/2001: Code cleanup                        - Function Version 1.0
;   10/16/2001: Mode corrections                    - Function Version 1.01
;   03/07/2002: Comment cleanup and variable change - Function Version 2.0
;   10/08/2002: Added jdf option                    - Function Version 2.1
;   07/21/2003: Converted to a function             - Function Version 3.0
;   05/25/2005: Added ISO8601 and MJD2000 keywords.
;               Input can either be a string or a numeric array
;               (see below for input details). Returns -99999.d
;               if the Input is invalid - Ian Boyd
;
; References ----------
;   Hughes, D.W., Yallop, B.D. and Hohenkerk, C.Y., "The Equation of Time",
;     Mon. Not. R. astr. Soc., 238, pp 1529-1535. 1989.
;
;   Results verified using:
;     Meeus, J, "Astronomical Algorithms", William-Bell, Inc., Richmond VA,
;     p62, 1991.
;
; Caveats ----------
;   NONE
;
; Input ----------
;   date_ut: Numeric array (UT: YYYY, MM, DD, [hh, mm, ss]), or if ISO8601
;            keyword set, a string containing datetime as YYYYMMDDThhmmssZ
;
; Output ---------
;   jd:      julian day, or if MJD2000 keyword set,
;            a double precision day in MJD2000 format
;
; External subroutines ---------
;   NONE

ON_IOERROR,TypeConversionError
valid=0 ;Type conversion check
inputerror=0 ;Check that input is as expected
;Check input format
IF KEYWORD_SET(iso) THEN BEGIN ;input is YYYYMMDDThhmmssZ
  IF STRLEN(date_ut) EQ 16 THEN BEGIN
    ;test for numeric values (will return type conversion error value if not a number)
    FOR i=0,7 DO check=FIX(STRMID(date_ut,i,1))
    FOR i=9,14 DO check=FIX(STRMID(date_ut,i,1))
    dt=INTARR(6)
    dt[0]=FIX(STRMID(date_ut,0,4)) & dt[1]=FIX(STRMID(date_ut,4,2))
    dt[2]=FIX(STRMID(date_ut,6,2)) & dt[3]=FIX(STRMID(date_ut,9,2))
    dt[4]=FIX(STRMID(date_ut,11,2)) & dt[5]=FIX(STRMID(date_ut,13,2))
    date_ut=dt & achk=[0,6] ;to add hhmmss component
  ENDIF ELSE inputerror=1
ENDIF ELSE BEGIN
  ;check that input contains at least YMD (and at most YMDhms) information and is either
  ;an integer, long, float or double array
  achk=SIZE(date_ut)
  IF (achk[1] LT 3) OR (achk[1] GT 6) OR (achk[2] LT 2) OR (achk[2] GT 5) THEN inputerror=1
ENDELSE

IF inputerror EQ 0 THEN BEGIN ;can perform JD calculation
  IF date_ut[1] GT 2 THEN BEGIN
    y=DOUBLE(date_ut[0])
    m=DOUBLE(date_ut[1]-3)
    d=DOUBLE(date_ut[2])
  ENDIF ELSE BEGIN
    y=DOUBLE(date_ut[0]-1)
    m=DOUBLE(date_ut[1]+9)
    d=DOUBLE(date_ut[2])
  ENDELSE

  ;Compute Julian date:
  j=LONG(365.25D0*(y+4712.D0))+LONG(30.6D0*m+0.5D0)+59.D0+d-0.5D0

  ;Check for Julian or Gregorian calendar:
  IF j LT 2299159.5D0 THEN jd=j $ ;If Julian calendar.
  ELSE BEGIN ;If Gregorian calendar.
    gn=38.D0-LONG(3.D0*LONG(49.D0+y/100.D0)/4.D0)
    jd=j+gn
  ENDELSE

  ;add hhmmss values if present
  CASE 1 OF
    achk[1] EQ 4: jd=jd+DOUBLE(date_ut[3])/24.D ;hh only
    achk[1] EQ 5: jd=jd+(DOUBLE(date_ut[3])*3600.D + DOUBLE(date_ut[4])*60.D)/86400.D ;hhmm only
    achk[1] EQ 6: jd=jd+(DOUBLE(date_ut[3])*3600.D + DOUBLE(date_ut[4])*60.D + $
                         DOUBLE(date_ut[5]))/86400.D ;hhmmss
    ELSE:
  ENDCASE

  valid=1 ;Type conversion OK

  ;Set to MJD2000 if required
  IF KEYWORD_SET(mjd2000) THEN jd=jd-2451544.5D
ENDIF

TypeConversionError:
IF valid EQ 0 THEN RETURN,-99999.D ELSE RETURN, jd

END ;Function Julian_Date



PRO var_units_test, vuvalue, rd, tab_type, var_si_conv, errstate
;Procedure to perform checks on the VAR_UNITS value in the metadata and, based on the VAR_UNITS input,
;calculate and return the VAR_SI_CONVERSION value.
; ----------
;Written by Ian Boyd for the AVDC - iboyd@astro.umass.edu
;
;  History:
;    20061012: Introduced to IDLCR8HDF - Version 2.0
;              (Previously these checks were performed in the Extract_and_Test routine)
;    20080302: var_unit_arr and unit_pre_arr added which hold, respectively, the valid VAR_UNITS and
;              corresponding VAR_SI_CONVERSION values, and the set of UNIT_PREFIXs (previously
;              values from the AVDC TAV file have been used). This has been done so that the
;              routine can be stand-alone (i.e. not dependent on also having to read in a TAV file),
;              and also because the original Envisat table.dat file does not contain the
;              VAR_SI_CONVERSION values. The input parameters have been changed to reflect this: bu
;              and up arrays (previously containing the VAR_UNIT and UNIT_PREFIX values from the TAV
;              file), and nbu and nup (the number of elements in the bu and up arrays) are no longer
;              used. The integer flag tab_type has been added to account for the different handling of
;              some of the VAR_UNIT and VAR_SI_CONVERSION values between AVDC and original Envisat.
;              The STOP_WITH_ERROR routine is no longer called in the event of an error, but an Error
;              State string is returned instead. Bug fixed when testing for a UNIT_PREFIX - previously
;              only the first character of the VAR_UNIT value was checked for a possible UNIT_PREFIX,
;              thus 'deka' (da) was excluded. Bug Fixed when the calculated Power Value of the last
;              base SI unit shown in VAR_SI_CONVERSION is '1', this is now set so that the power value
;              does not appear - Version 3.0
;    20090611: Galileo added to var_unit_arr (AVDC); Last VAR_SI_CONVERSION value for ppmv, ppbv,
;              pptv, and ppv changed to DIMENSIONLESS (was ppv) in var_unit_arr (AVDC);
;              VAR_SI_CONVERSION values for molec changed to 0;1.66054E-24;mol (was 0;1;molec) in
;              var_unit_arr (AVDC); VAR_SI_CONVERSION values for DU changed to 0;4.4615E-4;mol m-2
;              (was 0;2.6867E20;molec m-2) in var_unit_arr; Stop power units being added to
;              DIMENSIONLESS (e.g. for VAR_UNITS=ppmv2); Change EVDC VAR_SI_CONVERSION values in
;              var_unit_arr to match Envisat Metadata Guidelines values - Version 3.01
;    20091117: EVDC VAR_SI_CONVERSION values set to the same as AVDC (only one var_unit_arr set);
;              Fix bug which, in some cases, does not account for repeated units in VAR_UNITS when
;              determining the VAR_SI_CONVERSION (e.g. VAR_UNITS=W m-2 sr-1 m-1); Put VAR_SI_CONVERSION
;              units in power value order; Correctly scale multiple units by the power
;              e.g. W2 = (m2 kg s-3)^2 (previously assumed only a single unit was being scaled);
;              Generate error if third VAR_SI_CONVERSION value is 'DIMENSIONLESS' or 'NONE' but also
;              includes extra values - Version 4.0b1
;    20101001: Set up for GEOMS compliance e.g. DIMENSIONLESS changed to 1; MJD2000 changed to MJD2K;
;              NONE removed - Version 4.0b2
;    20101221: Bug fix - need to correctly account for when units cancel each other out. Was still
;              assigned the value DIMENSIONLESS - Version 4.0b3
;    20110303: Bug fix - when a dimensionless unit includes a power value (e.g. ppmv2), the base unit
;              in VAR_SI_CONVERSION stays as '1'; Add 'dB' to var_unit_arr - Version 4.0b4
;    20110719: Add 'pH' - Version 4.0b5
;
;  Inputs: vuvalue - a string containing the Metadata VAR_UNITS value to be checked (everything
;                    after the '=')
;          rd - integer flag to indicate if VAR_DATA_TYPE is real/double (-1) or not (1). Used to
;               make VAR_SI_CONVERSION values of the same type
;          tab_type - integer flag differentiating between AVDC (0) and original Envisat (1) styles
;
;  Output: var_si_conv - a string containing the VAR_SI_CONVERSION value determined by the program
;                        (returns '' if an error is encountered)
;          errstate - string describing an error state encountered during testing (o/w set to '')
;
;  Called by: CHECK_METADATA
;
;  External Subroutines Called: None

var_unit_arr=['%;0;0.01;1','A;0;1;A','C;0;1;s A','cd;0;1;cd','d;0;86400;s','deg;0;1.74533E-2;rad',$
              'degC;273.15;1;K','1;0;1;1','h;0;3600;s','Hz;0;1;s-1','J;0;1;m2 kg s-2','K;0;1;K',$
              'l;0;1E-3;m3','lm;0;1;cd sr','lx;0;1;cd sr m-2','m;0;1;m','min;0;60;s',$
              'MJD2K;0;86400;s','mol;0;1;mol','molec;0;1.66054E-24;mol','Np;0;1;1',$
              'N;0;1E3;m kg s-2','Pa;0;1;m-1 kg s-2','pH;0;1E-12;m2 kg s-2 A-2','photons;0;1;photons',$
              'ppbv;0;1E-9;1','ppmv;0;1E-6;1','pptv;0;1E-12;1','ppv;0;1;1','psu;0;1;psu',$
              'rad;0;1;rad','s;0;1;s','sr;0;1;sr','V;0;1;m2 kg s-3 A-1','W;0;1;m2 kg s-3','kg;0;1;kg',$
              'DU;0;4.4614E-4;mol m-2','Gal;0;1E-2;m s-2','dB;0;1;1']

unit_pre_arr=['Y;yotta;1E24','Z;zetta;1E21','E;exa;1E18','P;peta;1E15','T;tera;1E12','G;giga;1E9',$
              'M;mega;1E6','k;kilo;1E3','h;hecto;1E2','da;deka;1E1','d;deci;1E-1','c;centi;1E-2',$
              'm;milli;1E-3','u;micro;1E-6','n;nano;1E-9','p;pico;1E-12','f;femto;1E-15',$
              'a;atto;1E-18','z;zepto;1E-21','y;yocto;1E-24']

;Set up text in case an error is found in the input VAR_UNITS value
errtxt=STRARR(2)
IF tab_type EQ 1 THEN errtxt[0]='No match with Table.Dat BASE_UNIT/UNIT_PREFIX values' $
ELSE errtxt[0]='No match with Table Attribute Values file VAR_UNITS/UNIT_PREFIX values'
errtxt[1]='Not valid'
errtxt='VAR_UNITS='+vuvalue+': '+errtxt
var_si_conv='' & errstate='' ;initialize outputs

nta=N_ELEMENTS(var_unit_arr)
;extract var_unit_arr/unit_pre_arr sub-values into vu/up arrays
vuhold=STRSPLIT(var_unit_arr[0],';',/Extract) ;test for number of sub-values
nvu=N_ELEMENTS(vuhold) & vu=STRARR(nvu,nta)
FOR j=0,nta-1 DO BEGIN
  vuhold=STRSPLIT(var_unit_arr[j],';',/Extract)
  vu[*,j]=vuhold
ENDFOR
nta=N_ELEMENTS(unit_pre_arr)
vuhold=STRSPLIT(unit_pre_arr[0],';',/Extract) ;test for number of sub-values
nup=N_ELEMENTS(vuhold) & up=STRARR(nup,nta)
FOR j=0,nta-1 DO BEGIN
  vuhold=STRSPLIT(unit_pre_arr[j],';',/Extract)
  up[*,j]=vuhold
ENDFOR

;separate out metadata sub-values into component parts, and set-up holding arrays
vp=STRSPLIT(STRTRIM(vuvalue,2),' ',/Extract) & vpn=N_ELEMENTS(vp)
vpx=STRARR(2,vpn) ;0 holds VAR_UNIT value, 1 holds POWER component
bpx=STRARR(vpn) ;holding string array for base unit
ex=INTARR(vpn)+1 ;holding integer array for power value (defaults to 1)
tm=DBLARR(vpn) ;holding array for scale factor
j=0
WHILE (j LE vpn-1) AND (errstate EQ '') DO BEGIN
  ;test to see if the sub-value is a base unit in the TAV file
  ti=WHERE(vp[j] EQ vu[0,*],tcnt)
  IF tcnt NE 0 THEN vpx[0,j]=vp[j] $ it is a base unit
  ELSE BEGIN ;separate out into unit and power values as required
    vpx[0,j]=STRMID(vp[j],0,1) ;save first character of vp(j)
    stopchk=0
    IF (STRLEN(vp[j]) GE 2) THEN BEGIN
      FOR k=1,STRLEN(vp[j])-1 DO BEGIN
        ah=STRMID(vp[j],k,1)
        test1=(BYTE(ah) GE 65) AND (BYTE(ah) LE 90) ;A-Z
        test2=(BYTE(ah) GE 97) AND (BYTE(ah) LE 122) ;a-z
        IF (test1[0]) OR (test2[0]) THEN BEGIN
          IF stopchk EQ 0 THEN vpx[0,j]=vpx[0,j]+ah ELSE stopchk=2
          ;if stopchk EQ 2 THEN this means that VAR_UNITS is not legal
        ENDIF ELSE IF stopchk EQ 0 THEN BEGIN
          stopchk=1 ;first non-alpha character so check for numeric or '-' character
          test1=(BYTE(ah) EQ 45) OR ((BYTE(ah) GE 49) AND (BYTE(ah) LE 57)) ;- or 1-9
          IF NOT test1[0] THEN stopchk=2 ELSE vpx[1,j]=ah
        ENDIF ELSE IF stopchk EQ 1 THEN BEGIN
          ;need to check for a numeric character only
          test1=(BYTE(ah) GE 48) AND (BYTE(ah) LE 57) ;0-9
          IF NOT test1[0] THEN stopchk=2 ELSE vpx[1,j]=vpx[1,j]+ah
        ENDIF
      ENDFOR
      IF vpx[1,j] NE '' THEN ex[j]=FIX(vpx[1,j])
    ENDIF
    IF stopchk EQ 2 THEN vpx[0,j]=vp[j] ;in the event that the value is not valid, so will create error
    ;Do TAV check on the VAR_UNIT value
    ti=WHERE(vpx[0,j] EQ vu[0,*],tcnt)
  ENDELSE

  IF tcnt NE 0 THEN BEGIN ;VAR_UNIT is a BASE_VALUE
    bemult=(DOUBLE(vu[nvu-2,ti[0]]))^ex[j] & tm[j]=bemult
    bpx[j]=vu[nvu-3,ti[0]]+';'+STRTRIM(STRING(format='(E8.1)',bemult),2)+';'+vu[nvu-1,ti[0]]
  ENDIF ELSE IF vpx[0,j] EQ 'g' THEN BEGIN ;check for VAR_UNIT EQ g (gram) for AVDC style TAV file
    bemult=0.001d^ex[j] & tm[j]=bemult
    bpx[j]='0;'+STRTRIM(STRING(format='(E8.1)',bemult),2)+';kg'
  ENDIF ELSE BEGIN ;separate out vpx value into prefix and base-value and test
    ;check for valid prefix - first check for 'deka' (da)
    pref=STRMID(vpx[0,j],0,2) & bas=STRMID(vpx[0,j],2)
    pi=WHERE(pref EQ up[0,*],pcnt)
    IF pcnt EQ 0 THEN BEGIN ;test for the remaining prefixes
      pref=STRMID(vpx[0,j],0,1) & bas=STRMID(vpx[0,j],1)
      pi=WHERE(pref EQ up[0,*],pcnt)
    ENDIF
    IF pcnt NE 0 THEN BEGIN
      pmult=DOUBLE(up[nup-1,pi[0]])
      ;check for valid base
      ti=WHERE(bas EQ vu[0,*],tcnt)
      IF tcnt NE 0 THEN BEGIN
        bpmult=(DOUBLE(vu[nvu-2,ti[0]])*pmult)^ex[j] & tm[j]=bpmult
        bpx[j]=vu[nvu-3,ti[0]]+';'+STRTRIM(STRING(format='(E8.1)',bpmult),2)+';'+vu[nvu-1,ti[0]]
      ENDIF ELSE IF bas EQ 'g' THEN BEGIN ;check for VAR_UNIT EQ g (gram) for AVDC style TAV file
        bpmult=(pmult*0.001D)^ex[j] & tm[j]=bpmult
        bpx[j]='0;'+STRTRIM(STRING(format='(E8.1)',bpmult),2)+';kg'
      ENDIF ELSE errstate=errtxt[0]
    ENDIF ELSE errstate=errtxt[0]
  ENDELSE
  j=j+1
ENDWHILE

IF errstate EQ '' THEN BEGIN ;No errors detected so continue
  ;Create VAR_SI_CONVERSION value
  tmult=1.0D
  FOR j=0,vpn-1 DO tmult=tmult*tm[j]

  ;reformat the multiplier e.g. 1.000E+003 becomes 1E3
  ;convert to Exponential form if necessary
  IF (tmult EQ 273.15D) OR (tmult MOD 60.D EQ 0.D) OR ((tmult GE 0.01D) AND (tmult LT 1.D2) $
    AND (tmult*1.D4 MOD 1.D2 EQ 0.D)) THEN tmults=STRTRIM(STRUPCASE(tmult),2) $ ;keep the same format
  ELSE tmults=STRTRIM(STRING(format='(E14.6)',tmult),2)

  epos=STRPOS(tmults,'E') & ppos=STRPOS(tmults,'.')
  IF epos NE -1 THEN BEGIN ;remove unnecessary characters after 'E'
    ep=FIX(STRMID(tmults,epos+1)) & epx=STRTRIM(ep,2)
    tmults=STRMID(tmults,0,epos+1)+epx
  ENDIF
  IF ppos NE -1 THEN BEGIN ;remove any trailing zeroes after the decimal place
    IF epos NE -1 THEN ep=STRMID(tmults,ppos+1,epos-(ppos+1)) ELSE ep=STRMID(tmults,ppos+1)
    WHILE STRMID(ep,STRLEN(ep)-1,1) EQ '0' DO ep=STRMID(ep,0,STRLEN(ep)-1)
    IF ep NE '' THEN tmults=STRMID(tmults,0,ppos+1)+ep ELSE tmults=STRMID(tmults,0,ppos)
    IF epos NE -1 THEN tmults=tmults+'E'+epx
  ENDIF

  ;Scale the units by the power e.g. W2 = (m2 kg s-3)^2
  FOR j=0,vpn-1 DO BEGIN
    vsc=STRSPLIT(bpx[j],';',/EXTRACT)
    vspl=STRSPLIT(vsc[2],' ',/EXTRACT,COUNT=vscnt)
    sichk=STRARR(vscnt) & pwchk=sichk
    IF (vpx[1,j] NE '') AND (vpx[1,j] NE '1') AND (vsc[2] NE '1') THEN BEGIN
      bpx[j]=vsc[0]+';'+vsc[1]+';'
      FOR k=0,vscnt-1 DO BEGIN
        ;separate out SI units and power values
        sires=STRSPLIT(vspl[k],'-0123456789',/Extract)
        sichk[k]=sires[0] ;SI Unit
        IF STRLEN(sichk[k]) NE STRLEN(vspl[k]) THEN $
          pwchk[k]=STRMID(vspl[k],STRLEN(sichk[k])) $
        ELSE pwchk[k]='1' ;Power value
        pwm=FIX(pwchk[k])*FIX(vpx[1,j])
        IF k EQ 0 THEN sp='' ELSE sp=' '
        bpx[j]=bpx[j]+sp+sichk[k]+STRTRIM(pwm,2)
      ENDFOR
    ENDIF
  ENDFOR

  ;Put together VAR_SI_CONVERSION
  vsc=STRSPLIT(bpx[0],';',/EXTRACT)
  ;IF vsc[2] EQ '1' THEN vpx[1,0]=''
  var_si_conv=vsc[0]+';'+tmults+';'+vsc[2]
  ;add remaining base units to VAR_SI_CONVERSION
  IF vpn GT 1 THEN $
    FOR j=1,vpn-1 DO BEGIN
      vsc=STRSPLIT(bpx[j],';',/Extract)
      var_si_conv=var_si_conv+' '+vsc[2]
    ENDFOR

  ;check VAR_SI_CONVERSION for repeated SI units e.g. m m-3 becomes m-2
  schk=STRSPLIT(var_si_conv,' ;',/Extract) & scnt=N_ELEMENTS(schk)
  IF scnt GT 3 THEN BEGIN ;more than one SI unit in VAR_SI_CONVERSION
    sichk=STRARR(scnt-2) & pwchk=sichk
    FOR j=0,scnt-3 DO BEGIN ;separate out SI units and power values
      sires=STRSPLIT(schk[j+2],'-0123456789',/Extract)
      sichk[j]=sires[0] ;SI Unit
      IF STRLEN(sichk[j]) NE STRLEN(schk[j+2]) THEN $
        pwchk[j]=STRMID(schk[j+2],STRLEN(sichk[j])) $
      ELSE pwchk[j]='1' ;Power value
    ENDFOR
    j=0 & pwval=0
    WHILE j LT scnt-3 DO BEGIN
      si=WHERE((sichk[j] NE '') AND (sichk[j] EQ sichk[j+1:scnt-3]),sicnt)
      IF sicnt NE 0 THEN BEGIN
        FOR k=0,sicnt-1 DO BEGIN
          si[k]=si[k]+j+1
          pwval=FIX(pwchk[j])+FIX(pwchk[si[k]])
          IF (pwval[0] EQ 0) AND (k EQ sicnt-1) THEN BEGIN
            sichk[j]='' & pwchk[j]=''
          ENDIF ELSE IF (pwval[0] EQ 1) and (k EQ sicnt-1) THEN pwchk[j]='' $
          ELSE pwchk[j]=STRTRIM(pwval[0],2)
          ;make null all the other SI values
          sichk[si[k]]='' & pwchk[si[k]]=''
        ENDFOR
      ENDIF ELSE IF pwchk[j] EQ '1' THEN pwchk[j]=''
      j=j+1
    ENDWHILE
    ;Also do check on the power value of the last SI Unit
    IF pwchk[scnt-3] EQ '1' THEN pwchk[scnt-3]=''

    ;Put units in power value order
    pwhold=pwchk
    oi=WHERE(pwhold EQ '',ocnt)
    IF ocnt NE 0 THEN pwhold[oi]='1'
    pws=SORT(FIX(pwhold))
    sichk=sichk[REVERSE(pws)] & pwchk=pwchk[REVERSE(pws)]

    var_si_conv=schk[0]+';'+schk[1]+';'+sichk[0]+pwchk[0]
    si=WHERE(sichk NE '',sicnt)
    IF sicnt EQ 0 THEN var_si_conv=var_si_conv+'1' $ ;i.e. values cancelled out
    ELSE BEGIN
      FOR j=1,scnt-3 DO BEGIN
        si=WHERE(sichk[0:j-1] NE '',sicnt)
        IF (sichk[j] EQ '') OR (sicnt EQ 0) THEN var_si_conv=var_si_conv+sichk[j]+pwchk[j] $
        ELSE var_si_conv=var_si_conv+' '+sichk[j]+pwchk[j]
      ENDFOR
    ENDELSE

  ENDIF

  IF rd LT 0 THEN BEGIN
    ;VAR_DATA_TYPE is Real or Double so make VAR_SI_CONVERSION values floating point
    tchkh=STRSPLIT(var_si_conv,';',/Extract) & tup=STRUPCASE(tchkh)
    FOR j=0,1 DO BEGIN
      IF STRPOS(tup[j],'.') EQ -1 THEN BEGIN
        IF STRPOS(tup[j],'E') EQ -1 THEN tchkh[j]=tchkh[j]+'.0' $
        ELSE tchkh[j]=STRMID(tup[j],0,STRPOS(tup[j],'E'))+'.0'+STRMID(tup[j],STRPOS(tup[j],'E'))
      ENDIF
    ENDFOR
    var_si_conv=tchkh[0]+';'+tchkh[1]+';'+tchkh[2]
  ENDIF

  ;Check for invalid VAR_UNITS - third VAR_SI_CONVERSION is 1 plus extra values
  vsc=STRSPLIT(var_si_conv,';',/EXTRACT)
  vsc[2]=STRTRIM(vsc[2],2)
  IF (errstate EQ '') AND (STRMID(vsc[2],0,1) EQ '1') AND (STRLEN(vsc[2]) GT 1) THEN BEGIN
      errstate=errtxt[1] & var_si_conv=''
  ENDIF
ENDIF

END ;procedure Var_Units_Test



PRO read_tablefile, tablefile
;Procedure to identify the version number of the TAV file, read the contents of the
;LABELS/FIELDS (tab_arr), and determine the FILE_META_VERSION in the global attributes (tab_ver).
;This routine also creates a flag (tab_type) to determine whether the input file is an original
;table.dat file used by Envisat or the GEOMS TAV file, with the HDF/netCDF file generated
;accordingly.
; ----------
;Written by Ian Boyd for the AVDC - iboyd@astro.umass.edu
;
;History:
;  20050802: Original IDLCR8HDF Routine - Version 1.0
;  20061012: Bug-fix to separate semi-colons with a space when one immediately follows the other,
;            so that the number of subvalues is correctly determined by the program. Common
;            variable definition WIDGET_WIN added - Version 2.0
;  20080302: Added code to differentiate between the AVDC TAV file and original Envisat table.dat.
;            The tab_type flag is used to differentiate between the two formats. Note that some of the
;            table.dat labels are renamed to match the equivalent TAV file labels for compatibility
;            when testing Metadata entries - refer to EnviName/AVDCName arrays; Ensure that the TAV
;            Version value is correctly formatted and is version 03 or greater - Version 3.0
;  20100205: Add RETURN command after all STOP_WITH_ERROR calls, which allows program to return to the
;            calling program if the reterr argument is included in the idlcr8hdf call - Version 3.09
;  20110401: Change AVDC references to GEOMS; Test AVDC TAV file version is version 04 or greater;
;            Remove check on format of TAV file version; Conform to new INFOTXT_OUTPUT reporting
;            - Version 4.0b1
;
;Input: Tablefile - Name of the file containing the Table Attributes.
;
;Outputs: tab_ver - String containing FILE_META_VERSION value.
;         tab_arr - 2-D string array of size nf*mcnt+2, where nf=Number of Fields, and mcnt=maximum
;                   number of values detected in any one field. tab_arr(*,0) is the name of each field,
;                   and tab_arr(*,1) is the number of values in each field.
;         tab_type - 0/1 where 0 identifies an AVDC format TAV file and 1 identifies an original
;                    Envisat format table.dat file.
;
;Called by: IDLCR8HDF
;
;Subroutines Called: STOP_WITH_ERROR (if error state detected); INFOTXT_OUTPUT
;  Possible Conditions for STOP_WITH_ERROR call (plus [line number] where called):
;    1. Table Attribute Values file version not identified
;    2. Envisat table.dat ASC2HDF program version not found
;    3. First Envisat table.dat field value not found
;    4. Table Attribute Values file version is not in a valid format
;
;  Information Conditions (when the program reports issues and continues):
;    1. [Original EVDC]/[GEOMS] Reporting Guidelines Apply (depending on type of Table Attribute
;       Values file read in)
;    2. Old version of the TAV file in use. Update from http://avdc.gsfc.nasa.gov/Tools

COMMON TABLEDATA
COMMON WIDGET_WIN

;Possible error messages for this procedure
proname='Read_TableFile procedure: '
errtxt=STRARR(4)
errtxt[0]='Table Attribute Values file version not identified'
errtxt[1]='Envisat table.dat ASC2HDF program version not found'
errtxt[2]='First Envisat table.dat field value not found'
errtxt[3]='Table Attribute Values file version is invalid (should be ddRddd or ddRdddd): '
FOR i=0,2 do errtxt[i]=errtxt[i]+' with the search criteria used by this program'

;Array of Envisat Field names to be changed to equivalent AVDC Field names
enviname=['_NAME','_AFFILIATION','DATA_VARIABLES_00_00','BASE_UNIT']
avdcname=['ORIGINATOR','AFFILIATION','DATA_VAR_ALL_00_00','VAR_UNITS']

ON_IOERROR,TypeConversionError
dum='' & tab_ver=''
min_fmv=4 ;TAV Version must be GE this value e.g. 04R001, 06R002 but not 03R004
OPENR,lu,tablefile,/GET_LUN
;determine TAV file version
REPEAT BEGIN
  READF,lu,dum
  dumup=STRUPCASE(dum)
  envitest=STRPOS(STRCOMPRESS(dumup,/Remove_all),'!TABLE.DATVERSION') NE -1
  avdctest=STRPOS(STRCOMPRESS(dumup,/Remove_all),'!VERSION') NE -1
ENDREP UNTIL (envitest) OR (avdctest) OR (EOF(lu))
IF EOF(lu) THEN BEGIN
  STOP_WITH_ERROR,o3[3]+proname,errtxt[0],lu & RETURN
ENDIF
res=STRSPLIT(dumup,' ',/Extract) & vi=WHERE(res EQ 'VERSION')
IF N_ELEMENTS(res) LE vi[0]+1 THEN BEGIN
  STOP_WITH_ERROR,o3[3]+proname,errtxt[0]+': '+dum,lu & RETURN
ENDIF
IF envitest THEN BEGIN
  tab_type=1 & infotxt='0 EVDC original style table.dat'
  n_title=4
ENDIF ELSE BEGIN
  tab_type=0 & infotxt='0 GEOMS compliant Table Attribute Values'
  n_title=5
ENDELSE
infotxt=infotxt+' file input. '+STRMID(infotxt,2,n_title)+' Reporting Guidelines apply'
INFOTXT_OUTPUT,infotxt

;Ensure Meta Version has a valid format
;i. Check third character is an 'R'
;ii. Check number of characters is 6 or 7
;iii. Check dd and ddd(d) are numeric
;iv. Check that version number is 03 or greater for AVDC file type
;v. Issue warning and convert to ddRddd if format is ddRdddd
fmv=res[vi[0]+1] ;File_Meta_Version
valid=0 ;set to test for valid string to number conversion
FOR i=0,STRLEN(fmv)-1 DO IF i NE 2 THEN fmvtest=FIX(STRMID(fmv,i,1))
fmvtest=FIX(STRMID(fmv,0,2)) ;To test for TAV version 'min_fmv' or greater
valid=1 ;FILE_META_VERSION characters are numeric (except for the 'R') if program gets to here
TypeConversionError:
IF (STRMID(fmv,2,1) NE 'R') OR (STRLEN(fmv) lt 6) OR (STRLEN(fmv) gt 7) OR $
   (valid EQ 0) THEN BEGIN
  STOP_WITH_ERROR,o3[3]+proname,errtxt[3]+fmv,lu & RETURN
ENDIF

IF (tab_type EQ 0) AND (fmvtest LT min_fmv) THEN BEGIN
  infotxt=STRARR(2)
  infotxt[0]='3 Old version of the Table Attribute Values file used as input: '
  infotxt[0]=infotxt[0]+' TAV Version '+fmv
  infotxt[1]='    Please update from http://avdc.gsfc.nasa.gov/Tools'
  INFOTXT_OUTPUT,infotxt
ENDIF

IF envitest THEN BEGIN ;identify asc2hdf version in table.dat and read past the CHECK_ATTRIBUTE line
  WHILE (STRPOS(STRUPCASE(dum),'ASC2HDF') EQ -1) AND (NOT EOF(lu)) DO READF,lu,dum
  IF EOF(lu) THEN BEGIN
    STOP_WITH_ERROR,o3[3]+proname,errtxt[1],lu & RETURN
  ENDIF
  cr8_hdf_ver=';IDLCR8HDF'
  dum=STRTRIM(dum,2)
  WHILE ((STRMID(dum,0,1) EQ '!') OR (STRMID(dum,0,1) EQ '#') OR (dum EQ '')) AND $
        (NOT EOF(lu)) DO BEGIN
    READF,lu,dum & dum=STRTRIM(dum,2)
  ENDWHILE
  IF EOF(lu) THEN BEGIN
    STOP_WITH_ERROR,o3[3]+proname,errtxt[2],lu & RETURN
  ENDIF
  READF,lu,dum ;to get to the line after 'CHECK_ATTRIBUTE'
ENDIF ELSE cr8_hdf_ver=';IDLCR8HDF'
tab_ver=fmv+cr8_hdf_ver ;= the FILE_META_VERSION input value in the global attributes

;determine no. of FIELDS/LABELS as well as the maximum number of elements
nf=0 & mcnt=0 & firstfield=''
dum=STRTRIM(dum,2)
WHILE NOT EOF(lu) DO BEGIN
  ncnt=-1
  IF (STRMID(dum,0,1) NE '!') AND (STRMID(dum,0,1) NE '#') AND $
     (STRMID(dum,0,1) NE '=') AND (dum NE '') THEN BEGIN

    FOR i=0,N_ELEMENTS(enviname)-1 DO BEGIN
      epos=STRPOS(STRUPCASE(dum),enviname[i])
      IF (epos NE -1) AND (epos LE 3) THEN ncnt=i
    ENDFOR
    IF (ncnt EQ 0) OR (ncnt EQ 1) THEN BEGIN
      IF STRMID(STRUPCASE(dum),0,3) NE 'PI_' THEN nf=nf-1 ELSE ecnt=0
      ;This puts all PI_,DO_, and DS_NAME or AFFILIATION values into either the ORIGINATOR or AFFILIATION fields
    ENDIF ELSE ecnt=0

    nf=nf+1
    IF firstfield EQ '' THEN firstfield=dum
    IF ncnt NE -1 THEN dum=avdcname[ncnt] ;change the name of the Envisat field to AVDC equivalent
    REPEAT BEGIN
      READF,lu,dum & dum=STRTRIM(dum,2)
      IF STRMID(dum,0,1) EQ '=' THEN ecnt=ecnt+1
    ENDREP UNTIL (STRMID(dum,0,1) NE '=') OR (EOF(lu))
    IF ecnt GT mcnt THEN mcnt=ecnt
  ENDIF ELSE IF NOT EOF(lu) THEN BEGIN
    READF,lu,dum & dum=STRTRIM(dum,2)
  ENDIF
ENDWHILE
FREE_LUN,lu

;read in the contents of the file
tab_arr=STRARR(nf,mcnt+2) ;note tab_arr(*,0) EQ FIELD/LABEL name and
;tab_arr(*,1) EQ N_ELEMENTS under each FIELD/LABEL
tab_hold=STRARR(mcnt)

OPENR,lu,tablefile,/GET_LUN
READF,lu,dum & dum=STRTRIM(dum,2)
WHILE dum NE firstfield DO BEGIN
  READF,lu,dum & dum=STRTRIM(dum,2)
ENDWHILE

i=0
WHILE i LE nf-1 DO BEGIN
  ncnt=-1
  FOR j=0,N_ELEMENTS(enviname)-1 DO BEGIN
    epos=STRPOS(STRUPCASE(dum),enviname[j])
    IF (epos NE -1) AND (epos LE 3) THEN ncnt=j
  ENDFOR
  IF (ncnt EQ 0) OR (ncnt EQ 1) THEN BEGIN
    IF STRMID(STRUPCASE(dum),0,3) NE 'PI_' THEN i=i-1 ELSE ecnt=0
  ENDIF ELSE ecnt=0
  IF ncnt NE -1 THEN dum=avdcname[ncnt] ;change the name of the Envisat field to AVDC equivalent
  tab_arr[i,0]=dum
  REPEAT BEGIN
    READF,lu,dum & dum=STRTRIM(dum,2)
    IF STRMID(dum,0,1) EQ '=' THEN BEGIN
      tab_hold[ecnt]=STRMID(dum,1) ;strip the '=' sign
      ;add space between adjacent semi-colons so StrSplit finds correct number of sub-values
      REPEAT BEGIN
        cpos=STRPOS(tab_hold[ecnt],';;')
        IF cpos NE -1 THEN tab_hold[ecnt]=STRMID(tab_hold[ecnt],0,cpos+1)+ $
          ' '+STRMID(tab_hold[ecnt],cpos+1)
      ENDREP UNTIL cpos EQ -1
      ecnt=ecnt+1
    ENDIF
  ENDREP UNTIL (STRMID(dum,0,1) NE '=') OR (EOF(lu))
  tab_arr[i,1]=STRTRIM(ecnt,2)
  tab_arr[i,2:ecnt+1]=tab_hold[0:ecnt-1]
  IF i NE nf-1 THEN BEGIN
    WHILE (STRMID(dum,0,1) EQ '!') OR (STRMID(dum,0,1) EQ '#') OR (dum EQ '') DO BEGIN
      READF,lu,dum & dum=STRTRIM(dum,2)
    ENDWHILE
  ENDIF
  i=i+1
ENDWHILE
FREE_LUN,lu

END ;Procedure Read_TableFile



PRO test_file_input,aname,fentry
;Procedure to test that a file name given as an entry to a free text attribute is valid, based on Envisat
;Metadata Guidelines (not currently used by AVDC). Note: No longer permitted under GEOMS guidelines.
; ----------
;Written by Ian Boyd for the AVDC - iboyd@astro.umass.edu
;
;  History:
;    20050802: Original IDLCR8HDF Routine - Version 1.0
;    20061012: Common variable definition WIDGET_WIN added - Version 2.0
;    20100205: Add RETURN command after all STOP_WITH_ERROR calls, which allows program to return to the
;              calling program if the reterr argument is included in the idlcr8hdf call - Version 3.09
;
;  Inputs: aname - Global or Variable Attribute Label
;          fentry - Filename entry used as the Global or Variable Attribute value
;
;  Outputs: None
;
;  Called by: READ_METADATA
;
;  Subroutines Called: STOP_WITH_ERROR (if error state detected)
;    Possible Conditions for STOP_WITH_ERROR call (plus [line number] where called):
;      1. Syntax of Filename entry incorrect
;      2. Filename entry not found or not usable
;      3. File size is too large

COMMON WIDGET_WIN

sfile=4096 ;maximum permitted file size

;possible error message for this procedure
proname='Test_File_Input procedure: '
errtxt2=STRARR(3) & lu=-1
errtxt2[0]='Syntax of Filename entry incorrect: '
errtxt2[1]='Filename entry not found or not usable: '
errtxt2[2]='File size is too large (maximum permitted: '+STRTRIM(sfile,2)+' bytes)'

si=STRPOS(fentry,'"')+1 & ei=STRPOS(fentry,'"',/Reverse_Search)-si
IF ei EQ si THEN BEGIN
  STOP_WITH_ERROR,o3[3]+proname+aname+': ',errtxt2[0]+fentry,lu & RETURN
ENDIF
faname=STRMID(fentry,si,ei)
ftest=FILE_TEST(faname,/Read,/Regular)
IF ftest EQ 0 THEN BEGIN
  STOP_WITH_ERROR,o3[3]+proname+aname+': ',errtxt2[1]+faname,lu & RETURN
ENDIF
OPENR,fu,faname,/GET_LUN
ftest=FSTAT(fu) & FREE_LUN,fu
IF ftest.size GT sfile THEN BEGIN
  STOP_WITH_ERROR,o3[3]+proname+AName+'='+fentry+': ',errtxt2[2],lu & RETURN
ENDIF

END ;procedure Test_File_Input



PRO geoms_rule_changes, code, in1, in2, in3, in4
;Procedure to check Metadata for old/redundant rules, labels and/or values and update/report
;as required
; ----------
;Written by Ian Boyd for the AVDC - iboyd@astro.umass.edu
;
;  History:
;    20110401: Introduced. Incorporates GEOMS rules changes from v3.0 to v4.0 - Version 4.01
;    20111220: Add ISO646-US ASCII character set check - Version 4.0b7
;    20120313: Add UVVIS.DOAS plus additional gases to Code 6 checks - Version 4.0b8
;    20120703: Bug Fix when checking for non-ASCII characters (rule 10) - Version 4.0b11
;    20131023: Add GEOMS rule change for MIXING.RATIO[.VOLUME][.MASS], and UNCERTAINTY (rule 5);
;              Fixed bug that caused incorrect rep_list_2 values to be written to file (rule 5);
;              Stopped checks for illegal characters if ORIGINATOR values are present in the
;              TAV file (rule 10) - Version 4.0b16
;    20131029: Add array of invalid ASCII characters that can be replaced with valid ISO646 US
;              ASCII characters (rule 10) - Version 4.0b17
;    20131029: Modify UNCERTAINTY rules to allow more exceptions (rule 5); Allow for renaming of
;              LIDAR.DIAL DATA_SOURCE to LIDAR.WATERVAPORDIAL if conditions met (rule 6) -
;              Version 4.0b18
;    20140226: Add 'DU' to obsolete descriptor value and fix bug causing no new variable name to
;              be generated if more than 3 sub-values in the name (rule 5); Add 'RO.SAC.C' to
;              the obsolete DATA_SOURCE list (rule 6) - Version 4.0b19
;    20140521: Account for extra parameter in the UVVIS-DOAS data source when trying to determine
;              if file is based on a DATA_TEMPLATE (rule 9) - Version 4.0b21
;    20140806: Modify UNCERTAINTY rules to allow more exceptions including additional instrument
;              types and check on molec cm-2 to allow for prefix values (rule 5) - Version 4.0b22
;    20140909: Add RO.F3C.FM[1][2][3] to the obsolete list of DATA_SOURCES (replaced by
;              F3C.FM[1][2][3]) (rule 6) - Version 4.0b23
;    20141110: Numerous changes while testing harmonization of EVDC files - additional
;              DATA_SOURCEs and units added to identify correct new UNCERTAINTY values, including
;              identification of covariance uncertainties; Bug fix when doing checks on
;              AIR.CONCENTRATION; LIDAR.BACKSCATTER conversion and units definition added; List
;              of non-ASCII characters to be checked expanded; Bug fix when correcting
;              for non-ASCII characters when the replacement value has more than one character
;              e.g. ' deg. ' - Version 4.0b24
;    20150409: Strip trailing and leading spaces from DATA_TEMPLATE value when doing checks
;              (rule 9) - Version 4.0b28
;    20160213: Add extra satellite instruments to the obsolete list for DATA_SOURCE (rule 6);
;              Fix issue with DATA_TEMPLATE identification (rule 9) - Version 4.0b37
;    20171121: Fix bug that selected the incorrect DATA_TEMPLATE in some situations (rule 9)
;              - Version 4.0b43
Ian Boyd's avatar
Ian Boyd committed
1230
1231
1232
;    20190506: Add FILE_META_VERSION information to INFORMATION/ERROR comment when 
;              DATA_TEMPLATE value is not as expected; Fix bug that caused a crash if the
;              file does not use a DATA_TEMPLATE (rule 9) - Version 4.0b50
Ian Boyd's avatar
Ian Boyd committed
1233
1234
;    20191205: Add rule 11 which does checks on optional VERSION_NAME sub-field of
;              DATA_SOURCE - Version 4.0b53
Ian Boyd's avatar
Ian Boyd committed
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
;
;  Inputs: code - Integer value identifying type of check to carry out
;          in1 - First set of inputs required for checks (optional, dependent on code value)
;          in2 - Second set of inputs required for checks (optional, dependent on code value)
;          in3 - Third set of inputs required for checks (optional, dependent on code value)
;          in4 - Fourth set of inputs required for checks (optional, dependent on code value)
;
;  Outputs: Any or all of the inputs can be changed dependent on the code value
;
;  Called by: READ_METADATA; CHECK_METADATA; SET_UP_STRUCTURE; FIND_HDF_FILENAME
;
;  Subroutines Called: INFOTXT_OUTPUT
;    Information Conditions (when the program is able to make changes):
;      1. Attribute entry is not GEOMS compliant, and removed from the metadata saved to the output file
;      2. Dataset name renamed according to new DATETIME reporting conventions
;      3. Double underscore replaced with a single underscore in the variable name
;      4. VAR_DEPEND value made self-referencing for axis variable
;      5. Axis variable cannot be dependent on another variable
;      6. VAR_UNITS=MJD2000 not GEOMS compliant, changed to VAR_UNITS=MJD2K
;      7. DATA_FILE_VERSION must be in the form 'nnn'
;      8. Obsolete Dataset name renamed
;      9. Obsolete DATA_SOURCE value renamed
;     10. Obsolete FILE_ACCESS values renamed
;     11. Rename obsolete VAR_UNITS=DIMENSIONLESS/NONE values
;     12. DATA_TEMPLATE field is not present in the TAV file
;     13. DATA_TEMPLATE value renamed based on DATA_SOURCE value
;     14. Entry must only include valid characters from the ISO646-US ASCII character set
;
;  Code 0: Check for Attributes that have become redundant between v3.0 and v4.0.
;          Inputs: in1=mh, in2=mhgood
;  Code 1: Check for redundant _START/STOP/INTEGRATION.TIME variable names and replace with
;          DATETIME.START/STOP/INTEGRATION.TIME (for single occurences only). Also check
;          for double underscores in the variable names and replace with single underscore
;          Inputs: in1=vncnt, in2=dv
;  Code 2: Check for Axis Variables and, if found, make VAR_DEPEND=INDEPENDENT self-referencing
;          Inputs: in1=vardeptest, in2=vn[vc], in3=resvd[1], in4=holdvd
;  Code 3: Change VAR_UNIT value MJD2000 to MJD2K
;          Inputs: in1=meta_arr[i], in2=res[1], in3=writeonce
;  Code 4: Do DATA_FILE_VERSION checks
;          Inputs: in1=dfvv (DATA_FILE_VERSION value)
;  Code 5: Look for obsolete DATA_VARIABLES (based on list) and, if possible, modify values to
;          GEOMS compliance
;          Inputs: in1=vncnt, in2=dv, in3=vuv, in4=data_source
;  Code 6: Look for obsolete DATA_SOURCE (based on list) and, if possible, modify values to
;          GEOMS compliance
;          Inputs: in1=vncnt, in2=dv, in3=data_source
;  Code 7: Looks for and replaces obsolete CALVAL and NDSC FILE_ACCESS values
;          Inputs: in1=facnt, in2=fav
;  Code 8: Looks for VAR_UNITS=DIMENSIONLESS or NONE and replace with '' or '1'
;          Inputs: in1=vucnt, in2=vuv, in3=vdv
;  Code 9: Do DATA_TEMPLATE/DATA_QUALITY checks
;          Inputs: in1=m_v0, in2=m_v1, in3=ga_chk
;  Code 10: Do checks on ISO646-US ASCII character set
;           Inputs: in1=meta_arr or dtest, in2=vn[vc] (Variable Name for datasets only)
Ian Boyd's avatar
Ian Boyd committed
1289
1290
;  Code 11: Do checks on DATA_VERSION_NAME part of DATA_SOURCE value
;           Inputs: in1=DATA_VERSION_NAME, in2=meta_arr
Ian Boyd's avatar
Ian Boyd committed
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761

COMMON TABLEDATA
COMMON METADATA
COMMON DATA
COMMON WIDGET_WIN

CASE 1 OF
  code EQ 0: BEGIN
      ;Check for redundant attributes
      redundant=['DATA_TYPE','DATA_LEVEL','VAR_MONOTONE','VAR_DIMENSION',$
                 'VAR_AVG_TYPE','VIS_LABEL','VIS_FORMAT','VIS_PLOT_TYPE',$
                 'VIS_SCALE_TYPE','VIS_SCALE_MIN','VIS_SCALE_MAX']
      nrd=N_ELEMENTS(redundant)

      FOR i=0,nrd-1 DO BEGIN
        rdl=STRLEN(redundant[i])
        fai=WHERE(STRMID(STRUPCASE(in1),0,rdl) EQ redundant[i],facnt)
        IF facnt NE 0 THEN BEGIN
          in2[fai]=0 ;Metadata lines not wanted
          test1=(STRMID(redundant[i],0,3) EQ 'VAR') OR (STRMID(redundant[i],0,3) EQ 'VIS')
          IF test1 THEN att_type='Variable' ELSE att_type='Global'
          infotxt='1 '+redundant[i]+' not a GEOMS '+att_type+' attribute|.'
          infotxt=infotxt+' Removed from the metadata saved to the output file'
          INFOTXT_OUTPUT, infotxt
        ENDIF
      ENDFOR
    END ;Case 0

  code EQ 1: BEGIN
      ;Check for redundant _START/STOP sub-values and replace with
      ;DATETIME.START/STOP/INTEGRATION.TIME (for single occurences only)
      time_attr=['START','STOP','INTEGRATION','RESOLUTION']
      n_tim=N_ELEMENTS(time_attr)
      iscnt=0 ;Used for RESOLUTION/INTEGRATION.TIME checks
      FOR i=0,n_tim-1 DO BEGIN
        si=WHERE(STRPOS(in2,'_'+time_attr[i]+'.TIME') NE -1,scnt)
        IF (scnt EQ 1) AND (iscnt EQ 0) THEN BEGIN ;change name and also relevant VAR_NAME if present
          IF i EQ 2 THEN iscnt=1 ;used in the event both INTEGRATION and RESOLUTION.TIME are present
          vnchange[si[0]]=in2[si[0]]
          IF i GE 2 THEN in2[si[0]]=time_attr[2]+'.TIME' $
          ELSE in2[si[0]]='DATETIME.'+time_attr[i]
          IF qa_yes THEN itxt=' expected to be ' ELSE itxt=' renamed '
          infotxt='2 Dataset name '+vnchange[si[0]]+itxt+in2[si[0]]
          INFOTXT_OUTPUT,infotxt
        ENDIF
      ENDFOR
      ;Check for double underscore and replace with single underscore
      FOR i=0,in1-1 DO BEGIN
        IF STRPOS(in2[i],'__') NE -1 THEN BEGIN
          vnchange[i]=in2[i]
          infotxt=STRARR(2)
          infotxt[0]='2 Double underscore present in Variable Name '+in2[i]+'|'
          infotxt[1]='    replaced with a single underscore'
          INFOTXT_OUTPUT, infotxt
          vns=STRSPLIT(in2[i],'_',/Extract,COUNT=n_vns)
          FOR j=0,n_vns-1 DO IF j EQ 0 THEN in2[i]=vns[j] ELSE in2[i]=in2[i]+'_'+vns[j]
        ENDIF
      ENDFOR
    END ;Case 1

  code EQ 2: BEGIN ;Check for and Apply GEOMS rule changes for INDEPENDENT and axis variables
      ;RULE: Any variable that is mentioned in VAR_DEPENDS is by definition an axis variable
      vdtest=STRUPCASE(in1) & vnv=STRUPCASE(in2) & vdv=STRUPCASE(in3) ;vdtest is an array of VAR_DEPEND variables
      avi=WHERE(vnv EQ vdtest,avcnt) ;i.e. is the VAR_NAME an axis variable
      IF avcnt NE 0 THEN BEGIN ;possible axis variable so do axis variable checks
        test1=(vdv EQ 'INDEPENDENT') OR (vdv EQ vnv) ;value is independent or self-referencing
        test2=(vdv NE 'INDEPENDENT') AND (vdv NE vnv) AND (vdv NE 'CONSTANT') ;neither independent, self-referencing, nor constant
        test6=(vdv EQ 'CONSTANT') AND (vnv EQ 'DATETIME') ;i.e. VAR_NAME=DATETIME and VAR_DEPEND=CONSTANT
        IF (test1) OR (test6) THEN BEGIN
          IF vdv NE vnv THEN BEGIN
            infotxt='2 '+in4+' should be self-referencing for axis variable VAR_NAME='+vnv+'|.'
            infotxt=infotxt+' VAR_DEPEND value changed in metadata'
            INFOTXT_OUTPUT,infotxt
            in3=vnv ;self-referencing VAR_DEPEND value
          ENDIF
        ENDIF ELSE IF test2 THEN BEGIN
          in4[0]='E1' ;i.e. axis variable cannot be dependent on another variable so return error
        ENDIF
      ENDIF ELSE BEGIN ;not an axis variable so ensure VAR_DEPEND value is not self-referencing
        IF vdv EQ vnv THEN in4[0]='E3' ;i.e. VAR_DEPEND value cannot be self-referencing
      ENDELSE
    END ;Case 2

  code EQ 3: BEGIN ;Change MJD2000 to MJD2K
      in1='VAR_UNITS=MJD2K' & in2='MJD2K'
      IF in3 EQ 0 THEN BEGIN ;First instance of MJD2000 so include comment
        infotxt='2 VAR_UNITS=MJD2000 not GEOMS compliant|. Changed to VAR_UNITS=MJD2K'
        INFOTXT_OUTPUT,infotxt
      ENDIF
    END

  code EQ 4: BEGIN ;checks on DATA_FILE_VERSION
      in1=STRTRIM(in1,2)
      vp=0 & errval=0 & in1h=in1
      ON_IOERROR,TypeConversionError
      IF STRMID(STRUPCASE(in1),0,1) EQ 'V' THEN BEGIN
        vp=1 & in1=STRMID(in1,1)
      ENDIF
      dpos=STRPOS(in1,'.')
      valid=0 ;to check for conversion error
      IF  dpos NE -1 THEN BEGIN
        IF LONG(STRMID(in1,dpos+1)) EQ 0L THEN atxt='' ELSE atxt=STRMID(in1,dpos+1)
        in1=STRMID(in1,0,dpos)+atxt
        IF FIX(in1) GE 100 THEN in1=STRMID(in1,0,dpos) ;just use version up to the decimal point
      ENDIF
      IF (FIX(in1) GT 999) OR (FIX(in1) LT 1) THEN errval=1
      valid=1 ;OK if got to here

      TypeConversionError: ;Check for invalid DATA_FILE_VERSION value
      IF (valid EQ 0) OR (errval EQ 1) THEN BEGIN
        infotxt='3 DATA_FILE_VERSION='+in1h+' must be in the form ''nnn'''
        INFOTXT_OUTPUT, infotxt
        in1=in1h
      ENDIF ELSE BEGIN ;format value correctly (nnn)
        IF FIX(in1) LT 10 THEN in1='00'+STRTRIM(FIX(in1),2) $
        ELSE IF FIX(in1) LT 100 THEN in1='0'+STRTRIM(FIX(in1),2) $
        ELSE in1=STRTRIM(in1,2)
        IF (vp EQ 1) OR (in1h NE in1) THEN BEGIN
          infotxt='2 DATA_FILE_VERSION must be in the form ''nnn''|. Renamed from '
          infotxt=infotxt+in1h+' to '+in1
          INFOTXT_OUTPUT, infotxt
        ENDIF
      ENDELSE
    END

  code EQ 5: BEGIN ;Look for obsolete DATA_VARIABLES
      ;Notes = what to do about ALTITUDE.LAYER.INDEX, CHL.A.CONCENTRATION,
      ;        PRESSURE.LEVEL.INDEX (still in 04R001), TSM.CONCENTRATION
      ;List of obsolete DATA_VARIABLE_01 values
      obs_list_1=['AIR.TEMPERATURE','TEMPERATURE.AIR','COLUMN.VERTICAL','H2O.RELATIVE.HUMIDITY',$
                  'PRESSURE.SURFACE','TEMPERATURE.SURFACE','TEMPERATURE.INTERNAL.BOX',$
                  'TEMPERATURE.INTERNAL.INSTRUMENT','TEMPERATURE.SEA.SURFACE','AIR.NUMBER.DENSITY',$
                  'SEA.SURFACE.TEMPERATURE','OPACITY.ATMOSPHERIC.VERTICAL']
      ;Corresponding list of replacement DATA_VARIABLE_1 values (direct replacement)
      rep_list_1=['TEMPERATURE','TEMPERATURE','COLUMN','HUMIDITY.RELATIVE',$
                  'SURFACE.PRESSURE','SURFACE.TEMPERATURE','INTERNAL.BOX.TEMPERATURE',$
                  'INTERNAL.INSTRUMENT.TEMPERATURE','SEA.SURFACE.TEMPERATURE','NUMBER.DENSITY',$
                  'SEA.SURFACE.TEMPERATURE.SKIN','OPACITY.ATMOSPHERIC']
      ;Obsolete DATA_VARIABLE_01 which need special handling
      obs_list_spec_1=['.AMF','.AVK','.CONCENTRATION','.MIXING.RATIO']
      ;Obsolete DATA_VARIABLE_02 values
      obs_list_2=['SOLAR','VERTICAL.SOLAR',$
                  'EMISSION.VERTICAL','VERTICAL.EMISSION',$
                  'VERTICAL','SLANT','SOLAR.OCCULTATION','VERTICAL.ZENITH','VERTICAL.NADIR',$
                  'VERTICAL.LUNAR','VERTICAL.SOLAR.FOCUS']
      ;[0,1] for FTIR/UVVIS (except Brewer and Dobson) only, [2,3] for MWR only
      ;Corresponding list of replacement DATA_VARIABLE_2 values (direct replacement)
      rep_list_2=['ABSORPTION.SOLAR|SCATTER.SOLAR','ABSORPTION.SOLAR|SCATTER.SOLAR',$
                  'EMISSION','EMISSION',$
                  '','','OCCULTATION.SOLAR','ZENITH','NADIR',$
                  'ABSORPTION.LUNAR','ABSORPTION.SOLAR.FOCUS']
      ;Replacements for .CONCENTRATION - dependent on VAR_UNITS
      rep_list_conc=['.MIXING.RATIO.VOLUME','.MIXING.RATIO.MASS','.NUMBER.DENSITY','.PARTIAL.PRESSURE']
      ;Conditional Miscellaneous changes
      misc_list_1=['TEMPERATURE_SKIN','AIR.MASS.FACTOR']
      ;Corresponding change to miscellaneous variables
      misc_list_2=['SEA.SURFACE.TEMPERATURE.SKIN','O3.COLUMN_AMF']
      ;Obsolete DATA_VARIABLE_03 descriptors (may or may not be able to fix these)
      obs_list_3=['DU','UNCERTAINTY.STDEV','UNCERTAINTY.RMS','UNCERTAINTY.RELATIVE','UNCERTAINTY.TOTAL', $
                  'UNCERTAINTY.RANDOM','UNCERTAINTY.SYSTEMATIC']
      ;Corresponding list of replacement DATA_VARIABLE_03 descriptors
      rep_list_3=['','UNCERTAINTY.COMBINED.STANDARD','UNCERTAINTY.COMBINED.STANDARD', $
                  'UNCERTAINTY.COMBINED.STANDARD.RELATIVE','UNCERTAINTY.COMBINED.STANDARD', $
                  'UNCERTAINTY.RANDOM.STANDARD','UNCERTAINTY.SYSTEMATIC.STANDARD']
      rep_list_4=['','','','','UNCERTAINTY.COMBINED.COVARIANCE','UNCERTAINTY.RANDOM.COVARIANCE', $
                  'UNCERTAINTY.SYSTEMATIC.COVARIANCE']
      vn_v=STRARR(in1,3) & vn_new=STRARR(in1)
      ;Separate out Variable Name/[Mode or Descriptor]/[Descriptor]
      FOR i=0,in1-1 DO BEGIN
        res=STRSPLIT(in2[i],'_',/EXTRACT,COUNT=nres)
        IF nres GT 3 THEN nres=3
        FOR j=0,nres-1 DO vn_v[i,j]=res[j]
      ENDFOR
      ;Test for values in obs_list_1
      FOR i=0,N_ELEMENTS(obs_list_1)-1 DO BEGIN
        ri=WHERE(STRPOS(STRUPCASE(vn_v[*,0]),obs_list_1[i]) NE -1,rcnt)
        IF rcnt NE 0 THEN BEGIN ;obsolete values found so replace
          vlen=STRLEN(obs_list_1[i])
          FOR j=0,rcnt-1 DO BEGIN
            change_val=1
            ;Special case for SEA.SURFACE.TEMPERATURE (only change if mode is _SKIN)
            IF obs_list_1[i] EQ 'SEA.SURFACE.TEMPERATURE' THEN BEGIN
              IF STRUPCASE(vn_v[ri[j],1]) EQ 'SKIN' THEN vn_v[ri[j],1]='' ELSE change_val=0
            ENDIF
            IF change_val EQ 1 THEN BEGIN
              vpos=STRPOS(STRUPCASE(vn_v[ri[j],0]),obs_list_1[i])
              vn_v[ri[j],0]=STRMID(vn_v[ri[j],0],0,vpos)+rep_list_1[i]+STRMID(vn_v[ri[j],0],vpos+vlen)
            ENDIF
          ENDFOR
        ENDIF
      ENDFOR
      ;Special cases for DATA_VARIABLE_01 values .AMF, .AVK, .CONCENTRATION, and .MIXING.RATIO
      nols1=N_ELEMENTS(obs_list_spec_1)
      FOR i=0,nols1-1 DO BEGIN
        ri=WHERE(STRPOS(STRUPCASE(vn_v[*,0]),obs_list_spec_1[i]) NE -1,rcnt)
        IF (i EQ nols1-1) AND (rcnt NE 0) THEN BEGIN
          ;do check for .MIXING.RATIO. in which case no correction required
          rxi=WHERE(STRPOS(STRUPCASE(vn_v[ri,0]),'.MIXING.RATIO.') EQ -1,rxcnt)
          IF rxcnt NE 0 THEN BEGIN ;some values are just .MIXING.RATIO so correct these only
            rcnt=rxcnt & ri=ri[rxi]
          ENDIF ELSE rcnt=0
        ENDIF
        IF rcnt NE 0 THEN BEGIN ;obsolete values found
          vlen=STRLEN(obs_list_spec_1[i])
          IF i LE 1 THEN BEGIN ;move .AMF and .AVK to DESCRIPTOR section (2 or 3)
            FOR j=0,rcnt-1 DO BEGIN
              vpos=STRPOS(STRUPCASE(vn_v[ri[j],0]),obs_list_spec_1[i])
              ai=WHERE(vn_v[ri[j],*] EQ '',acnt)
              IF acnt NE 0 THEN BEGIN
                ;the DESCRIPTOR field is empty, so can convert value OK (otherwise no change made)
                vn_v[ri[j],0]=STRMID(vn_v[ri[j],0],0,vpos)
                vn_v[ri[j],ai[0]]=STRMID(obs_list_spec_1[i],1) ;write to first empty field
              ENDIF
            ENDFOR
          ENDIF ELSE BEGIN
            ;.CONCENTRATION or .MIXING.RATIO so attempt to identify actual type of measurement
            ;(mixing ratio etc) by looking at corresponding VAR_UNIT values
            mrvi=WHERE((STRPOS(STRLOWCASE(in3[ri]),'pp') NE -1) OR $
                       (STRPOS(STRLOWCASE(in3[ri]),'m-1 sr-1') NE -1),mrvcnt) ;testing for volume mixing ratio
            mrmi=WHERE(STRPOS(STRLOWCASE(in3[ri]),' g ') NE -1,mrmcnt) ;test for mass mixing ratio
            ndi=WHERE(STRPOS(STRLOWCASE(in3[ri]),'mol') NE -1,ndcnt) ;test for number density
            ppi=WHERE(STRPOS(STRLOWCASE(in3[ri]),'pa') NE -1,ppcnt) ;test for partial pressure
            IF i EQ nols1-1 THEN cnti=[mrvcnt,mrmcnt,0,0] ELSE cnti=[mrvcnt,mrmcnt,ndcnt,ppcnt]
            ci=WHERE(cnti GT 0,ccnt)
            IF ccnt EQ 1 THEN BEGIN ;type of measurement successfully found
              ;Note: not set up to handle if more than one type of measurement found in the file
              vlen=STRLEN(obs_list_spec_1[i])
              FOR j=0,rcnt-1 DO BEGIN
                IF (STRUPCASE(vn_v[ri[j],0]) EQ 'AIR.CONCENTRATION') AND (ci[0] EQ 2) THEN $
                  ;Do special case check for AIR.CONCENTRATION - can only change to NUMBER.DENSITY
                  vn_v[ri[j],0]='NUMBER.DENSITY' $
                ELSE BEGIN
                  vpos=STRPOS(STRUPCASE(vn_v[ri[j],0]),obs_list_spec_1[i])
                  vn_v[ri[j],0]=STRMID(vn_v[ri[j],0],0,vpos)+rep_list_conc[ci[0]]+STRMID(vn_v[ri[j],0],vpos+vlen)
                ENDELSE
              ENDFOR
            ENDIF
          ENDELSE
        ENDIF
      ENDFOR
      FOR i=0,N_ELEMENTS(obs_list_2)-1 DO BEGIN ;Check for obsolete mode values
        ri=WHERE(STRUPCASE(vn_v[*,1]) EQ obs_list_2[i],rcnt)
        IF rcnt NE 0 THEN BEGIN
          test1=(i LE 1) AND (STRPOS(STRUPCASE(in4),'FTIR') NE -1)
          test2=(i LE 1) AND (STRPOS(STRUPCASE(in4),'UVVIS') NE -1) AND $
                (STRPOS(STRUPCASE(in4),'DOBSON') EQ -1) AND (STRPOS(STRUPCASE(in4),'BREWER') EQ -1)
          test3=((i EQ 2) OR (i EQ 3)) AND ((STRPOS(STRUPCASE(in4),'MWR') NE -1) OR $
                 (STRPOS(STRUPCASE(in4),'MICROWAVE') NE -1))
          IF i LE 1 THEN mi=STRSPLIT(rep_list_2[i],'|',/EXTRACT)
          ;test1, test2, and test3 are special case obsolete values
          IF test1 THEN vn_v[ri,1]=mi[0] $
          ELSE IF test2 THEN vn_v[ri,1]=mi[1] $
          ELSE IF test3 THEN vn_v[ri,1]=rep_list_2[i] $ ;direct replacement
          ELSE BEGIN ;all other obsolete mode values
            IF i LE 1 THEN vn_v[ri,1]=mi[0] ELSE vn_v[ri,1]=rep_list_2[i] ;direct replacement
            IF i EQ 5 THEN BEGIN ;SLANT (if column measurement then add to main variable name)
              FOR j=0,rcnt-1 DO BEGIN
                IF STRLEN(vn_v[ri[j],0])-STRPOS(vn_v[ri[j],0],'COLUMN') EQ 6 THEN $
                vn_v[ri[j],0]=vn_v[ri[j],0]+'.'+obs_list_2[i]
              ENDFOR
            ENDIF
          ENDELSE
        ENDIF
      ENDFOR
      ;Check conditional miscellaneous changes
      FOR i=0,N_ELEMENTS(misc_list_1)-1 DO BEGIN
        IF i EQ 0 THEN ri=WHERE(STRPOS(STRUPCASE(in2),misc_list_1[i]) NE -1,rcnt) $
        ELSE ri=WHERE(STRUPCASE(in2) EQ misc_list_1[i],rcnt)
        IF rcnt NE 0 THEN BEGIN
          IF i EQ 0 THEN BEGIN ;ensure DATA_SOURCE is BUOY before changing
            IF STRPOS(STRUPCASE(in4),'BUOY') NE -1 THEN BEGIN
              FOR j=0,rcnt-1 DO BEGIN
                vn_v[ri[j],0]=misc_list_2[i] & vn_v[ri[j],1]=''
              ENDFOR
            ENDIF
          ENDIF ELSE BEGIN ;ensure O3.COLUMN is also present as a DATA_VARIABLE before changing
            mi=WHERE(STRUPCASE(vn_v[*,0]) EQ 'O3.COLUMN',mcnt)
            IF mcnt NE 0 THEN BEGIN
              vn_v[ri[0],0]='O3.COLUMN' & vn_v[ri[0],1]='AMF'
            ENDIF
          ENDELSE
        ENDIF
      ENDFOR

      FOR i=0,N_ELEMENTS(obs_list_3)-1 DO BEGIN ;Check for obsolete descriptor values (UNCERTAINTY)
        ;acceptable units for identifying correct replacement name (test1 instruments only)
        vuok=['ppv','ppmv','ppbv','pptv','k','du','molec cm-2','molec cm-3','degc','m-1 sr-1','dimensionless']
        vuok2=['ppv2','ppmv2','ppbv2','pptv2'] ;covariance
        test1=(STRPOS(STRUPCASE(in4),'MWR') NE -1) OR (STRPOS(STRUPCASE(in4),'MICROWAVE') NE -1) OR $
              (STRPOS(STRUPCASE(in4),'HAGAR') NE -1) OR (STRPOS(STRUPCASE(in4),'MIPAS.STR') NE -1) OR $
              (STRPOS(STRUPCASE(in4),'SIOUX') NE -1) OR (STRPOS(STRUPCASE(in4),'UVVIS.AMAXDOAS') NE -1) OR $
              (STRPOS(STRUPCASE(in4),'UVVIS.SAOZ') NE -1) OR (STRPOS(STRUPCASE(in4),'ATMOINSPECTOR') NE -1) OR $
              (STRPOS(STRUPCASE(in4),'FISH') NE -1) OR (STRPOS(STRUPCASE(in4),'AMON') NE -1) OR $
              (STRPOS(STRUPCASE(in4),'LPMA') NE -1) OR (STRPOS(STRUPCASE(in4),'SALOMON') NE -1) OR $
              (STRPOS(STRUPCASE(in4),'RADIOMETER.IR.CIMEL') NE -1) OR (STRPOS(STRUPCASE(in4),'UVVIS') NE -1) OR $
              (STRPOS(STRUPCASE(in4),'SPECTROMETER') NE -1) OR (STRPOS(STRUPCASE(in4),'LIDAR.WATERVAPORRAMAN_UNIVAQ') NE -1)
        ;check for DATA_SOURCE where CoVariance as well as Standard uncertainties are reported
        test2=(STRPOS(STRUPCASE(in4),'FTIR.CO_KIT') NE -1) OR (STRPOS(STRUPCASE(in4),'FTIR.CO_ULG') NE -1) OR $
              (STRPOS(STRUPCASE(in4),'FTIR.HNO3_KIT') NE -1) OR (STRPOS(STRUPCASE(in4),'FTIR.HNO3_ULG') NE -1) OR $
              (STRPOS(STRUPCASE(in4),'FTIR.N2O_KIT') NE -1) OR (STRPOS(STRUPCASE(in4),'FTIR.N2O_ULG') NE -1) OR $
              (STRPOS(STRUPCASE(in4),'FTIR.NO2_KIT') NE -1) OR (STRPOS(STRUPCASE(in4),'FTIR.NO2_ULG') NE -1)
        vnvi=1 ;determines the location index of the descriptor variable
        ri=WHERE(STRUPCASE(vn_v[*,1]) EQ obs_list_3[i],rcnt)
        IF rcnt EQ 0 THEN BEGIN
          vnvi=2 & ri=WHERE(STRUPCASE(vn_v[*,2]) EQ obs_list_3[i],rcnt)
        ENDIF
        IF rcnt NE 0 THEN BEGIN
          IF i LE 3 THEN vn_v[ri,vnvi]=rep_list_3[i] $ ;direct replacement
          ELSE BEGIN ;need to identify whether the UNITS are % or fit vuok criteria
            FOR j=0,rcnt-1 DO BEGIN
              IF STRTRIM(in3[ri[j]],2) EQ '%' THEN vn_v[ri[j],vnvi]=rep_list_3[i]+'.RELATIVE' $
              ELSE BEGIN
                ;test for Standard Uncertainties
                vi=WHERE(STRTRIM(STRLOWCASE(in3[ri[j]]),2) EQ vuok,vcnt)
                IF vcnt EQ 0 THEN $ ;do test for [Prefix]molec cm-2/molec cm-3
                  IF (STRPOS(STRLOWCASE(in3[ri[j]]),'molec cm-2') NE -1) OR $
                     (STRPOS(STRLOWCASE(in3[ri[j]]),'molec cm-3') NE -1) THEN vcnt=1
                IF ((test1) OR (test2)) AND (vcnt NE 0) THEN $ can assume that the given error values are Standard Deviation
                  vn_v[ri[j],vnvi]=rep_list_3[i]
                ;test for Covariance Uncertainties
                vi=WHERE(STRTRIM(STRLOWCASE(in3[ri[j]]),2) EQ vuok2,vcnt)
                IF (test2) AND (vcnt NE 0) THEN vn_v[ri[j],vnvi]=rep_list_4[i]
              ENDELSE
            ENDFOR
          ENDELSE
        ENDIF
      ENDFOR
      ;create any comments for INFOTXT_OUTPUT
      FOR i=0,in1-1 DO BEGIN
        ;put full DATA_VARIABLE names back together
        vn_new[i]=vn_v[i,0]
        FOR j=1,2 DO IF vn_v[i,j] NE '' THEN vn_new[i]=vn_new[i]+'_'+vn_v[i,j]
        ;Check to see if it is different to the original DATA_VARIABLE
        IF vn_new[i] NE in2[i] THEN BEGIN
          vnchange[i]=in2[i] & in2[i]=vn_new[i]
          IF qa_yes THEN itxt=' expected to be ' ELSE itxt=' renamed '
          infotxt='2 Dataset name '+vnchange[i]+itxt+in2[i]
          INFOTXT_OUTPUT,infotxt
        ENDIF
      ENDFOR
    END

  code EQ 6: BEGIN ;Look for obsolete DATA_SOURCE
      ;List of obsolete DATA_SOURCE_01 values
      obs_list=['RO.CHAMP','RO.SAC.C','RO.F3C.FM','RO.CNOFS','RO.GRACE.A','FTIR_', $
                'UVVIS.DOAS_','MICROWAVE.RADIOMETER','LIDAR.BACKSCATTER',$
                'LIDAR.DIAL','LIDAR.OLEX','LIDAR.RIEGL','LIDAR.RMR']
      ;Corresponding replacement list
      rep_list=['CHAMP','SAC.C','F3C.FM','CNOFS','GRACE.A','FTIR.','UVVIS.DOAS.','MWR.','LIDAR.']
      ;Possible list of species for the above instruments (note: gases will take precedence
      ;over Aerosol and Temperature, o/w the first species found will be considered the
      ;primary species measured)
      spc_list=['BrO.','C2H2.','C2H6.','CCl2F2.','CCl3F.','CH4.','CHF2Cl.','CHOCHO.','ClO.','ClONO2.','CO.',$
                'CO2.','COF2.','H2CO.','H2O.','HCFC22.','HCl.','HCN.','HCOOH.','HF.','HNO3.','HONO.','IO.',$
                'N2O.','NO.','NO2.','O3.','OClO.','OCS.','SF6.','SO2.','AEROSOL','TEMPERATURE']

      oli=-1 ;will change to obs_list index value if obsolete DATA_SOURCE_01 found
      n_obsl=N_ELEMENTS(obs_list)
      lid_i=n_obsl-5 ;needs to be start index of the lidar measurements in obs_list
      spc_i=n_obsl-8 ;needs to be start index of the instruments requiring species information in obs_list
      uv_i=n_obsl-7  ;needs to be UVVIS.DOAS index in obs_list
      FOR i=0,N_ELEMENTS(obs_list)-1 DO IF STRPOS(STRUPCASE(in3),obs_list[i]) NE -1 THEN oli=i
      IF oli NE -1 THEN BEGIN ;obsolete DATA_SOURCE value found
        IF oli GE lid_i THEN oli=lid_i ;to correspond with rep_list index for lidar
        pri=-1 ;will change if species found
        IF oli GE spc_i THEN BEGIN ;if not an RO DATA_SOURCE then need to determine species
          n_sl=N_ELEMENTS(spc_list)
          ;Look through VAR_NAME values to try and identify new DATA_SOURCE from species list
          spc_found=INTARR(in1)-1 ;identifies index of found species names in the Datasets
          FOR i=0,n_sl-1 DO BEGIN
            fi=WHERE(STRPOS(STRMID(in2,0,STRLEN(spc_list[i])),spc_list[i]) NE -1,fcnt)
            IF fcnt NE 0 THEN spc_found[fi]=i
          ENDFOR
          ;Check list of found species - don't include Aerosol and Temperature
          fi=WHERE((spc_found NE -1) AND (spc_found LT n_sl-2),fcnt)
          IF fcnt NE 0 THEN pri=spc_found[fi[0]] $ ;primary species index determined
          ELSE IF (oli EQ uv_i) OR (oli EQ lid_i) THEN BEGIN ;Test for Temperature (Lidar Only) and Aerosol
            fi=WHERE(spc_found NE -1,fcnt)
            IF fcnt NE 0 THEN BEGIN
              pri=MIN(spc_found[fi]) ;i.e. Aerosol takes precedence over Temperature
              IF (oli EQ uv_i) AND (pri EQ n_sl-1) THEN pri=-1 ;UVVIS.DOAS can't be Temperature
            ENDIF
          ENDIF
        ENDIF ELSE BEGIN ;cases with straight replacement
          old_ds=STRTRIM(in3,2) & in3=rep_list[oli]+STRMID(old_ds,STRLEN(obs_list[oli])) & pri=-2
        ENDELSE
        IF pri NE -1 THEN BEGIN ;update DATA_SOURCE
          IF pri GE 0 THEN BEGIN
            old_ds=in3
            ;check special cases for LIDAR water vapor measurements
            IF (spc_list[pri] EQ 'H2O.') AND (STRPOS(STRUPCASE(in3),'LIDAR.DIAL') NE -1) THEN $
              in3='WATERVAPORDIAL' $
            ELSE IF (spc_list[pri] EQ 'H2O.') AND (STRPOS(STRUPCASE(in3),'LIDAR.BACKSCATTER') NE -1) THEN $
              in3='WATERVAPORRAMAN' $
            ELSE in3=spc_list[pri]
            IF STRPOS(in3,'.') NE -1 THEN in3=STRMID(in3,0,STRLEN(in3)-1)
            in3=rep_list[oli]+in3+STRMID(old_ds,STRPOS(old_ds,'_'))
          ENDIF
          IF qa_yes THEN itxt=' expected to be ' ELSE itxt=' renamed '
          infotxt='2 DATA_SOURCE value '+old_ds+itxt+in3
          INFOTXT_OUTPUT,infotxt
        ENDIF
      ENDIF
    END

  code EQ 7: BEGIN ;Look for obsolete CALVAL and NDSC FILE_ACCESS values
      obs_list=['CALVAL','NDSC']
      rep_list=['EVDC','NDACC']

      FOR i=0,in1-1 DO BEGIN
        oi=WHERE(in2[i] EQ obs_list,ocnt)
        IF ocnt NE 0 THEN BEGIN
          IF qa_yes THEN itxt=' expected to be ' ELSE itxt=' renamed '
          infotxt='2 FILE_ACCESS value '+in2[i]+itxt+rep_list[oi[0]]
          INFOTXT_OUTPUT,infotxt
          in2[i]=rep_list[oi[0]]
        ENDIF
      ENDFOR
    END

  code EQ 8: BEGIN ;Look for VAR_UNITS=DIMENSIONLESS or NONE and replace with '' or '1'
      vuv=STRUPCASE(in2)
      FOR i=0,in1-1 DO BEGIN
        IF (vuv[i] EQ 'DIMENSIONLESS') OR (vuv[i] EQ 'NONE') THEN BEGIN
          IF in3[i] EQ 'STRING' THEN newv='' ELSE newv='1'
          IF qa_yes THEN itxt=' expected to be ' ELSE itxt=' renamed '
          infotxt='2 VAR_UNITS='+vuv[i]+itxt+'VAR_UNITS='+newv+' based on VAR_DATA_TYPE='+in3[i]
          INFOTXT_OUTPUT,infotxt
          in2[i]=newv
        ENDIF
      ENDFOR
    END

  code EQ 9: BEGIN ;DATA_QUALITY/DATA_TEMPLATE checks
      attr_arr_glob_prov=['DATA_TEMPLATE','DATA_QUALITY']
      geoms_te='' & meta_te='' & std_txt=' Required Metadata Template: '
      ;Check if DATA_TEMPLATE field is present in the TAV file
      dti=WHERE(tab_arr[*,0] EQ attr_arr_glob_prov[0],dtcnt)
      ;Check if DATA_TEMPLATE label is present in the Metadata
      gi=WHERE(in1 EQ attr_arr_glob_prov[0],gcnt)
      IF gcnt EQ 1 THEN IF in2[gi[0]] NE '' THEN meta_te=STRUPCASE(STRTRIM(in2[gi[0]],2))
      IF dtcnt NE 0 THEN tab_arr_sub=tab_arr[dti[0],2:FIX(tab_arr[dti[0],1]+1)] ;Valid Template values from TAV file
      IF dtcnt EQ 0 THEN BEGIN
        infotxt='3'+std_txt+'DATA_TEMPLATE field is not present in the TAV file. Checks cannot be performed'
        INFOTXT_OUTPUT,infotxt
        std_txt=''
      ENDIF ELSE IF meta_te NE '' THEN BEGIN
        ;check if value is present in the TAV file
        ci=WHERE(STRUPCASE(tab_arr_sub) EQ meta_te,ccnt)
        IF ccnt NE 0 THEN BEGIN
          geoms_te=tab_arr_sub[ci[0]] & in2[gi[0]]=geoms_te
          infotxt='0'+std_txt+geoms_te
          INFOTXT_OUTPUT,infotxt
          std_txt=''
        ENDIF
      ENDIF
      IF std_txt NE '' THEN BEGIN ;template value not present in the metadata or value not in the TAV file
        ;Determine DATA_TEMPLATE value based on DATA_SOURCE value (if applicable)
        di=WHERE(in1 EQ 'DATA_SOURCE',dcnt)
        IF dcnt EQ 1 THEN BEGIN ;DATA_SOURCE found
          ;Extract DATA_SOURCE value
          IF in2[di[0]] NE '' THEN BEGIN
            res=STRSPLIT(STRTRIM(in2[di[0]],2),'_',/EXTRACT)
            ;First check - First part of DATA_SOURCE matches text in TAV entry
            res1=STRUPCASE(STRSPLIT(res[0],'.',/EXTRACT,COUNT=res1cnt))
            ci=WHERE(STRPOS(STRUPCASE(tab_arr_sub),res1[0]) NE -1,ccnt)
            IF ccnt EQ 1 THEN geoms_te=tab_arr_sub[ci[0]] $ ;Template found
            ELSE IF (ccnt GT 1) AND (N_ELEMENTS(res1) GT 1) THEN BEGIN
              ;2nd Check - More than one option so try and match 2nd part of DATA_SOURCE with 4th part in TAV entry
              tab_arr_sub=tab_arr_sub[ci]
              tab_arr_part_sub=STRARR(ccnt)
Ian Boyd's avatar
Ian Boyd committed
1762
              numi=-1
Ian Boyd's avatar
Ian Boyd committed
1763
1764
1765
1766
1767
1768
1769
              FOR i=0,ccnt-1 DO BEGIN
                rest=STRSPLIT(tab_arr_sub[i],'-',/EXTRACT)
                rest=[rest,'','',''] ;ensure there are at least 4 sub-values
                tab_arr_part_sub[i]=rest[3]
                IF IS_A_NUMBER_HDF(rest[3]) THEN numi=i
              ENDFOR
              ci=WHERE(STRUPCASE(tab_arr_part_sub) EQ res1[1],ccnt)
Ian Boyd's avatar
Ian Boyd committed
1770
              IF (ccnt EQ 0) AND (numi NE -1) THEN BEGIN
Ian Boyd's avatar
Ian Boyd committed
1771
1772
1773
                ;2nd part is likely a gas e.g. FTIR.CO, so use template where 4th part of TAV entry is a number
                geoms_te=tab_arr_sub[numi]
              ENDIF ELSE IF ccnt EQ 1 THEN geoms_te=tab_arr_sub[ci[0]] $ ;unique template found
Ian Boyd's avatar
Ian Boyd committed
1774
              ELSE IF ccnt GT 1 THEN BEGIN ;ccnt GT 1, Try alternatives for H2O and O3, or test for fourth part
Ian Boyd's avatar
Ian Boyd committed
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
                tasx=tab_arr_sub[ci]
                CASE 1 OF
                  (res1[0] EQ 'UVVIS') AND (res1[1] EQ 'DOAS'): BEGIN
                      IF res1cnt GE 4 THEN BEGIN
                        IF res1[3] EQ 'AEROSOL' THEN res1x=STRUPCASE(res1[2])+'-AEROSOL' $
                        ELSE res1x=STRUPCASE(res1[2])+'-GAS'
                      ENDIF ELSE res1x=res1[1]
                    END
                  res1[1] EQ 'H2O': res1x='WATERVAPOR'
                  res1[1] EQ 'O3': res1x='OZONE'
                  ELSE: res1x=res1[1]
                ENDCASE
                ci=WHERE(STRPOS(STRUPCASE(tasx),res1x) NE -1,ccnt)
                IF ccnt EQ 1 THEN geoms_te=tasx[ci[0]]
Ian Boyd's avatar
Ian Boyd committed
1789
              ENDIF
Ian Boyd's avatar
Ian Boyd committed
1790
1791
1792
            ENDIF
            IF geoms_te NE '' THEN BEGIN ;DATA_TEMPLATE value found based on DATA_SOURCE
              IF gcnt EQ 1 THEN BEGIN ;DATA_TEMPLATE label is present
Ian Boyd's avatar
Ian Boyd committed
1793
1794
1795
1796
1797
1798
1799
1800
1801
                fmvi=WHERE(in1 EQ 'FILE_META_VERSION',fmvcnt)
                fmvtxt=''
                IF fmvcnt EQ 1 THEN BEGIN
                  resfmv=STRSPLIT(in2[fmvi[0]],' ;',/EXTRACT,COUNT=fmvcnt)
                  IF fmvcnt GE 1 THEN BEGIN
                    fmv_ver=resfmv[0]
                    fmvtxt=' and '+fmv_ver+' metadata definitions'
                  ENDIF
                ENDIF
Ian Boyd's avatar
Ian Boyd committed
1802
1803
                in2[gi[0]]=geoms_te ;Add value to DATA_TEMPLATE label
                IF qa_yes THEN itxt=' expected to be ' ELSE itxt=' renamed '
Ian Boyd's avatar
Ian Boyd committed
1804
                infotxt='2 DATA_TEMPLATE value'+itxt+geoms_te+' based on DATA_SOURCE value'+fmvtxt
Ian Boyd's avatar
Ian Boyd committed
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
                INFOTXT_OUTPUT,infotxt
                std_txt=''
              ENDIF ELSE IF gcnt EQ 0 THEN in3[0]=geoms_te ;Need to add new Global Attribute Information
            ENDIF
          ENDIF
        ENDIF
      ENDIF
      IF geoms_te NE '' THEN BEGIN ;Check if DATA_QUALITY label is present
        gi=WHERE(in1 EQ attr_arr_glob_prov[1],gcnt)
        IF gcnt EQ 0 THEN in3[1]='DATA_QUALITY' ;Need to add new Global Attribute Information
      ENDIF
      IF std_txt NE '' THEN BEGIN ;No template file required based on selection criteria
        infotxt='0'+std_txt+'DATA_TEMPLATE value not required based on selection criteria'
        INFOTXT_OUTPUT,infotxt
      ENDIF
    END

  code EQ 10: BEGIN ;check printable characters in Metadata and Datasets (of type String)
      ;Note: Also allowed are Control Characters Null Character (0B), Horizontal Tab (9B),
      ;Line Feed (10B), and Carriage Return (13B) - Note: 10B and 13B for Metadata only
      exempt=['PI_NAME','DO_NAME','DS_NAME','PI_AFFILIATION','DO_AFFILIATION','DS_AFFILIATION',$
              'PI_ADDRESS','DO_ADDRESS','DS_ADDRESS','PI_EMAIL','DO_EMAIL','DS_EMAIL']
      non_char=[27B, 128B, 147B, 176B, 186B, 197B, 216B, 226B, 232B, 233B, 248B, 252B] ;128B = Euro
      rep_char=['',' ','"',' deg. ',' deg. ','A','O','a','e','e','o','u'] ;replace above non-US ASCII chars with these values

      ;Do check for the Full TAV which includes ORIGINATOR attributes
      ochk=STRPOS(tab_arr(*,0),'ORIGINATOR') NE -1
      oi=WHERE(ochk NE 0,ocnt)
      mentry=0  ;Default assumes String Dataset Entry
      n_nchar=N_ELEMENTS(non_char)
      IF N_PARAMS() EQ 3 THEN in2='Dataset '+STRTRIM(in2,2) $ ;Dataset Entry
      ELSE mentry=1 ;Metadata entry
      FOR i=0L,N_ELEMENTS(in1)-1L DO BEGIN
        exempt_att=0 ;default assumes attribute is not exempt from checks
        IF mentry EQ 1 THEN BEGIN
          res=STRSPLIT(in1[i],'=',/EXTRACT)
          in2='Metadata Attribute Entry '+res[0]
          IF ocnt NE 0 THEN BEGIN
            ;Do not need to check 'exempt' attributes if using the full version of the TAV file
            ei=WHERE(STRUPCASE(res[0]) EQ exempt,ecnt)
            IF ecnt NE 0 THEN exempt_att=1 ;invalid entries can be corrected by the code
          ENDIF
        ENDIF

        IF (~qa_yes) AND (exempt_att EQ 0) THEN BEGIN
          ;If not doing QA then check for characters that can be replaced
          non_char_i=[-1]
          FOR j=0,n_nchar-1 DO BEGIN
            testb=BYTE(in1[i])
            ni=WHERE(testb EQ non_char[j],ncnt)
            IF ncnt NE 0 THEN BEGIN
              IF non_char_i[0] EQ -1 THEN non_char_i=[j] $
              ELSE non_char_i=[non_char_i,j]
              inval=in1[i]
              FOR k=0,ncnt-1 DO BEGIN
                IF k NE 0 THEN ni[k]=ni[k]+((STRLEN(rep_char[j])-1)*k)
                inval=STRMID(inval,0,ni[k])+rep_char[j]+STRMID(inval,ni[k]+1)
              ENDFOR
              in1[i]=inval
            ENDIF
          ENDFOR
          IF non_char_i[0] NE -1 THEN BEGIN
            infotxt='2 Illegal character(s) in '+in2+' replaced with valid ISO646-US ASCII character(s): '
            FOR j=0,N_ELEMENTS(non_char_i)-1 DO BEGIN
              IF j EQ 0 THEN infotxt=infotxt+STRTRIM(non_char[non_char_i[j]],2) $
              ELSE infotxt=infotxt+', '+STRTRIM(non_char[non_char_i[j]],2)
            ENDFOR
            INFOTXT_OUTPUT,infotxt
          ENDIF
        ENDIF
        testb=BYTE(in1[i])

        IF exempt_att EQ 0 THEN BEGIN
          IF mentry EQ 1 THEN $
            test=(testb NE 0B) AND (testb NE 9B) AND (testb NE 10B) AND (testb NE 13B) $
          ELSE test=(testb NE 0B) AND (testb NE 9B) ;Check for Dataset Control Characters
          ;check for non-char value and replace
          bi=WHERE(((testb LT 32B) OR (testb GT 126B)) AND (test),bcnt)
          IF bcnt NE 0 THEN BEGIN ;Illegal character present
            IF bcnt GT 1L THEN BEGIN ;check for repeat illegal characters and only display once
              bix=[bi[0]]
              FOR j=1L,bcnt-1L DO BEGIN
                ri=WHERE(testb[bi[j]] EQ testb[bi[0L:j-1L]],rcnt)
                IF rcnt EQ 0 THEN bix=[bix,bi[j]]
              ENDFOR
              bi=bix & bcnt=N_ELEMENTS(bi)
            ENDIF
            infotxt=STRARR(2)
            infotxt[0]='3 Entry must only include valid characters from the ISO646-US ASCII character set.'
            infotxt[1]='    Illegal character(s) in '+in2+' (may not display correctly): '
            FOR j=0L,bcnt-1L DO BEGIN
              cval=STRTRIM(testb[bi[j]],2)
              ;print,testb[bi[j]] ;to output actual byte value
              IF (cval EQ STRTRIM(8B,2)) OR (cval EQ '') THEN BEGIN
                ;Illegal non-printable character so print in Decimal Byte form
                cval=STRING(format='(I4)',testb[bi[j]]) & cval=STRTRIM(cval,2)+'B'
              ENDIF
              IF j NE bcnt-1L THEN itxt=', ' ELSE itxt=''
              infotxt[1]=infotxt[1]+cval+itxt
            ENDFOR
            INFOTXT_OUTPUT,infotxt
          ENDIF
        ENDIF
      ENDFOR
    END
Ian Boyd's avatar
Ian Boyd committed
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
  code EQ 11: BEGIN ;Check DATA_VERSION_NAME part of the DATA_SOURCE
      ;Check that entry is made up of dot-separated alpha_numeric words
      n_char=STRLEN(in1)
      strok=1B ;default is that entry is OK
      test2=0B & test3=0B ;will change to 1 if test conditions are true
      charchkm1=0B ;to check test character against the previous character
      FOR i=0,n_char-1 DO BEGIN
        charchk=BYTE(STRMID(in1,i,1))
        test1=(charchk EQ 46B) OR ((charchk GE 65B) AND (charchk LE 90B)) OR $ ;'.' or A-Z
              ((charchk GE 97B) AND (charchk LE 122B)) OR $ ;a-z
              ((charchk GE 48B) AND (charchk LE 57B)) ;0-9
        IF (i EQ 0) OR (i EQ n_char-1) THEN test2=charchk EQ 46B ;i.e. first or last character is a '.'
        test3=(charchk EQ 46B) AND (charchkm1 EQ 46B) ;i.e. two consecutive dots
        IF (~test1) OR (test2) OR (test3) THEN strok=0B
        charchkm1=charchk 
      ENDFOR
      IF ~strok THEN BEGIN
        ;write error message
        infotxt='3 VERSION_NAME sub-value of the DATA_SOURCE must consist of dot-separated alpha-numeric words: '
        infotxt=infotxt+in1
        INFOTXT_OUTPUT,infotxt
      ENDIF
    END  
Ian Boyd's avatar
Ian Boyd committed
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
ENDCASE

END ;Procedure GEOMS_Rule_Changes



PRO pre_defined_att_checks, ct, nav, vav, vnx, vdtv,verror
;Procedure to perform checks on the HDF4 pre-defined attributes, as well as check that
;the numeric variable attribute values are of the same data type as that given by the
;corresponding VAR_DATA_TYPE value (if input is via session memory)
; ----------
;Written by Ian Boyd for the AVDC - iboyd@astro.umass.edu
;
;  History:
;    20101120: Introduced - Version 4.0
;
;  Inputs: ct - Identifies the type of check to be performed on the inputs
;          nav - The HDF pre-defined attribute to check
;          vav - The Variable Attribute corresponding to the pre-defined attribute
;          vnx - The Dataset from which the attributes are derived
;          vdtv - The Data Type of the Dataset
;          verror - Boolean to indicate whether the Variable Attribute data type error has
;                   already been written
;
;  Outputs: None - Any Pre-defined attributes written to an HDF file are recreated by the program
;
;  Called by: IDLCR8HDF, READ_METADATA
;
<