;Main Program Version: geoms_tools.pro v2.17, 20241126 ; Written by Ian Boyd for AVDC/EVDC - iboyd@bryanscientific.org ; FUNCTION is_a_number_gt,value ON_IOERROR, ConversionError IF STRTRIM(value,2) EQ '' THEN RETURN, 0B n=DOUBLE(value) return,1B ConversionError: return, 0B END PRO geoms_version_check, geoms_iteration, ga, newdfv, nodfvinc ;Procedure to determine a new value for the DATA_FILE_VERSION if a new HDF file is to be ;created when performing Harmonization ; ---------- ;Written by Ian Boyd for AVDC/EVDC - iboyd@bryanscientific.org ; ;History: ; 20110113: Introduced to geoms_tools - Version 0.4 ; 20161230: Add nodfvinc switch (0/1). If 1 then forces the DATA_FILE_VERSION not to ; increment - Version 1.59 ; ;Inputs: geoms_iteration - determines which rule to apply when determining the new ; DATA_FILE_VALUE ; ga - a string array holding the Global Attributes read in using idlcr8ascii ; nodfvinc - switch to force DATA_FILE_VERSION not to increment ; ;Outputs: newdfv - A 3-element string array containing the index number of the ga ; array holding the DATA_FILE_VERSION information, the new ; DATA_FILE_VERSION label and value, and a value indicating whether the ; DATA_FILE_VERSION is absent from the original file or is non-conforming ; (0). Default value is (1) ; ga - If there is no DATA_FILE_VERSION entry, this will be appended to the array ; ;Called by: GEOMS_TOOLS ; ;Subroutines Called: None CASE 1 OF geoms_iteration EQ 1: BEGIN ;Checks for existence of the DATA_STOP_DATE in the filename and FILE_DOI in the global ;attributes. If both are present then the file has passed initial GEOMS compliance and ;the DATA_FILE_VERSION will be incremented, o/w the DATA_FILE_VERSION will be reset to 001 newdfv=STRARR(3) ;initialize newdfv information dfv='DATA_FILE_VERSION=001' & dfvok='1' ;default values for 2nd and 3rd elements of the array valok=1 ;boolean to indicate that DFV value conforms to current standards i.e. 'nnn' gax=STRUPCASE(STRCOMPRESS(ga,/REMOVE_ALL)) si=WHERE(STRMID(gax,0,15) EQ 'DATA_STOP_DATE=',scnt) di=WHERE(STRMID(gax,0,9) EQ 'FILE_DOI=',dcnt) vi=WHERE(STRMID(gax,0,18) EQ 'DATA_FILE_VERSION=',vcnt) fi=WHERE(STRMID(gax,0,10) EQ 'FILE_NAME=',fcnt) IF vcnt EQ 1 THEN BEGIN ;Do check to see if DATA_FILE_VERSION is currently compliant i.e. 'nnn' res=STRSPLIT(gax[vi[0]],'=',/EXTRACT,COUNT=xcnt) IF xcnt EQ 2 THEN BEGIN dfval=res[1] vlen=STRLEN(dfval) IF vlen NE 3 THEN valok=0 $ ELSE BEGIN FOR i=0,vlen-1 DO BEGIN bchar=BYTE(STRMID(dfval,i,1)) ; IF (bchar LT 48) OR (bchar GT 57) THEN valok=0 ;i.e. non-numeric character ENDFOR ENDELSE ENDIF ELSE valok=0 ENDIF ELSE valok=0 IF (scnt EQ 1) AND (dcnt EQ 1) AND (vcnt EQ 1) AND (fcnt EQ 1) THEN BEGIN ;Extract DATA_STOP_DATE and FILE_NAME values res=STRSPLIT(gax[si[0]],'=',/EXTRACT,COUNT=rcnt) IF rcnt EQ 2 THEN dsd_val=res[1] ELSE dsd_val='' res=STRSPLIT(gax[fi[0]],'=',/EXTRACT,COUNT=rcnt) IF rcnt EQ 2 THEN fn_val=res[1] ELSE fn_val='N/A' ;Is the DATA_STOP_DATE present in the file name? IF STRPOS(fn_val,dsd_val) NE -1 THEN BEGIN ;Yes, so increment DATA_FILE_VALUE IF (xcnt EQ 2) AND (valok EQ 1) THEN BEGIN ;DATA_FILE_VERSION stays as the default if value is not a number or is > 999 IF nodfvinc EQ 1 THEN newval=LONG(dfval) ELSE newval=LONG(dfval)+1L IF newval LE 999L THEN BEGIN IF newval LT 10 THEN nvS='00'+STRTRIM(newval,2) $ ELSE IF newval LT 100 THEN nvS='0'+STRTRIM(newval,2) $ ELSE nvS=STRTRIM(newval,2) dfv='DATA_FILE_VERSION='+nvS ENDIF ENDIF ENDIF ;otherwise don't need to do anything as default values will be used for newdfv ENDIF IF vcnt EQ 0 THEN BEGIN ;no DATA_FILE_VERSION entry so can append to ga array vi=[STRTRIM(N_ELEMENTS(ga),2)] & dfvok='0' ga=[ga,dfv] ENDIF ELSE IF valok EQ 0 THEN BEGIN ;non-conforming DATA_FILE_VERSION ga[vi[0]]=dfv & dfvok='0' ENDIF newdfv=[STRTRIM(vi[0],2),dfv,dfvok] END ENDCASE END PRO geoms_one_off_check,tav,ga,one_off_fix ;Procedure to check the TAV file for individual corrections required for harmonization. Fields start ;with the title 'GEOMS_HARMON'. ; ---------- ;Written by Ian Boyd for AVDC/EVDC - iboyd@bryanscientific.org ; ;History: ; 20110215: Introduced to geoms_tools - Version 0.5 ; 20140909: Allow corrections to be made to [PI][DO][DS]_AFFILIATION values based on the ; GEOMS_HARMON_DATA_SOURCE values - Version 1.40 ; ; ;Inputs: tav - The file containing Table Attribute Values ; ga - a string array holding the Global Attributes read in using idlcr8ascii ; ;Outputs: ga - changes may be made to some Global Attribute entries ; one_off_fix - A string array containing comments on any changes made ; ;Called by: GEOMS_TOOLS ; ;Subroutines Called: None ;Check to see if TAV file contains GEOMS_HARMON fields dum='' & gcnt=0 & one_off_fix=[''] & acnt=0L ;list of global attributes that require exact matches exact_list=['PI_NAME','DO_NAME','DS_NAME','DATA_GROUP','DATA_LOCATION','DATA_SOURCE'] OPENR,tu,tav,/GET_LUN WHILE NOT EOF(tu) DO BEGIN READF,tu,dum & dum=STRTRIM(STRUPCASE(dum),2) IF dum EQ 'AFFILIATION' THEN BEGIN ;count number of affiliation values REPEAT BEGIN READF,tu,dum & dum=STRTRIM(dum,2) acnt++ ENDREP UNTIL (EOF(tu)) OR (STRMID(dum,0,1) NE '=') IF STRMID(dum,0,1) NE '=' THEN acnt-- tavaf=STRARR(acnt) ENDIF IF STRPOS(dum,'GEOMS_HARMON_') EQ 0 THEN BEGIN IF dum EQ 'GEOMS_HARMON_NAME' THEN gcnt=gcnt+3 $ ELSE IF dum EQ 'GEOMS_HARMON_DATA_SOURCE' THEN gcnt=gcnt+4 $ ELSE gcnt=gcnt+1 ENDIF ENDWHILE FREE_LUN,tu IF gcnt NE 0 THEN BEGIN ;TAV file contains GEOMS_HARMON fields dum='' IF acnt NE 0L THEN BEGIN ;Read in AFFILIATION values OPENR,tu,tav,/GET_LUN WHILE dum NE 'AFFILIATION' DO BEGIN READF,tu,dum & dum=STRTRIM(STRUPCASE(dum),2) ENDWHILE READF,tu,tavaf tavaf=STRMID(STRTRIM(tavaf,2),1) ;strip the '=' from the value FREE_LUN,tu ENDIF ;Determine maximum number of entries for the GEOMS_HARMON fields to set array size vcnt=INTARR(gcnt) ;array to hold the number of entries in each field dum='' & gcnt=0 OPENR,tu,tav,/GET_LUN WHILE gcnt LT N_ELEMENTS(vcnt) DO BEGIN WHILE STRPOS(STRUPCASE(dum),'GEOMS_HARMON_') NE 0 DO READF,tu,dum dum=STRTRIM(STRUPCASE(dum),2) IF dum EQ 'GEOMS_HARMON_NAME' THEN ginc=3 $ ELSE IF dum EQ 'GEOMS_HARMON_DATA_SOURCE' THEN ginc=4 $ ELSE ginc=1 ecnt=0 REPEAT BEGIN READF,tu,dum & dum=STRMID(STRTRIM(dum,2),0,1) ecnt++ ENDREP UNTIL (dum NE '=') OR (EOF(tu)) IF dum NE '=' THEN ecnt-- vcnt[gcnt]=ecnt gcnt=gcnt+ginc ENDWHILE FREE_LUN,tu ;Now read Field name and entries into string array mvcnt=MAX(vcnt) ;Maximum number of values in a field gh_arr=STRARR(gcnt,2,mvcnt+1) dum='' & gcnt=0 OPENR,tu,tav,/GET_LUN WHILE gcnt LT N_ELEMENTS(vcnt) DO BEGIN WHILE STRPOS(STRUPCASE(dum),'GEOMS_HARMON_') NE 0 DO READF,tu,dum ginc=1 ;Strip GEOMS_HARMON from field name and adds to the array gh_arr[gcnt,1,0]=STRMID(STRTRIM(STRUPCASE(dum),2),13) IF gh_arr[gcnt,1,0] EQ 'NAME' THEN BEGIN gh_arr[gcnt,1,0]='PI_NAME' & gh_arr[gcnt+1,1,0]='DO_NAME' gh_arr[gcnt+2,1,0]='DS_NAME' & ginc=3 ENDIF IF gh_arr[gcnt,1,0] EQ 'DATA_SOURCE' THEN BEGIN gh_arr[gcnt,1,0]='DATA_SOURCE' & gh_arr[gcnt+1,1,0]='PI_AFFILIATION' gh_arr[gcnt+2,1,0]='DO_AFFILIATION' & gh_arr[gcnt+3,1,0]='DS_AFFILIATION' & ginc=4 ENDIF ecnt=0 REPEAT BEGIN READF,tu,dum & dumx=STRMID(STRTRIM(dum,2),0,1) IF dumx EQ '=' THEN BEGIN ecnt=ecnt+1 IF STRPOS(dum,'[') NE -1 THEN BEGIN ;need to extract values between [...] rcnt=2 & res=STRARR(rcnt) bpos1=STRPOS(dum,'[') & bpos3=STRPOS(dum,'[',/REVERSE_SEARCH) bpos2=STRPOS(dum,']') & bpos4=STRPOS(dum,']',/REVERSE_SEARCH) bx=STRMID(dum,bpos1+1,(bpos2-bpos1)-1) & res[0]=bx bx=STRMID(dum,bpos3+1,(bpos4-bpos3)-1) & res[1]=bx ENDIF ELSE res=STRSPLIT(dum,'=;',/EXTRACT,COUNT=rcnt) FOR i=0,1 DO BEGIN IF i LT rcnt THEN FOR j=0,ginc-1 DO gh_arr[gcnt+j,i,ecnt]=STRTRIM(res[i],2) ENDFOR ENDIF ENDREP UNTIL (dumx NE '=') OR (EOF(tu)) gcnt=gcnt+ginc ENDWHILE FREE_LUN,tu ;Do checks against Global Attribute Entries FOR i=0,N_ELEMENTS(ga)-1 DO BEGIN res=STRSPLIT(ga[i],'=',/EXTRACT,COUNT=rcnt) res=STRTRIM(res,2) & res[0]=STRUPCASE(res[0]) ci=WHERE(res[0] EQ gh_arr[*,1,0],ccnt) ;Check Attribute Label reshold=res ;so that the Global Array value can be manipulated ;(res[1] will change for DATA_SOURCE) IF (res[0] EQ 'DATA_GROUP') AND (ccnt NE 0) THEN BEGIN ;Only make correction if DATA_SOURCE is FTIR dsv='' & lc=0 & ccnt=0 WHILE (dsv EQ '') AND (lc LE N_ELEMENTS(ga)-1) DO BEGIN dsres=STRSPLIT(ga[lc],'=',/EXTRACT,COUNT=dscnt) IF STRTRIM(STRUPCASE(dsres[0]),2) EQ 'DATA_SOURCE' THEN BEGIN dsv=dsres[0] IF dscnt GT 1 THEN IF STRPOS(dsres[1],'FTIR') NE -1 THEN ccnt=1 ENDIF lc=lc+1 ENDWHILE ENDIF IF (res[0] EQ 'DATA_SOURCE') AND (rcnt GT 1) THEN BEGIN ;separate out AFFILIATION and test that only dsres=STRSPLIT(res[1],'_',/EXTRACT,COUNT=dscnt) IF dscnt EQ 2 THEN BEGIN dslen=STRLEN(dsres[1]) IF dslen GT 3 THEN reshold[1]=STRMID(dsres[1],0,dslen-3) ENDIF ENDIF IF (ccnt NE 0) AND (rcnt GT 1) THEN BEGIN ;Check attribute values xi=WHERE(res[0] EQ exact_list,xcnt) ;to identify the type of check to carry out mf=[-1] FOR j=1,mvcnt DO BEGIN IF gh_arr[ci[0],0,j] NE '' THEN BEGIN IF xcnt NE 0 THEN BEGIN ;Look for exact match only IF STRCOMPRESS(STRUPCASE(reshold[1]),/REMOVE_ALL) EQ $ STRUPCASE(STRCOMPRESS(gh_arr[ci[0],0,j],/REMOVE_ALL)) THEN mf[0]=j ENDIF ELSE BEGIN ;Look for sub-value within the value IF STRPOS(STRUPCASE(reshold[1]),STRUPCASE(gh_arr[ci[0],0,j])) NE -1 THEN $ IF mf[0] EQ -1 THEN mf[0]=j ELSE mf=[mf,j] ENDELSE ENDIF ENDFOR IF mf[0] NE -1 THEN BEGIN ;Need to change Global Attribute Value IF (xcnt NE 0) AND (res[0] NE 'DATA_SOURCE') THEN BEGIN ;straight replacement res[1]=gh_arr[ci[0],1,mf[0]] & nvfound=1 ENDIF ELSE BEGIN ;may need to insert correct value into the full value IF STRPOS(res[0],'_AFFILIATION') NE -1 THEN BEGIN ;need to find full affiliation from the available TAV values aacnt=0L & nvfound=0 WHILE (aacnt LE acnt-1L) AND (nvfound EQ 0) DO BEGIN tres=STRSPLIT(tavaf[aacnt],';',/EXTRACT,COUNT=trcnt) IF trcnt EQ 2 THEN BEGIN IF STRUPCASE(tres[1]) EQ STRUPCASE(gh_arr[ci[0],1,mf[0]]) THEN BEGIN nvfound=1 & res[1]=tavaf[aacnt] ENDIF ENDIF aacnt++ ENDWHILE ENDIF ELSE BEGIN FOR j=0,N_ELEMENTS(mf)-1 DO BEGIN tpos=STRPOS(res[1],STRUPCASE(gh_arr[ci[0],0,mf[j]])) tlen=STRLEN(gh_arr[ci[0],0,mf[j]]) res[1]=STRMID(res[1],0,tpos)+gh_arr[ci[0],1,mf[j]]+STRMID(res[1],tpos+tlen) ENDFOR nvfound=1 ENDELSE ENDELSE IF nvfound EQ 1 THEN BEGIN infotxt=' INFORMATION: '+ga[i]+' renamed '+res[0]+'='+res[1] PRINT,infotxt ga[i]=res[0]+'='+res[1] IF one_off_fix[0] EQ '' THEN one_off_fix[0]=infotxt $ ELSE one_off_fix=[one_off_fix,infotxt] ENDIF ENDIF ENDIF ENDFOR ENDIF END ;Procedure GEOMS_One_Off_Check PRO read_tav_dates, tavdir, min_ver, tavall, tavmjd2k, tavver ;Procedure to create arrays of all available TAV files and their creation dates ; ---------- ;Written by Ian Boyd for AVDC/EVDC - iboyd@bryanscientific.org ; ;History: ; 20201122: Introduced to geoms_tools - Version 2.08 ; 20201211: Add tavver; include search for '_idl' versions of the TAV files and ; make searches case insensitive; ensure TAV files are saved in ; chronological order - Version 2.09 ; 20220808: Initialise tavall, tavmjd2k, tavver and ensure they are only ; updated when TAV files are found - Version 2.11 ; ;Inputs: tavdir - Directory containing all the TAV files ; min_ver - Minimum version numner for valid TAV files ; ;Output: tavall - an array containing the fully qualified TAV file names ; tavmjd2k - an array containing the TAV file creation dates in MJD2K form ; tavver - an array containing the TAV version values ; ;Called by: TAV_Files_For_QA ; ;Subroutines Called: None ; pathspec=['tableattrvalue_??R???.dat','tableattrvalue_??R???_idl.dat'] FOR i=0,N_ELEMENTS(pathspec)-1 DO pathspec[i]=tavdir+pathspec[i] tavall=FILE_SEARCH(pathspec,FOLD_CASE=1,/FULLY_QUALIFY_PATH,/TEST_READ,COUNT=tcnt) tavmjd2k=[-90000.d] ;default TAVMJD2K array tavver=[''] ;default TAV version array dum='' IF tcnt NE 0 THEN BEGIN tavver=STRARR(tcnt) & tav_vr_vals=INTARR(2)-1 FOR i=0,tcnt-1 DO BEGIN tavname=FILE_BASENAME(tavall[i]) tav_chk=STRSPLIT(tavname,'_.',/EXTRACT,COUNT=rcnt) ;Determine version and revision values IF STRLEN(tav_chk[1]) EQ 6 THEN BEGIN tav_vr=[STRMID(tav_chk[1],0,2),STRMID(tav_chk[1],3)] FOR fi=0,1 DO BEGIN IF IS_A_NUMBER_GT(tav_vr[fi]) THEN tav_vr_vals[fi]=FIX(tav_vr[fi]) ENDFOR ;Check for valid TAV version and revision IF (tav_vr_vals[0] GE min_ver) AND (tav_vr_vals[1] GE 0) THEN tavver[i]=tav_chk[1] ENDIF ENDFOR ;Create list of all valid TAV files gi=WHERE(tavver NE '',gcnt) IF gcnt NE 0 THEN BEGIN tavall=tavall[gi] tavver=tavver[gi] tavmjd2k=DBLARR(gcnt) FOR i=0,gcnt-1 DO BEGIN OPENR,tu,tavall[i],/GET_LUN REPEAT READF,tu,dum UNTIL STRPOS(STRUPCASE(dum),'BASED') NE -1 FREE_LUN,tu res=STRSPLIT(dum,' ',/EXTRACT,COUNT=rcnt) IF rcnt GE 2 THEN BEGIN tymd=STRSPLIT(res[rcnt-2],'-',/EXTRACT,COUNT=tycnt) thms=STRSPLIT(res[rcnt-1],':',/EXTRACT,COUNT=thcnt) IF (tycnt EQ 3) AND (thcnt EQ 3) THEN BEGIN n_chk=1B FOR j=0,2 DO IF ~IS_A_NUMBER_GT(tymd[j]) THEN n_chk=0B FOR j=0,2 DO IF ~IS_A_NUMBER_GT(thms[j]) THEN n_chk=0B tiso=tymd[0]+tymd[1]+tymd[2]+'T'+thms[0]+thms[1]+thms[2]+'Z' IF n_chk THEN BEGIN mjdcalc=JULIAN_DATE(tiso,/I,/M) tavmjd2k[i]=mjdcalc ENDIF ENDIF ENDIF ENDFOR ;Sort tav files into chronological order (in case there are both full and _idl versions of the files) tmjdi=SORT(tavmjd2k) tavmjd2k=tavmjd2k[tmjdi] & tavall=tavall[tmjdi] & tavver=tavver[tmjdi] ENDIF ENDIF END PRO tav_files_for_qa, tav, ga, lu, tavlist, gf_err, cur_tav ;Procedure to determine TAV file versions needed for definition checks when doing QA, ;based on FILE_GENERATION_DATE (FGD) and FILE_META_VERSION (FMV) values ; ---------- ;Written by Ian Boyd for AVDC/EVDC - iboyd@bryanscientific.org ; ;History: ; 20201122: Introduced to geoms_tools - Version 2.08 ; 20201211: Include search for '_idl' versions of the TAV files and make searches ; case insensitive - Version 2.09 ; ;Inputs: tav - TAV file used in call to geoms_qa ; ga - Global attributes read in by idlcr8ascii ; lu - File unit for writing messages to file ; ;Output: tavlist - an array containing the file names of the TAV files needed for testing. ; The first index is the TAV determined from FGD if determined, or the input TAV ; The second index is the TAV determined from FMV, if different to the first TAV ; gf_err - returns 1B if an error occurs when determining TAV file versions ; cur_tav = reutrns 1B if using the input TAV rather than the one determined by FGD ; ;Called by: GEOMS_TOOLS ; ;Subroutines Called: Read_TAV_Dates ; tavlist=STRARR(2) tavdir=FILE_DIRNAME(tav,/MARK_DIRECTORY) min_ver=4 ;minimum allowable TAV version ;Read in all the TAV filenames and definition creation dates READ_TAV_DATES,tavdir,min_ver,tavall,tavmjd2k,tavver ;Identify input TAV file version and revision number tav_vr_vals=INTARR(2)-1 tavfile=FILE_BASENAME(tav) res=STRSPLIT(tavfile,'_.',/EXTRACT,COUNT=rcnt) IF rcnt GE 2 THEN BEGIN tav_ver=STRTRIM(STRUPCASE(res[1]),2) IF STRLEN(tav_ver) EQ 6 THEN BEGIN tav_vr=[STRMID(tav_ver,0,2),STRMID(tav_ver,3)] FOR fi=0,1 DO BEGIN IF IS_A_NUMBER_GT(tav_vr[fi]) THEN tav_vr_vals[fi]=FIX(tav_vr[fi]) ENDFOR ENDIF ENDIF ;Check FILE_GENERATION_DATE and FILE_META_VERSION to identify correct TAV ;file to check metadata against gf_chk=['FILE_GENERATION_DATE','FILE_META_VERSION'] n_spl=[1,2] ;expected number of (sub-)values in the attribute FOR i=0,1 DO BEGIN gf_tav='' gf_vr_vals=INTARR(2)-1 invalid_gf=1B & test2=0B gfi=WHERE(STRMID(ga,0,STRLEN(gf_chk[i])) EQ gf_chk[i],gfcnt) IF gfcnt EQ 1 THEN BEGIN res=STRSPLIT(ga[gfi[0]],' =;',/EXTRACT,COUNT=rcnt) IF rcnt EQ n_spl[i]+1 THEN BEGIN gf_val=STRTRIM(res[1],2) IF i EQ 0 THEN BEGIN gf_valh=gf_val fgd_chk=JULIAN_DATE(gf_valh,/I,/M) systmjd2k=SYSTIME(/JULIAN,/UTC)-2451544.5D + 1.D ;current MJD2K + 1 day IF (fgd_chk GE 0.D) AND (fgd_chk LE systmjd2k) THEN BEGIN invalid_gf=0B di=WHERE(fgd_chk GE tavmjd2k,dcnt) IF dcnt NE 0 THEN BEGIN gf_tav=tavall[di[dcnt-1]] test2=1B ;version and revision values already checked in READ_TAV_DATES ENDIF ENDIF ENDIF ELSE BEGIN invalid_gf=0B IF STRLEN(gf_val) EQ 6 THEN BEGIN ;Check version and revision values ;gf_vr=[STRMID(gf_val,0,2),STRMID(gf_val,3)] ;FOR fi=0,1 DO BEGIN ; IF IS_A_NUMBER_GT(gf_vr[fi]) THEN gf_vr_vals[fi]=FIX(gf_vr[fi]) ;ENDFOR ;test2=(gf_vr_vals[0] GE min_ver) AND (gf_vr_vals[1] GE 0) test2=1B ;test is redundant because tavver version and revision values already checked gi=WHERE(gf_val EQ tavver,gcnt) ;case sensitive check IF gcnt EQ 0 THEN $ gi=WHERE(STRUPCASE(gf_val) EQ STRUPCASE(tavver),gcnt) ;case insensitive check IF gcnt NE 0 THEN gf_tav=tavall[gi[0]] ENDIF ENDELSE ENDIF ENDIF test1=gf_tav NE '' ;i.e. tableattrvalue file identified is available IF (test1) AND (test2) THEN tavlist[i]=gf_tav $ ELSE IF (invalid_gf) OR (~test1) OR (~test2) THEN BEGIN IF i EQ 0 THEN BEGIN itxtfgd='. Metadata checks based on '+tav_ver+' definitions' tavlist[i]=tav & cur_tav=1B ENDIF ELSE itxtfgd='' PRINTF,lu,'' IF invalid_gf THEN BEGIN IF i EQ 0 THEN BEGIN PRINTF,lu,'ERROR: TAV file version containing metadata definitions cannot be identified due to missing or' PRINTF,lu,' invalid '+gf_chk[i]+' global attribute'+itxtfgd ENDIF ELSE BEGIN PRINTF,lu,'ERROR: TAV file version cannot be identified due to missing or invalid '+gf_chk[i]+' global attribute' ENDELSE ENDIF ELSE IF ~test1 THEN BEGIN IF i EQ 0 THEN itxt='corresponding to' ELSE itxt='matching' PRINTF,lu,'ERROR: TAV file '+itxt+' the '+gf_chk[i]+' entry ('+gf_val+') not found'+itxtfgd ENDIF ELSE BEGIN ;Can only occur with FILE_META_VERSION value IF min_ver LT 10 THEN mvs='0'+STRTRIM(min_ver,2) ELSE mvs=STRTRIM(min_ver,2) PRINTF,lu,'ERROR: '+gf_chk[i]+' metadata definitions must be version '+mvs+ $ ' or greater' ENDELSE gf_err=1B ENDIF ENDFOR IF STRUPCASE(tavlist[0]) EQ STRUPCASE(tavlist[1]) THEN tavlist[1]='' END ;Procedure tav_files_for_QA PRO ppv_check,sds,dvcnt,n_lab,template_change ;Procedure to convert ppv units to ppmv/ppbv/pptv ; ---------- ;Written by Ian Boyd for AVDC/EVDC - iboyd@bryanscientific.org ; ;History: ; 20131023: Introduced to geoms_tools - Version 1.3 ; 20131206: Improved coding for determining new VAR_VALID_MIN, VAR_VALID_MAX, and ; VAR_FILL_VALUES after changing units; include '1' as a possible ; VAR_UNIT for UNCERTAINTIES (done for some FTIR covariance measurements); ; Add check for Valid Pointers - Version 1.33 ; ;Inputs: sds - a structure containing the Metadata and Data read in using idlcr8ascii ; dvcnt - an integer indicating the number of DATA_VARIABLES reported in the file ; n_lab - number of metadata labels in the datasets ; ;Outputs: sds - changes may be made to some of the Metadata and Data entries ; template_change - A string array containing comments on any changes made ; ;Called by: GEOMS_TOOLS ; ;Subroutines Called: None newvu=['ppmv','ppbv','pptv','ppmv2','ppbv2','pptv2'] newvsc=['1.0E-6','1.0E-9','1.0E-12','1.0E-12','1.0E-18','1.0E-24'] newmult=[1.0E6,1.0E9,1.0E12,1.0E12,1.0E18,1.0E24] new_search=1 ;Boolean to try and link UNCERTAINTY units to the source dataset units FOR i=0,dvcnt-1 DO BEGIN sds_lab=STRARR(n_lab) FOR j=0,n_lab-1 DO $ IF PTR_VALID(sds[i,j].va_l) THEN sds_lab[j]=STRTRIM(STRUPCASE(*sds[i,j].va_l),2) ;Find VAR_UNITS value vui=WHERE(sds_lab EQ 'VAR_UNITS',vucnt) vu2i=WHERE(sds_lab EQ 'UNITS',vu2cnt) ;test for standard units entry IF vucnt EQ 0 THEN BEGIN ;use standard entry if that is all that is available vui=vu2i & vucnt=vu2cnt ENDIF ;Find VAR_FILL_VALUE vfi=WHERE(sds_lab EQ 'VAR_FILL_VALUE',vfcnt) vf2i=WHERE(sds_lab EQ '_FILLVALUE',vf2cnt) ;test for standard fill value entry IF vfcnt EQ 0 THEN BEGIN ;use standard entry if that is all that is available vfi=vfi2 & vfcnt=vf2cnt ENDIF IF vfcnt NE 0 THEN vf_val=*sds[i,vfi[0]].va_v ;Find VAR_NAME value vni=WHERE(sds_lab EQ 'VAR_NAME',vncnt) IF vncnt EQ 1 THEN vn_val=STRUPCASE(*sds[i,vni[0]].va_v) ELSE vn_val='' ;Find VAR_SI_CONVERSION index vsi=WHERE(sds_lab EQ 'VAR_SI_CONVERSION',vscnt) ;Find VAR_VALID_MIN/MAX and valid_range mini=WHERE(sds_lab EQ 'VAR_VALID_MIN',mincnt) maxi=WHERE(sds_lab EQ 'VAR_VALID_MAX',maxcnt) vri=WHERE(sds_lab EQ 'VALID_RANGE',vrcnt) ;Find VAR_DESCRIPTION vdi=WHERE(sds_lab EQ 'VAR_DESCRIPTION',vdcnt) ;Determine if it is an Uncertainty dataset or not un_val=STRPOS(vn_val,'UNCERTAINTY') NE -1 IF vucnt EQ 1 THEN BEGIN vu_val=STRTRIM(STRLOWCASE(*sds[i,vui[0]].va_v),2) IF (vu_val EQ 'ppv') OR (vu_val EQ 'ppv2') OR ((vu_val EQ '1') AND (un_val)) THEN BEGIN sds_data=*sds[i,0].data IF un_val EQ 0 THEN BEGIN new_search=1 & nvui=-1 ;need to calculate units (default ppmv) ENDIF ELSE IF (new_search EQ 1) AND (nvui NE -1) THEN new_search=0 ;use already calculated units IF new_search EQ 1 THEN BEGIN ;determine the maximum data value and base new unit on it maxv=MAX(sds_data) & good_data=1 IF vfcnt NE 0 THEN BEGIN gi=WHERE(sds_data NE vf_val[0],gcnt) IF gcnt NE 0 THEN maxv=MAX(sds_data[gi]) ELSE good_data=0 ENDIF IF vu_val EQ 'ppv' THEN nvui=0 ELSE nvui=3 ;default to ppmv or ppmv2 IF (good_data EQ 1) AND (maxv GT 0.) AND (maxv LT 1.) THEN BEGIN exv=0 WHILE maxv LT 1. DO BEGIN maxv=maxv*10. & exv++ ENDWHILE CASE 1 OF exv LE 8: IF vu_val EQ 'ppv' THEN nvui=0 ELSE nvui=3 exv LE 11: IF vu_val EQ 'ppv' THEN nvui=1 ELSE nvui=4 ELSE: IF vu_val EQ 'ppv' THEN nvui=2 ELSE nvui=5 ENDCASE ENDIF ENDIF ELSE BEGIN ;use previously calculated units IF (nvui LE 2) AND ((vu_val EQ 'ppv2') OR (vu_val EQ '1')) THEN nvui=nvui+3 $ ELSE IF (nvui GT 2) AND (vu_val EQ 'ppv') THEN nvui=nvui-3 ENDELSE ;Make corrections to VAR_VALID_MIN/MAX, valid_range, VAR_UNITS, units, ;VAR_SI_CONVERSION, VAR_DESCRIPTION and data sds[i,vui[0]].va_v=PTR_NEW(newvu[nvui]) IF vu2cnt EQ 1 THEN sds[i,vu2i[0]].va_v=PTR_NEW(newvu[nvui]) new_data=sds_data IF mincnt EQ 1 THEN vmin_val=*sds[i,mini[0]].va_v IF maxcnt EQ 1 THEN vmax_val=*sds[i,maxi[0]].va_v IF ((mincnt EQ 0) OR (maxcnt EQ 0)) AND (vrcnt EQ 1) THEN BEGIN vx_val=*sds[i,vri[0]].va_v IF mincnt EQ 0 THEN vmin_val=vx_val[0] IF maxcnt EQ 0 THEN vmax_val=vx_val[1] ENDIF IF (mincnt EQ 0) AND (maxcnt EQ 0) AND (vrcnt EQ 0) THEN nominmax=1 $ ELSE nominmax=0 ;Scale the original dataset IF vfcnt EQ 1 THEN BEGIN gi=WHERE(new_data NE vf_val[0],gcnt) IF gcnt NE 0 THEN BEGIN new_data[gi]=new_data[gi]*newmult[nvui] ;determine min and max data values maxv=MAX(new_data[gi],MIN=minv) ENDIF ELSE nominmax=1 ENDIF ELSE BEGIN new_data=new_data*newmult[nvui] ;determine min and max data values maxv=MAX(new_data,MIN=minv) ENDELSE IF nominmax EQ 0 THEN BEGIN ;Determine new Min, Max and Fill Values IF vmax_val GT 0. THEN WHILE maxv GT vmax_val DO vmax_val=vmax_val*1.e1 IF vmin_val GT 0. THEN BEGIN WHILE vmin_val LT minv DO vmin_val=vmin_val*1.e1 vmin_val=vmin_val*1.e-1 ENDIF ELSE IF vmin_val LT 0. THEN BEGIN WHILE vmin_val GT minv DO vmin_val=vmin_val*1.e1 ENDIF IF vfcnt EQ 1 THEN BEGIN IF vf_val[0] NE 0. THEN $ WHILE (vf_val[0] LE vmax_val) AND (vf_val[0] GE vmin_val) DO vf_val[0]=vf_val[0]*1.e1 ENDIF ENDIF sds[i,0].data=PTR_NEW(new_data) IF mincnt EQ 1 THEN sds[i,mini[0]].va_v=PTR_NEW(vmin_val) IF maxcnt EQ 1 THEN sds[i,maxi[0]].va_v=PTR_NEW(vmax_val) IF vrcnt EQ 1 THEN sds[i,vri[0]].va_v=PTR_NEW([vmin_val,vmax_val]) IF vscnt EQ 1 THEN sds[i,vsi[0]].va_v=PTR_NEW('0.0;'+newvsc[nvui]+';1') IF vdcnt EQ 1 THEN BEGIN vd_val=*sds[i,vdi[0]].va_v ;Look for ppv text and replace with new units WHILE STRPOS(STRUPCASE(vd_val),'PPV') NE -1 DO $ vd_val=STRMID(vd_val,0,STRPOS(STRUPCASE(vd_val),'PPV'))+newvu[nvui]+ $ STRMID(vd_val,STRPOS(STRUPCASE(vd_val),'PPV')+3) sds[i,vdi[0]].va_v=PTR_NEW(vd_val) ENDIF infotxt=' INFORMATION: Metadata and Data Values updated for '+vn_val+' due to VAR_UNITS change from '+ $ vu_val+' to '+newvu[nvui] PRINT,infotxt IF template_change[0] EQ '' THEN template_change[0]=infotxt $ ELSE template_change=[template_change,infotxt] ENDIF ENDIF ENDFOR END ;Procedure PPV_Check PRO percent_to_abs,sds,sds_lab,vui,dvnames,dvi,template_change ;Procedure to convert percentage uncertainty values to the same units as the accompanying ;species measurements ; ---------- ;Written by Ian Boyd for AVDC/EVDC - iboyd@bryanscientific.org ; ;History: ; 20131023: Introduced to geoms_tools - Version 1.3 ; 20131206: Add check for Valid Pointers - Version 1.33 ; ;Inputs: sds - a structure containing the Metadata and Data read in using idlcr8ascii ; sds_lab - a string array containing the metadata labels of the UNCERTAINTY variable ; vui - index value of the VAR_UNITS entry ; dvnames - a string array containing the data variable names ; dvi - index value of the DATA_VARIABLE entry ; ;Outputs: sds - changes may be made to some of the Metadata and Data entries ; template_change - A string array containing comments on any changes made ; ;Called by: GEOMS_TOOLS ; ;Subroutines Called: None ;Determine source variable name error_mode=[0] ;0/1 changed to 1 if problem with the conversion vu_val='absolute units' source_dataset=STRMID(dvnames[dvi],0,STRPOS(dvnames[dvi],'_UNCERTAINTY')) dsi=WHERE(dvnames EQ source_dataset,dscnt) IF dscnt EQ 1 THEN BEGIN ;source for UNCERTAINTY measurements have been found at_size=N_ELEMENTS(sds_lab) ds_lab=STRARR(at_size) ;write source variable name labels to string array FOR j=0,at_size-1 DO $ IF PTR_VALID(sds[dsi[0],j].va_l) THEN ds_lab[j]=STRUPCASE(*sds[dsi[0],j].va_l) ;determine VAR_UNITS of the source dataset vxi=WHERE(ds_lab EQ 'VAR_UNITS',vxicnt) IF vxicnt NE 1 THEN error_mode=[error_mode,2] $ ;VAR_UNITS of the source dataset not found ELSE BEGIN ;rename VAR_UNITS for UNCERTAINTY dataset vu_val=STRTRIM(*sds[dsi[0],vxi[0]].va_v,2) sds[dvi,vui].va_v=PTR_NEW(vu_val) ;also test for and replace H4 reporting 'units' vxi=WHERE(sds_lab EQ 'UNITS',vxicnt) IF vxicnt EQ 1 THEN sds[dvi,vxi[0]].va_v=PTR_NEW(vu_val) ENDELSE ;determine the UNCERTAINTY and source fill values vfvi=WHERE(sds_lab EQ 'VAR_FILL_VALUE',vfvcnt) dfvi=WHERE(ds_lab EQ 'VAR_FILL_VALUE',dfvcnt) IF vfvcnt NE 1 THEN error_mode=[error_mode,3] ;No fill value for UNCERTAINTY dataset IF dfvcnt NE 1 THEN error_mode=[error_mode,4] ;No fill value for source dataset ;determine VAR_VALID_MIN and VAR_VALID_MAX values (will be converted to equivalent absolute units) mini=WHERE(sds_lab EQ 'VAR_VALID_MIN',mincnt) maxi=WHERE(sds_lab EQ 'VAR_VALID_MAX',maxcnt) IF mincnt NE 1 THEN error_mode=[error_mode,5] ;No VAR_VALID_MIN value for UNCERTAINTY dataset IF maxcnt NE 1 THEN error_mode=[error_mode,6] ;No VAR_VALID_MAX value for UNCERTAINTY dataset ;put UNCERTAINTY and source data into arrays ds_data=*sds[dsi[0],0].data & sds_data=*sds[dvi,0].data res=SIZE(ds_data) & ds_size=res(N_ELEMENTS(res)-1) res=SIZE(sds_data) & sds_size=res(N_ELEMENTS(res)-1) IF ds_size NE sds_size THEN error_mode=5 ENDIF ELSE error_mode=[error_mode,1] ;cannot find source dataset IF N_ELEMENTS(error_mode) EQ 1 THEN BEGIN ;can make all the corrections vxi=WHERE(sds_lab EQ 'VAR_DESCRIPTION',vxicnt) IF vxicnt EQ 1 THEN BEGIN ;Make new VAR_DESCRIPTION entry ;currently assumes that only MWR requiring this routine itxt=STRMID(dvnames[dvi],0,STRPOS(dvnames[dvi],'.')) CASE 1 OF STRPOS(dvnames[dvi],'RANDOM') NE -1: itxt2='Random Uncertainty' STRPOS(dvnames[dvi],'SYSTEMATIC') NE -1: itxt2='Systematic Uncertainty' STRPOS(dvnames[dvi],'COMBINED') NE -1: itxt2='Combined Uncertainty' ENDCASE sds[dvi,vxi[0]].va_v=PTR_NEW(itxt+' Profile 1-sigma '+itxt2) ENDIF ;remove existing entry for VAR_SI_CONVERSION (re-calculated by idlcr8hdf) vsci=WHERE(sds_lab EQ 'VAR_SI_CONVERSION',vsccnt) IF vsccnt EQ 1 THEN sds[dvi,vsci[0]].va_v=PTR_NEW(' ') ;rename VAR_FILL_VALUE to the same as source dataset vfv_val=*sds[dvi,vfvi[0]].va_v dfv_val=*sds[dsi[0],dfvi[0]].va_v sds[dvi,vfvi[0]].va_v=PTR_NEW(dfv_val[0]) ;also test for and replace H4 reporting '_FillValue' vxi=WHERE(sds_lab EQ '_FILLVALUE',vxicnt) IF vxicnt EQ 1 THEN sds[dvi,vxi[0]].va_v=PTR_NEW(dfv_val[0]) ;change data from % to absolute units sds_new=sds_data FOR j=0L,sds_size-1L DO BEGIN IF (sds_data[j] NE vfv_val[0]) AND (ds_data[j] NE dfv_val[0]) THEN $ sds_new[j]=ABS(sds_data[j]*ds_data[j]/100.) $ ELSE sds_new[j]=dfv_val[0] ENDFOR sds[dvi,0].data=PTR_NEW(sds_new) ;derive new VAR_VALID_MIN and VAR_VALID_MAX values sds[dvi,mini[0]].va_v=PTR_NEW(0.) gi=WHERE(sds_new NE dfv_val[0],gcnt) IF gcnt NE 0 THEN BEGIN maxv=MAX(sds_new[gi]) exv=0 IF maxv GE 1. THEN BEGIN WHILE maxv GT 10. DO BEGIN maxv=maxv/10. & exv++ ENDWHILE ENDIF ELSE IF maxv NE 0. THEN BEGIN WHILE maxv LT 1. DO BEGIN maxv=maxv*10. & exv-- ENDWHILE ENDIF IF maxv GT 5. THEN maxv=10.^(exv+1) ELSE maxv=5.*10.^exv sds[dvi,maxi[0]].va_v=PTR_NEW(maxv) ;also test for and replace H4 reporting 'valid_range' vr=[0.,maxv] vxi=WHERE(sds_lab EQ 'VALID_RANGE',vxicnt) IF vxicnt EQ 1 THEN sds[dvi,vxi[0]].va_v=PTR_NEW(vr) ENDIF infotxt=' INFORMATION: Metadata and Data Values updated for '+dvnames[dvi]+' due to VAR_UNITS change '+ $ 'from % to '+vu_val PRINT,infotxt IF template_change[0] EQ '' THEN template_change[0]=infotxt $ ELSE template_change=[template_change,infotxt] ENDIF ELSE BEGIN ;CONVERSION not possible so report errors FOR j=1,N_ELEMENTS(error_mode)-1 DO BEGIN CASE error_mode[j] OF 1: itxt=source_dataset+' required but not found' 2: itxt='VAR_UNITS value for '+source_dataset+' not found' 3: itxt='VAR_FILL_VALUE for '+dvnames[dvi]+' required but not found' 4: itxt='VAR_FILL_VALUE for '+source_dataset+' required but not found' 5: itxt='VAR_VALID_MIN value for '+dvnames[dvi]+' required but not found' 6: itxt='VAR_VALID_MAX value for '+dvnames[dvi]+' required but not found' ENDCASE infotxt=' ERROR: '+itxt+' when trying to convert UNCERTAINTY Metadata and data values PRINT,infotxt IF template_change[0] EQ '' THEN template_change[0]=infotxt $ ELSE template_change=[template_change,infotxt] ENDFOR ENDELSE END ;Procedure Percent_to_Abs PRO molecm_to_moleccm,sds,sds_lab,vui,dvnames,dvi,template_change ;Procedure to convert molec m-2 values to molec cm-2 ; ---------- ;Written by Ian Boyd for AVDC/EVDC - iboyd@bryanscientific.org ; ;History: ; 20140311: Introduced to geoms_tools - Version 1.35 ; ;Inputs: sds - a structure containing the Metadata and Data read in using idlcr8ascii ; sds_lab - a string array containing the metadata labels of the variable ; vui - index value of the VAR_UNITS entry ; dvnames - a string array containing the data variable names ; dvi - index value of the DATA_VARIABLE entry ; ;Outputs: sds - changes may be made to some of the Metadata and Data entries ; template_change - A string array containing comments on any changes made ; ;Called by: GEOMS_TOOLS ; ;Subroutines Called: None error_mode=[0] ;0/1 changed to 1 if problem with the conversion at_size=N_ELEMENTS(sds_lab) FOR j=0,at_size-1 DO $ sds[dvi,vui].va_v=PTR_NEW('molec cm-2') ;also test for and replace H4 reporting 'units' vxi=WHERE(sds_lab EQ 'UNITS',vxicnt) IF vxicnt EQ 1 THEN sds[dvi,vxi].va_v=PTR_NEW('molec cm-2') ;determine new fill values vfvi=WHERE(sds_lab EQ 'VAR_FILL_VALUE',vfvcnt) IF vfvcnt NE 1 THEN error_mode=[error_mode,0] ;No fill value for dataset ;determine VAR_VALID_MIN and VAR_VALID_MAX values mini=WHERE(sds_lab EQ 'VAR_VALID_MIN',mincnt) maxi=WHERE(sds_lab EQ 'VAR_VALID_MAX',maxcnt) IF mincnt NE 1 THEN error_mode=[error_mode,1] ;No VAR_VALID_MIN value for UNCERTAINTY dataset IF maxcnt NE 1 THEN error_mode=[error_mode,2] ;No VAR_VALID_MAX value for UNCERTAINTY dataset sds_data=*sds[dvi,0].data ;put data into an array res=SIZE(sds_data) & sds_size=res(N_ELEMENTS(res)-1) IF N_ELEMENTS(error_mode) EQ 1 THEN BEGIN ;can make all the corrections ;replace existing entry for VAR_SI_CONVERSION vsci=WHERE(sds_lab EQ 'VAR_SI_CONVERSION',vsccnt) IF vsccnt EQ 1 THEN sds[dvi,vsci[0]].va_v=PTR_NEW('0.0;1.66054E-20;mol m-2') ;derive new VAR_VALID_MIN and VAR_VALID_MAX values min_val=*sds[dvi,mini[0]].va_v min_val=float(strtrim(min_val*1.e-4,2)) sds[dvi,mini[0]].va_v=PTR_NEW(min_val*1.e-4) max_val=*sds[dvi,maxi[0]].va_v max_val=float(strtrim(max_val*1.e-4,2)) sds[dvi,maxi[0]].va_v=PTR_NEW(max_val) ;also test for and replace H4 reporting 'valid_range' vr=[min_val,max_val] vxi=WHERE(sds_lab EQ 'VALID_RANGE',vxicnt) IF vxicnt EQ 1 THEN sds[dvi,vxi[0]].va_v=PTR_NEW(vr) ;extract VAR_FILL_VALUE vfv_val=*sds[dvi,vfvi[0]].va_v ;also test for H4 reporting '_FillValue' vxi=WHERE(sds_lab EQ '_FILLVALUE',vxicnt) new_vfv=float(strtrim(vfv_val[0]*1.0d-4,2)) print,format='(e20.8)',new_vfv IF (new_vfv LT 0.) AND ((new_vfv GT -90000.) OR (min_val GT -90000.)) THEN new_vfv=-90000. sds[dvi,vfvi[0]].va_v=PTR_NEW(new_vfv) IF vxicnt NE 0 THEN sds[dvi,vxi[0]].va_v=PTR_NEW(new_vfv) ;change data from molec m-2 to molec cm-2 sds_new=sds_data FOR j=0L,sds_size-1L DO BEGIN IF sds_data[j] NE vfv_val[0] THEN sds_new[j]=sds_data[j]*1.e-4 $ ELSE sds_new[j]=new_vfv ENDFOR sds[dvi,0].data=PTR_NEW(sds_new) infotxt=' INFORMATION: Metadata and Data Values updated for '+dvnames[dvi]+' due to VAR_UNITS change '+ $ 'from molec m-2 to molec cm-2' PRINT,infotxt IF template_change[0] EQ '' THEN template_change[0]=infotxt $ ELSE template_change=[template_change,infotxt] ENDIF ELSE BEGIN ;CONVERSION not possible so report errors FOR j=1,N_ELEMENTS(error_mode)-1 DO BEGIN CASE error_mode[j] OF 0: itxt='VAR_FILL_VALUE for '+dvnames[dvi]+' required but not found' 1: itxt='VAR_VALID_MIN value for '+dvnames[dvi]+' required but not found' 2: itxt='VAR_VALID_MAX value for '+dvnames[dvi]+' required but not found' ENDCASE infotxt=' ERROR: '+itxt+' when trying to convert Metadata and data values PRINT,infotxt IF template_change[0] EQ '' THEN template_change[0]=infotxt $ ELSE template_change=[template_change,infotxt] ENDFOR ENDELSE END ;Procedure MolecM_to_MolecCm PRO geoms_template_change,ga,sds,template_change ;Procedure to make changes to the contents of the file due to changes made to the templates ;or, in the case of La Reunion, to the DATA_LOCATION value (added St. Denis or Maido) ; ---------- ;Written by Ian Boyd for AVDC/EVDC - iboyd@bryanscientific.org ; ;History: ; 20131023: Introduced to geoms_tools - Version 1.3 ; 20131028: Fixed bug that caused an endless loop when calculating VAR_VALID_MAX values ; for UNCERTAINITY values changed from % to absolute units - Version 1.31 ; 20131206: Include ppvchk call for FTIR measurements - Version 1.33 ; 20140311: Update MWR template and include mmchk call to convert molec m-2 to molec cm-2 ; - Version 1.35 ; 20140806: Add additional criteria for identifying LA.REUNION location - Version 1.39 ; ;Inputs: ga - a string array holding the Global Attributes read in using idlcr8ascii ; sds - a structure containing the Metadata and Data read in using idlcr8ascii ; ;Outputs: ga - changes may be made to some Global Attribute entries ; sds - changes may be made to some of the Metadata and Data entries ; template_change - A string array containing comments on any changes made ; ;Called by: GEOMS_TOOLS ; ;Subroutines Called: PERCENT_TO_ABS ; PPV_CHECK dum='' & template_change=[''] & obs_tem=0 ;List of current templates and versions (no need to do lidar at this time) temlist=['MWR-003','FTIR-002','FTUV-002'] ;,'SONDE-002','UVVIS-DOAS-GAS-003','UVVIS-DOAS-AEROSOL-003'] n_tem=N_ELEMENTS(temlist) temspc=STRARR(n_tem) FOR i=0,n_tem-1 DO temspc[i]=STRMID(temlist[i],0,STRLEN(temlist[i])-4) ;Add FTS to temspc (FTS changed to FTUV from 002 onwards) temspc=[temspc,'FTS'] ;Find DATA_VARIABLES field di=WHERE(strmid(STRUPCASE(ga),0,14) EQ 'DATA_VARIABLES',dvlcnt) res=STRSPLIT(ga[di[0]],' =;',/EXTRACT,COUNT=dvcnt) ;Get Size information for the data structure sds_size=SIZE(sds) ;Do DATA_LOCATION check for Reunion based on instrument or surface altitude lrchk=STRPOS(STRUPCASE(ga),'LA.REUNION') NE -1 lri=WHERE(lrchk NE 0,lrcnt) IF lrcnt NE 0 THEN BEGIN ;file is for Reunion so do remaining checks resdv=STRUPCASE(res) dli=WHERE(STRMID(ga,0,13) EQ 'DATA_LOCATION',dlcnt) IF dlcnt EQ 1 THEN BEGIN dlvres=STRSPLIT(STRUPCASE(ga[dli[0]]),' =;',/EXTRACT,COUNT=dlvcnt) dlvres=[dlvres,''] ;add dummy value so number of elements is at least 2 ENDIF IF dlvres[1] EQ 'LA.REUNION' THEN BEGIN ;need to try and identify actual site ;Look for specific DATA_VARIABLES to allow checks to be made a0i=WHERE(resdv EQ 'ALTITUDE.INSTRUMENT',a0cnt) a1i=WHERE(resdv EQ 'ALTITUDE',a1cnt) a2i=WHERE(resdv EQ 'PRESSURE',a2cnt) ;Note: Pressure currently expected to be index 2 a3i=WHERE(resdv EQ 'DATETIME',a3cnt) ;Note: DATETIME currently expected to be index 3 alli=[a0i[0],a1i[0],a2i[0],a3i[0]] & allcnt=[a0cnt,a1cnt,a2cnt,a3cnt] stcrit=[1000.,1000.,90000.,4565.] ;cut-off criteria to identify St. Denis in m, Pa, or MJD2K vscchk=[-1.,-1.,-1.,86400.] ;VAR_SI_CONVERSION check values. sds_lab=STRARR(sds_size[2]) ;Go through variables in order to see if site can be positively identified wcnt=0 & nchks=N_ELEMENTS(allcnt) WHILE wcnt LE nchks-1 DO BEGIN IF allcnt[wcnt] EQ 1 THEN BEGIN ;Information found alti=alli[wcnt]-1 altv=*sds[alti,0].data FOR j=0,sds_size[2]-1 DO sds_lab[j]=STRUPCASE(*sds[alti,j].va_l) vsci=WHERE(sds_lab EQ 'VAR_SI_CONVERSION',vsccnt) vfvi=WHERE(sds_lab EQ '_FILLVALUE',vfvcnt) IF vfvcnt EQ 0 THEN vfvi=WHERE(sds_lab EQ 'VAR_FILL_VALUE',vfvcnt) IF (vsccnt EQ 1) AND (vfvcnt EQ 1) THEN BEGIN vsc_val=*sds[alti,vsci[0]].va_v & vfv_val=*sds[alti,vfvi[0]].va_v vscres=STRSPLIT(vsc_val,';',/EXTRACT,COUNT=vrescnt) ;needs to be 3 sub-values for VAR_SI_CONVERSION vscres=[vscres,'x'] IF (vrescnt EQ 3) AND (IS_A_NUMBER_GT(vscres[1])) AND (IS_A_NUMBER_GT(vfv_val[0])) THEN BEGIN vscscl=FLOAT(vscres[1]) & vfv_val=DOUBLE(vfv_val) IF wcnt EQ nchks-1 THEN BEGIN ;check that scale factor is 86400. but do not scale the values IF ROUND(vscscl) EQ ROUND(vscchk[nchks-1]) THEN vscscl=1.0 ELSE vscscl=-1. ENDIF gi=WHERE(ROUND(DOUBLE(altv)) NE ROUND(vfv_val),gcnt) IF (vscscl GT 0.) AND (gcnt NE 0) THEN BEGIN newdlv='' IF wcnt EQ 2 THEN BEGIN ;Pressure testval=MAX(altv[gi]*vscscl) ;pressure scaled to Pa IF testval GT stcrit[2] THEN newdlv=dlvres[1]+'.STDENIS' ELSE newdlv=dlvres[1]+'.MAIDO' wcnt=999 ;no more checks required so exit loop ENDIF ELSE BEGIN testval=MIN(altv[gi]*vscscl) ;variable scaled to the SI unit (m, MJD2K) IF testval LT stcrit[wcnt] THEN newdlv=dlvres[1]+'.STDENIS' $ ELSE IF wcnt LT nchks-1 THEN newdlv=dlvres[1]+'.MAIDO' ;For DATETIME still cannot be sure of the site wcnt=999 ;no more checks required or can be made so exit loop ENDELSE IF newdlv NE '' THEN BEGIN ;change DATA_LOCATION value and write information to LOG ga[dli[0]]='DATA_LOCATION='+newdlv ;create comment for valid change infotxt=' INFORMATION: DATA_LOCATION '+dlvres[1]+' renamed '+newdlv PRINT,infotxt IF template_change[0] EQ '' THEN template_change[0]=infotxt $ ELSE template_change=[template_change,infotxt] ENDIF ENDIF ENDIF ENDIF ENDIF wcnt=wcnt+1 ENDWHILE ENDIF ENDIF ;identify the template in the HDF file and, if valid, check to see if it is obsolete ti=WHERE(strmid(STRUPCASE(ga),0,13) EQ 'DATA_TEMPLATE',tcnt) IF (tcnt EQ 1) AND (dvlcnt EQ 1) THEN BEGIN ;DATA_VARIABLES and DATA_TEMPLATE fields present in the file dtorig=STRTRIM(STRMID(ga[ti[0]],STRPOS(ga[ti[0]],'=')+1),2) tli=0 & temfound=0 WHILE (~ temfound) AND (tli LE n_tem) DO $ IF STRPOS(STRUPCASE(ga[ti[0]]),temspc[tli]) NE -1 THEN temfound=1 ELSE tli++ IF (temfound) AND (tli NE n_tem) THEN $ ;check for obsolete version of the template IF STRPOS(STRUPCASE(ga[ti[0]]),temlist[tli]) EQ -1 THEN obs_tem=1 $ ELSE IF (temfound) AND (tli EQ n_tem) THEN BEGIN tli=2 & obs_tem=1 ;obsolete FTS ENDIF ENDIF IF obs_tem THEN BEGIN ;template is obsolete so perform checks ;Extract DATA_VARIABLES values dvorig=res[1:dvcnt-1] & dvnew=dvorig & dv=STRUPCASE(dvnew) dvsc=dv+';' & dvcnt-- pchk=0 & ppvchk=0 & mmchk=0 ;Booleans to set additional checks, depending on the template m_orig=['_UNCERTAINTY.RANDOM;','_UNCERTAINTY.SYSTEMATIC;','_UNCERTAINTY.TOTAL;', $ 'O3.COLUMN_EMISSION;','H2O.COLUMN;'] m_new=['_UNCERTAINTY.RANDOM.STANDARD','_UNCERTAINTY.SYSTEMATIC.STANDARD', $ '_UNCERTAINTY.COMBINED.STANDARD','O3.COLUMN.PARTIAL_EMISSION','H2O.COLUMN_DERIVED'] f_orig=['_UNCERTAINTY.RANDOM;','_UNCERTAINTY.SYSTEMATIC;'] f_new=['_UNCERTAINTY.RANDOM.COVARIANCE|_UNCERTAINTY.RANDOM.STANDARD', $ '_UNCERTAINTY.SYSTEMATIC.COVARIANCE|_UNCERTAINTY.SYSTEMATIC.STANDARD'] CASE 1 of tli EQ 0: BEGIN ;MWR measurement t_orig=m_orig & t_new=m_new & pchk=1 & ppvchk=1 & mmchk=1 END tli EQ 1: BEGIN ;FTIR measurement t_orig=f_orig & t_new=f_new & ppvchk=1 END ELSE: ENDCASE IF ppvchk EQ 1 THEN $ ;Do checks for ppv units and change to ppmv/ppbv/pptv PPV_CHECK,sds,dvcnt,sds_size[2],template_change n_chks=N_ELEMENTS(t_orig) FOR i=0,n_chks-1 DO BEGIN REPEAT BEGIN ;Variable needing changing could appear more than once in the template dvi=0 & dvfound=0 WHILE (~ dvfound) AND (dvi LE dvcnt-1) DO $ IF STRPOS(dvsc[dvi],t_orig[i]) NE -1 THEN dvfound=1 ELSE dvi++ IF dvfound THEN BEGIN ;harmonize to the changed variable name ;check for more than one choice for the new UNCERATINTY name IF STRPOS(t_new[i],'|') NE -1 THEN BEGIN ;split the two values out tnres=STRSPLIT(t_new[i],' |',/EXTRACT) ;check for 'COLUMN' in the variable name IF STRPOS(dv[dvi],'COLUMN') NE -1 THEN tnew=tnres[1] ELSE tnew=tnres[0] ENDIF ELSE tnew=t_new[i] dvnew[dvi]=STRMID(dvorig[dvi],0,STRPOS(dvsc[dvi],t_orig[i]))+tnew dvsc[dvi]='' ;change Metadata VARIABLE_NAME sds_lab=STRARR(sds_size[2]) FOR j=0,sds_size[2]-1 DO sds_lab[j]=STRUPCASE(*sds[dvi,j].va_l) vni=WHERE(sds_lab EQ 'VAR_NAME',vnicnt) IF vnicnt EQ 1 THEN BEGIN sds[dvi,vni[0]].va_v=PTR_NEW(dvnew[dvi]) ;create comment for valid change infotxt=' INFORMATION: Dataset name '+dvorig[dvi]+' renamed '+dvnew[dvi] PRINT,infotxt IF template_change[0] EQ '' THEN template_change[0]=infotxt $ ELSE template_change=[template_change,infotxt] ;Do Additional Checks vui=WHERE(sds_lab EQ 'VAR_UNITS',vuicnt) IF vuicnt EQ 1 THEN vu_val=STRTRIM(STRUPCASE(*sds[dvi,vui[0]].va_v),2) ELSE vu_val='' test1=(STRPOS(t_orig[i],'_UNCERTAINTY') NE -1) AND (pchk EQ 1) AND (vu_val EQ '%') test2=(STRPOS(t_orig[i],'H2O.COLUMN') NE -1) AND (mmchk EQ 1) AND (vu_val EQ 'MOLEC M-2') IF test1 THEN $ ;UNCERTAINTY values need to be converted from % to absolute value PERCENT_TO_ABS,sds,sds_lab,vui,dvnew,dvi,template_change IF test2 THEN $ ;H2O.COLUMN values need to be converted from molec m-2 to molec cm-2 MOLECM_TO_MOLECCM,sds,sds_lab,vui,dvnew,dvi,template_change ENDIF ENDIF ENDREP UNTIL dvfound EQ 0 ENDFOR ;update the DATA_VARIABLES Global Attribute ga[di[0]]='DATA_VARIABLES=' FOR i=0,dvcnt-1 DO $ IF i NE dvcnt-1 THEN ga[di[0]]=ga[di[0]]+dvnew[i]+';' ELSE ga[di[0]]=ga[di[0]]+dvnew[i] ;update the DATA_TEMPLATE Global Attribute dtnew='GEOMS-TE-'+temlist[tli] ga[ti[0]]='DATA_TEMPLATE='+dtnew ;create comment for valid change infotxt=' INFORMATION: DATA_TEMPLATE value '+dtorig+' renamed '+dtnew PRINT,infotxt IF template_change[0] EQ '' THEN template_change[0]=infotxt $ ELSE template_change=[template_change,infotxt] ENDIF END ;Procedure GEOMS_Template_Change PRO dependency_order, ga, sds, order_change ;Procedure to check that dependent variables are reported after the independent or axis variables. ;If not changes the variable order until this is the case ; ---------- ;Written by Ian Boyd for AVDC/EVDC - iboyd@bryanscientific.org ; ;History: ; 20131121: Introduced to geoms_tools - Version 1.32 ; 20131206: Add check for Valid Pointers - Version 1.33 ; ;Inputs: ga - a string array holding the Global Attributes read in using idlcr8ascii ; sds - a structure containing the Metadata and Data read in using idlcr8ascii ; ;Outputs: sds - changes may be made to some of the Metadata and data ordering ; ga - changes may be made to the data variable ordering ; order_change - A string array containing comments on any changes made ; ;Called by: GEOMS_TOOLS ; ;Subroutines Called: NONE ;Get Size information for the data structure sds_size=SIZE(sds) vnv=STRARR(sds_size[1]) & vdv=vnv ;Get values for VAR_NAME and VAR_DEPEND for each dataset FOR i=0,sds_size[1]-1 DO BEGIN FOR j=0,sds_size[2]-1 DO BEGIN IF PTR_VALID(sds[i,j].va_l) THEN BEGIN lab=*sds[i,j].va_l IF STRUPCASE(lab) EQ 'VAR_NAME' THEN vnv[i]=*sds[i,j].va_v $ ELSE IF STRUPCASE(lab) EQ 'VAR_DEPEND' THEN vdv[i]=*sds[i,j].va_v ENDIF ENDFOR ENDFOR ;determine if there is a DATA_VARIABLES entry gi=WHERE(STRMID(STRUPCASE(ga),0,14) EQ 'DATA_VARIABLES',gcnt) order_ok=1 ;default assuming dataset ordering is OK REPEAT BEGIN out_of_order=0 ;will change if an out of order variable name is found ;Go through Variable Names until an out of order value is found (or reach the end of the list) i=0 WHILE (i LE sds_size[1]-1) AND (out_of_order EQ 0) DO BEGIN ;seperate out VAR_DEPEND values vdvs=STRSPLIT(vdv[i],' ;',/EXTRACT,COUNT=vdcnt) imax=-1 IF vdcnt GE 1 THEN BEGIN FOR j=0,vdcnt-1 DO BEGIN vi=WHERE(STRUPCASE(vdvs[j]) EQ STRUPCASE(vnv)) IF vi[0] GT i THEN IF vi[0] GT imax THEN imax=vi[0] ENDFOR ENDIF IF imax NE -1 THEN out_of_order=1 ELSE i=i+1 ENDWHILE IF out_of_order EQ 1 THEN BEGIN ;swap order of datasets order_ok=0 ;read in the invalid dataset, move datasets between i+1 and imax up by one, insert invalid dataset ;at imax index value laba=STRARR(sds_size[2]) & vala=laba FOR j=0,sds_size[2]-1 DO BEGIN IF PTR_VALID(sds[i,j].va_l) THEN BEGIN laba[j]=*sds[i,j].va_l CASE STRUPCASE(laba[j]) OF 'VAR_VALID_MIN': vvmin=*sds[i,j].va_v 'VAR_VALID_MAX': vvmax=*sds[i,j].va_v 'VAR_FILL_VALUE': vfv=*sds[i,j].va_v '_FILLVALUE': vfv2=*sds[i,j].va_v 'VALID_RANGE': vr=*sds[i,j].va_v ELSE: vala[j]=STRTRIM(*sds[i,j].va_v,2) ENDCASE ENDIF ENDFOR dataset=*sds[i,0].data ;move affected datasets up by one FOR k=i+1,imax DO BEGIN FOR j=0,sds_size[2]-1 DO BEGIN IF PTR_VALID(sds[k,j].va_l) THEN BEGIN labs=*sds[k,j].va_l & vals=*sds[k,j].va_v sds[k-1,j].va_l=PTR_NEW(labs) sds[k-1,j].va_v=PTR_NEW(vals) ENDIF ENDFOR dset=*sds[k,0].data sds[k-1,0].data=PTR_NEW(dset) ENDFOR ;insert invalid dataset into imax index FOR j=0,sds_size[2]-1 DO BEGIN IF PTR_VALID(sds[imax,j].va_l) THEN BEGIN sds[imax,j].va_l=PTR_NEW(laba[j]) CASE STRUPCASE(laba[j]) OF 'VAR_VALID_MIN': sds[imax,j].va_v=PTR_NEW(vvmin) 'VAR_VALID_MAX': sds[imax,j].va_v=PTR_NEW(vvmax) 'VAR_FILL_VALUE': sds[imax,j].va_v=PTR_NEW(vfv) '_FILLVALUE': sds[imax,j].va_v=PTR_NEW(vfv2) 'VALID_RANGE': sds[imax,j].va_v=PTR_NEW(vr) ELSE: sds[imax,j].va_v=PTR_NEW(vala[j]) ENDCASE ENDIF ENDFOR sds[imax,0].data=PTR_NEW(dataset) ;Get reordered values for VAR_NAME and VAR_DEPEND for each dataset for next run through FOR i=0,sds_size[1]-1 DO BEGIN FOR j=0,sds_size[2]-1 DO BEGIN IF PTR_VALID(sds[i,j].va_l) THEN BEGIN lab=*sds[i,j].va_l IF STRUPCASE(lab) EQ 'VAR_NAME' THEN vnv[i]=*sds[i,j].va_v $ ELSE IF STRUPCASE(lab) EQ 'VAR_DEPEND' THEN vdv[i]=*sds[i,j].va_v ENDIF ENDFOR ENDFOR IF gcnt EQ 1 THEN BEGIN ;generate revised DATA_VARIABLES attribute ga[gi[0]]='DATA_VARIABLES=' FOR i=0,sds_size[1]-1 DO $ IF i EQ 0 THEN ga[gi[0]]=ga[gi[0]]+vnv[i] ELSE ga[gi[0]]=ga[gi[0]]+';'+vnv[i] ENDIF ENDIF ENDREP UNTIL out_of_order eq 0 IF order_ok EQ 0 THEN BEGIN ;Generate INFORMATION message infotxt=' INFORMATION: Dataset ordering changed so that AXIS variable dataset(s) are listed before dependent datasets' PRINT,infotxt IF order_change[0] EQ '' THEN order_change[0]=infotxt $ ELSE order_change=[order_change,infotxt] ENDIF END ;Procedure Dependency_Order FUNCTION geoms_tools, optqh, tav, hdfqh, outdir, dataformat, DATETIME=o1, NOTC=o2, FV=o3 ;Program to perform QA or Harmonization on GEOMS compliant HDF and netCDF files. ; ;Program documentation: geoms-qa-v1.0_Readme.pdf, available from http://avdc.gsfc.nasa.gov. ; geoms-harmon-v1.0_Readme.pdf, available from http://avdc.gsfc.nasa.gov. ; idlcr8ascii-v4.0_Readme.pdf, available from http://avdc.gsfc.nasa.gov. ; idlcr8hdf-v4.0_Readme.pdf, available from http://avdc.gsfc.nasa.gov. ;Program sub-version 2.17 (20241126) ; ---------- ;Written by Ian Boyd for AVDC/EVDC - iboyd@bryanscientific.org ; ; History: ; 20101210: Original Release - Version 0.1 ; 20101221: Change PROCEDURE to a FUNCTION, and return an error code on exit ; (assumes single HDF file input) - Version 0.2 ; 20101222: Change name to GEOMS_TOOLS; add Harmonization option, which includes extra ; input parameter optqh=0/1 (QA or Harmonization); add optional fourth parameter ; which gives an output directory (required if optqh EQ 1 and new HDF file(s) are ; required) - Version 0.3 ; 20110113: Include log output from the idlcr8ascii call; idlcr8hdf upgraded to correct for ; additional obsolete values; In Harmonization mode do initial run to look for ; issues, then second run only if harmonization required; SOURCE.PRODUCT ; information added or appended to the HDF file when it is successfully ; Harmonized; Increment or reset DATA_FILE_VALUE according to given criteria ; - Version 0.4 ; 20110215: Account for messages returned in the call to idlcr8ascii; GEOMS_ONE_OFF_CHECK ; procedure added which corrects for obsolete Global Attribute Values - Version 0.5 ; 20110325: Check whether testing HDF5 or HDF4 files and call idlcr8hdf accordingly ; - Version 0.6 ; 20110329: Change exit code status for incorrect input from -1 to 3 - Version 0.7 ; 20110426: Include DATA_TEMPLATE information after the TAV information in the output log ; file - Version 0.8 ; 20110608: Add optional dataformat parameter which forces any new file generation to be in the ; specified format (currently H4/H5). Applies to harmonization option only, it is ; ignored for QA. If not included any new file is in the same format as the input ; file - Version 1.0 ; 20110819: Add /DATETIME keyword, which causes idlcr8hdf to be called with relaxed DATETIME ; checking criteria - Version 1.1 ; 20120314: Add netCDF support. Add /NOTC keyword which changes the Required DATA_TEMPLATE comment ; so that Template Checking does not take place in the final QA/TC checks - applies to ; harmonization option only - Version 1.2 ; 20120426: Fix bug when reading contents of idlcr8hdf log file when doing QA/TC checks; add ; correct extension name for new file in SOURCE.PRODUCT comment - Version 1.21 ; 20131023: Add routines ppv_check, percent_to_abs, and geoms_template_change to be able ; to detect and correct for updates in the GEOMS templates - Version 1.3 ; 20131121: Add routine dependency_order to check for and reorder dependent variables that are ; reported before the independent or axis variables - Version 1.32 ; 20140128: Align geoms_tools version due to change in geoms_tctools - Version 1.34 ; 20140325: Update due to modification to idlcr8ascii - Version 1.36 ; 20140329: Update due to modification to idlcr8hdf - Version 1.37 ; 20140522: Update due to modification to idlcr8hdf - Version 1.38 ; 20140806: Add additional criteria for identifying LA.REUNION location; Add IS_A_NUMBER_GT ; function - Version 1.39 ; 20141110: Update due to modification to idlcr8hdf and idlcr8ascii - Version 1.41 ; 20150127: Update due to modification to idlcr8hdf and idlcr8ascii - Version 1.42 ; 20150217: Update due to modification to idlcr8hdf and idlcr8ascii - Version 1.43 ; 20150302: Update due to bug fix in idlcr8hdf - Version 1.44 ; 20150409: Update due to bug fix in idlcr8hdf - Version 1.45 ; 20150811, Update due to bug fixes and improvements in idlcr8hdf - Version 1.46 ; 20150926, Update due to bug fixes and improvements in idlcr8hdf - Version 1.47 ; 20151021, Update due to bug fixes and improvements in idlcr8hdf - Version 1.48 ; 20151104, Update due to fix in idlcr8hdf - Version 1.49 ; 20151109, Update due to bug fixes and improvements in idlcr8hdf and idlcr8ascii ; - Version 1.50 ; 20151116, Update due to fix in idlcr8hdf - Version 1.51 ; 20151215, Update due to fix in idlcr8hdf - Version 1.52 ; 20160213, Add extra satellite instruments to the obsolete list for DATA_SOURCE ; in idlcr8hdf. Modifications to idlcr8hdf and idlc8ascii - Version 1.53 ; 20160614, Update due to fix in idlcr8ascii - Version 1.54 ; 20160725, Update due to fix in idlcr8ascii and idlcr8hdf - Version 1.55 ; 20161116, Update due to fix in idlcr8hdf - Version 1.56 ; 20161130, Update due to fix in idlcr8ascii and idlcr8hdf and account for non_GEOMS ; variable attributes (ptr_valid check) in geoms_tctools - Version 1.57 ; 20161217, Update due to fixes in idlcr8ascii - Version 1.58 ; 20161230, Add FV keyword. If called then forces the DATA_FILE_VERSION not to ; increment when performing harmonization; Incorporates fixes made to ; idlcr8hdf - Version 1.59 ; 20170331, Update due to improvements to idlcr8hdf - Version 1.60 ; 20171121, Update due to fixes in idlcr8hdf - Version 1.61 ; 20180218, Update due to fixes in idlcr8ascii and idlcr8hdf - Version 1.62 ; 20180314, Update due to fix in idlcr8hdf - Version 1.63 ; 20180528, Update due to fix in idlcr8hdf - Version 1.64 ; 20180901, Update due to fix in idlcr8hdf - Version 1.65 ; 20181116, Update due to fix in idlcr8ascii - Version 1.66 ; 20190506, QA checks now carried out using the metadata definitions in the TAV version ; given in the FILE_META_VERSION global attribute. If this attribute is invalid ; or not present, or the TAV file can't be located, the input TAV file will ; be used for checks but an error code will be generated - Version 1.67 ; 20190514, Update due to fix in idlcr8ascii - Version 1.68 ; 20190807, Update due to modifications to idlcr8ascii and idlcr8hdf - Version 1.69 ; 20190821, Update due to fix in idlcr8ascii - Version 1.70 ; 20191205, Update due to changes in geoms_qa to support checks on the optional DATA_SOURCE ; Version Name value (third field in DATA_SOURCE) - Version 2.00 ; 20200226, Update due to geoms_vntools updates - Version 2.01 ; 20200311, Update due to fix to idlcr8hdf - Version 2.02 ; 20200427, Update due to geoms_vntools update - Version 2.03 ; 20200709, Update due to fix to idlcr8hdf - Version 2.04 ; 20200930, Update due to fix to idlcr8ascii; Return gt_error_code value of -2 to geoms_qa ; if idlcr8ascii can't read the input file to stop template and version name ; checks - Version 2.05 ; 20201020, Update due to fixes to idlcr8hdf - Version 2.06 ; 20201026, Update due to fix to idlcr8hdf - Version 2.07 ; 20201122, QA checks carried out using the metadata definitions in the TAV version ; determined by the FILE_GENERATION_DATE (FGD), in the first instance, then the ; FILE_META_VERSION (FMV), if different. This adds a check for the FMV TAV version ; using definitions that are out-of-date. If either TAV can't be determined or found ; then the input TAV will be used for checks and an error generated - Version 2.08 ; 20201211: Apply fixes and improvements to procedures identifying correct TAV version(s) to ; use for checks; Fix IDL Math Error (Floating Illegal Operand) in idlcr8hdf ; - Version 2.09 ; 20220808: Separate SOURCE.PRODUCT entries with ';' instead of '_'; Generate an error message for ; the 2nd TAV check if the error only occurs in the idlcr8ascii checks; Delete ; idlcr8ascii.log if Harmonization is not possible - Version 2.11 ; 20231218: Update due to fixes to idlcr8hdf and geoms_vntools - Version 2.12 ; 20240112: Update due to fix to idlcr8hdf - Version 2.13 ; 20240912: Update due to fixes to idlcr8ascii and idlcr8hdf - Version 2.14 ; 20240927: Update due to fix to idlcr8ascii - Version 2.15 ; 20241029: Update due to fixes to geoms_tctools and idlcr8hdf - Version 2.16 ; 20241126: Add run-time error handling capability in geoms_qa.pro and geoms_harmon.pro, ; plus fixes in idlcr8hdf and idlcrascii - Version 2.17 ; ; Input: optqh - Integer defining type of testing 0=QA, 1=Harmonization ; tav - Table Attribute Values file used to test Metadata ; hdfqh - Either a) a single HDF file, or ; b) a filespec, or ; c) a string array containing a list of HDF files ; Note: Full path must be included as part of the filenames ; outdir - optional parameter which defines an output directory for the HDF files ; (if optqh EQ 1) and the log files. When optqh EQ 1, if this parameter ; is included a new HDF file will be created. If it is not included a new ; HDF will not be created ; dataformat - optional parameter which defines the output format of the file ; (currently accepts 'H4'/'H5'/'NC3'). Applies to harmonization option only, ; it is ignored for QA, and if no outdir parameter is given. If not ; included any new file is in the same format as the input file. ; o1 - /DATETIME KEYWORD that, if included, will relax checks on DATETIME in ; idlcr8hdf. ; o2 - /NOTC KEYWORD that, if included, changes the Required DATA_TEMPLATE comment ; for harmonized files so that Template Checking does not take place during ; QA/TC checks ; o3 - /FV KEYWORD that, if included, does NOT increment the DATA_FILE_VERSION ; ; Output: A log file for each HDF file with the same filename as the HDF file except ; with the extension '.log', as well as an exit error code. Currently the exit ; error code will only apply to the last file tested. ; Exit Codes: ; 0: Passed QA/Harmonization ; 1: QA - GEOMS compliant with non-standard components; ; Harmonization - Failed ; 2: QA - Failed ; 3: Incorrect input parameters ; ; External Routines Called: IDLCR8ASCII - to read the contents of the HDF file into session ; memory. The output from IDLCR8ASCII is then used in the call to: ; IDLCR8HDF - which performs the QA gtver='geoms_tools version 2.17, 2024-11-26' gt_error_code=3 ;Integer to identify return error code (3 = Invalid input) par1chk=0 & parchk=0 qhs1error=0B IF N_PARAMS() GE 1 THEN BEGIN ;test first parameter only ;Test parameter data type intype0=SIZE(optqh,/TYPE) IF ((intype0 GE 1) AND (intype0 LE 5)) OR ((intype0 GE 12) AND (intype0 LE 15)) THEN BEGIN ;input value is numeric so test for 0/1 optqh=FIX(optqh) IF (optqh EQ 0) OR (optqh EQ 1) THEN BEGIN par1chk=1 & parchk=1 ENDIF ENDIF ENDIF IF (N_PARAMS() GE 3) AND (parchk EQ 1) THEN BEGIN ;test remaining parameters ;Initialize input parameter test criteria - will be non-zero for valid input tavcnt=0 & hdfcnt=0 & qhdir='' ;Test parameter data types intype1=SIZE(tav,/TYPE) intype2=SIZE(hdfqh,/TYPE) IF N_PARAMS() GE 4 THEN intype3=SIZE(outdir,/TYPE) ELSE intype3=-1 IF (intype1 EQ 7) AND (intype2 EQ 7) THEN BEGIN ;string input, so check for valid filenames ;check TAV file is present tav=FILE_SEARCH(tav,/TEST_READ,/TEST_REGULAR,/FULLY_QUALIFY_PATH,COUNT=tavcnt) ;create an array of valid HDF filenames hdfqh=FILE_SEARCH(hdfqh,/TEST_READ,/TEST_REGULAR,/FULLY_QUALIFY_PATH,COUNT=hdfcnt) ENDIF IF (intype3 NE 7) AND (intype3 NE -1) THEN parchk=0 $ ;invalid input for 4th parameter ELSE IF intype3 EQ 7 THEN BEGIN ;check output directory exists dirtest=FILE_TEST(outdir,/DIRECTORY) IF (dirtest[0] EQ 1) AND (N_ELEMENTS(dirtest) EQ 1) THEN $ qhdir=FILE_SEARCH(outdir,/FULLY_QUALIFY_PATH,/MARK_DIRECTORY) $ ELSE parchk=0 END IF (tavcnt EQ 1) AND (hdfcnt GE 1) AND (parchk EQ 1) THEN gt_error_code=0 ;valid input ENDIF IF KEYWORD_SET(o1) THEN o4='DTFVOK' ELSE o4='0' ;relaxes checks on DATETIME values in idlcr8hdf call IF KEYWORD_SET(o2) THEN notc=1 ELSE notc=0 ;changes Required DATA_TEMPLATE comment for harmonization IF KEYWORD_SET(o3) THEN nodfvinc=1 ELSE nodfvinc=0 ;does not increment the DATA_FILE_VERSION IF gt_error_code EQ 3 THEN BEGIN PRINT,gtver & PRINT,'' IF par1chk EQ 0 THEN BEGIN itxt='geoms_tools expects the first input to be an integer value defining the' itxt=itxt+' testing mode: 0=QA, 1=Harmonization' PRINT,itxt ENDIF ELSE BEGIN IF optqh EQ 0 THEN itxt='QA mode' ELSE itxt='Harmonization mode' PRINT,'geoms_tools expects the following inputs when running in '+itxt PRINT,' * A scalar string containing the Table Attribute Values filename' PRINT,' * HDF file(s) in the form of -' PRINT,' a) a scalar string containing a single HDF file, or' PRINT,' b) a scalar string containing a filespec, or' PRINT,' c) a string array containing a list of HDF files' IF optqh EQ 1 THEN BEGIN itxt='(Optional - if not included Harmonized HDF files will not be created)' PRINT,' * An Output Directory '+itxt itxt='(Optional - if not included Harmonized HDF files will be in the same' PRINT,' * An Output Format

'+itxt PRINT,' format as the input file)' ENDIF ENDELSE PRINT,'Exit Status = 3 (incorrect input parameters)' ENDIF ELSE BEGIN ;Commence QA or Harmonization on HDF files IF optqh EQ 0 THEN BEGIN ;QA Option logname='idlcr8qa.log' lgtxt='Quality Assurance' gtcode=['0 (Passed QA)','1 (GEOMS compliant with non-standard components)','2 (Failed QA)'] gttxt='(0=PASS, 1=GEOMS compliant with non-standard components, 2=FAIL)' n_lc=1 ;number of times to call idlcr8hdf ENDIF ELSE BEGIN ;Harmonization Option logname='idlcr8hdf.log' lgtxt='Harmonization' gtcode=['0 (Passed Harmonization)','1 (Failed Harmonization)','1 (Failed Harmonization)'] gttxt='(0=SUCCESS, 1=FAIL)' dontuse='Reading input Global' ;Will call idlcr8hdf twice if wanting to make an HDF file, and file requires modification ;(note - if file meets all GEOMS criteria, a new file will not be generated) IF intype3 EQ -1 THEN n_lc=1 ELSE n_lc=2 ENDELSE h_typeall='' ;set default for output file format (assumes same as input) ;Is the dataformat parameter defined as an input parameter? IF N_PARAMS() GE 5 THEN BEGIN dataformat=STRTRIM(STRUPCASE(dataformat),2) IF (dataformat EQ 'H4') OR (dataformat EQ 'H5') OR (dataformat EQ 'NC') THEN $ h_typeall=dataformat ENDIF FOR i=0,hdfcnt-1 DO BEGIN hdfdir=FILE_DIRNAME(hdfqh[i],/MARK_DIRECTORY) ;directory containing HDF file ;assign HDF input directory as the output directory if not given as an input parameter IF intype3 EQ -1 THEN qhdir=hdfdir ;Create log file name - changing the extension of the HDF filename to .log qhlog=STRMID(hdfqh[i],0,STRPOS(hdfqh[i],'.',/REVERSE_SEARCH))+'.log' ;Change to new output directory if it is different to the HDF file directory IF qhdir NE hdfdir THEN qhlog=qhdir+FILE_BASENAME(qhlog) newlname=qhlog ;log file name may change to match any new HDF file created OPENW,lu,qhlog,/GET_LUN PRINTF,lu,gtver PRINTF,lu,'GEOMS '+lgtxt+' Tool' PRINTF,lu,'Maintained by the ESA Validation Data Center, EVDC' PRINTF,lu,'https://evdc.esa.int/documentation/geoms/' PRINTF,lu,'' PRINTF,lu,'Log started on '+SYSTIME(0) PRINTF,lu,'Result of performing '+lgtxt+' on '+hdfqh[i] PRINTF,lu,gttxt ;Read contents of HDF file into session memory reterr='' ;Return string containing any errors encountered in procedure calls FILE_DELETE,hdfdir+'idlcr8ascii.log',/QUIET IDLCR8ASCII,hdfqh[i],ga,sds,reterr,/L ;Check for read error IF STRMID(reterr,0,2) EQ 'Un' THEN BEGIN ;Unable to read HDF file PRINTF,lu,reterr PRINTF,lu,'' PRINTF,lu,gtcode[2] gt_error_code=FIX(STRMID(gtcode[2],0,1))*(-1) ;Make negative so no other checks are done ENDIF ELSE BEGIN ;File read OK so continue with checks IF h_typeall NE '' THEN h_type=h_typeall $ ;output format determined by program call ELSE IF H5F_IS_HDF5(hdfqh[i]) THEN h_type='H5' $ ELSE IF HDF_ISHDF(hdfqh[i]) THEN h_type='H4' ELSE h_type='NC' ;Check whether HDF4, HDF5 or netCDF file to determine default output format of the HDF file lc=0 & newstr='' & list_changes=[''] & gf_err=0B & cur_tav=0B WHILE lc LT n_lc DO BEGIN IF (lc EQ 0) AND (optqh EQ 1) THEN BEGIN ;If performing Harmonization, need to determine a new DATA_FILE_VERSION (DFV) value ;according to the rule defined by the geoms_iteration value geoms_iteration=1 ;identifies which check to perform in GEOMS_VERSION_CHECK newdfv=STRARR(3) ;returns DFV index in ga, new DFV value and label, and boolean to ;indicate whether DFV is present in the original Global Attributes or not GEOMS_VERSION_CHECK,geoms_iteration,ga,newdfv,nodfvinc ;Also need to check for one-off corrections concatenated to the TAV file GEOMS_ONE_OFF_CHECK,tav,ga,list_changes ;Also need to perform corrections based on revised templates GEOMS_TEMPLATE_CHANGE,ga,sds,list_changes ;Also need to check that dependent variables are not list before the variables they are dependent on DEPENDENCY_ORDER,ga,sds,list_changes ENDIF ;Read idlcr8ascii.log contents into string array dum='' & la=0 & lq=0 OPENR,hu,hdfdir+'idlcr8ascii.log',/GET_LUN WHILE NOT EOF(hu) DO BEGIN READF,hu,dum & la=la+1 ENDWHILE FREE_LUN,hu qh_s1=STRARR(la) IF lc EQ n_lc-1 THEN OPENR,hu,hdfdir+'idlcr8ascii.log',/GET_LUN,/DELETE $ ELSE OPENR,hu,hdfdir+'idlcr8ascii.log',/GET_LUN READF,hu,qh_s1 FREE_LUN,hu IF optqh EQ 0 THEN BEGIN ;QA option so identify FILE_META_VERSION ;Identify TAV files needed for checks based on FILE_GENERATION_DATE and FILE_META_VERSION values TAV_FILES_FOR_QA, tav, ga, lu, tavlist, gf_err, cur_tav ti=WHERE(tavlist NE '',n_tav_chks) ;number of TAV files to do QA checks on (either 1 or 2) ;Change INFORMATION text in idlcr8ascii.log entries to ERROR ei=WHERE(STRPOS(qh_s1,'INFORMATION:') NE -1,ecnt) IF ecnt NE 0 THEN BEGIN qh_s1[ei]=' ERROR:'+STRMID(qh_s1[ei],14) qhs1error=1B ENDIF ENDIF ;Check for 'Calibrated Data' messages ci=WHERE(STRPOS(STRLOWCASE(qh_s1),'calibrated') NE -1,ccnt) IF (ccnt NE 0) AND (optqh EQ 0) THEN BEGIN ;change messages for QA output FOR j=0,ccnt-1 DO BEGIN CASE 1 OF STRPOS(qh_s1[ci[j]],'VAR_DATA_TYPE') NE -1: qh_s1[ci[j]:ci[j]+1]='' ;message removed ELSE: qh_s1[ci[j]+1:ci[j]+2]='' ;only first line of message kept ENDCASE ENDFOR ENDIF ;Check for 'dimension ordering' message and change language for QA testing ci=WHERE(STRPOS(STRLOWCASE(qh_s1),'dimension ordering') NE -1,ccnt) IF (ccnt NE 0) AND (optqh EQ 0) THEN BEGIN FOR j=0,ccnt-1 DO BEGIN ipos=STRPOS(qh_s1[ci[j]],'changed to') qh_s1[ci[j]]=STRMID(qh_s1[ci[j]],0,ipos)+'does not'+STRMID(qh_s1[ci[j]],ipos+10) ENDFOR ENDIF ;Check for 'language convention' message and change language for QA testing ci=WHERE(STRPOS(STRLOWCASE(qh_s1),'language convention') NE -1,ccnt) IF (ccnt NE 0) AND (optqh EQ 0) THEN BEGIN FOR j=0,ccnt-1 DO BEGIN ipos=STRPOS(qh_s1[ci[j]],'changed to') qh_s1[ci[j]]=STRMID(qh_s1[ci[j]],0,ipos)+'does not'+STRMID(qh_s1[ci[j]],ipos+10) ENDFOR ENDIF ;Check for 'correct listing order' message and change INFORMATION to ERROR for Harmonization ci=WHERE(STRPOS(STRLOWCASE(qh_s1),'correct listing order') NE -1,ccnt) IF (ccnt NE 0) AND (optqh EQ 1) THEN BEGIN FOR j=0,ccnt-1 DO BEGIN ipos=STRPOS(qh_s1[ci[j]],'INFORMATION:') qh_s1[ci[j]]=STRMID(qh_s1[ci[j]],0,ipos)+'ERROR:'+STRMID(qh_s1[ci[j]],ipos+12) ENDFOR ENDIF ;Delete log file if it is present in the output directory (a new file will be created by idlcr8hdf) FILE_DELETE,qhdir+logname,/QUIET IF optqh EQ 0 THEN BEGIN FOR j=0,n_tav_chks-1 DO BEGIN IF o4 EQ '0' THEN BEGIN IF h_type EQ 'H5' THEN IDLCR8HDF,ga,sds,tavlist[j],qhdir,reterr,/H5,/QA $ ELSE IF h_type EQ 'NC' THEN IDLCR8HDF,ga,sds,tavlist[j],qhdir,reterr,/NC,/QA $ ELSE IDLCR8HDF,ga,sds,tavlist[j],qhdir,reterr,/QA ENDIF ELSE BEGIN IF h_type EQ 'H5' THEN IDLCR8HDF,ga,sds,tavlist[j],qhdir,reterr,/H5,/QA,/D $ ELSE IF h_type EQ 'NC' THEN IDLCR8HDF,ga,sds,tavlist[j],qhdir,reterr,/NC,/QA,/D $ ELSE IDLCR8HDF,ga,sds,tavlist[j],qhdir,reterr,/QA,/D ENDELSE ENDFOR ENDIF ELSE IF (intype3 EQ -1) OR (lc EQ 0) THEN BEGIN ;Do not create a new HDF file IF o4 EQ '0' THEN BEGIN IF h_type EQ 'H5' THEN IDLCR8HDF,ga,sds,tav,qhdir,reterr,/L,/H5,/NOHDF $ ELSE IF h_type EQ 'NC' THEN IDLCR8HDF,ga,sds,tav,qhdir,reterr,/L,/NC,/NOHDF $ ELSE IDLCR8HDF,ga,sds,tav,qhdir,reterr,/L,/NOHDF ENDIF ELSE BEGIN IF h_type EQ 'H5' THEN IDLCR8HDF,ga,sds,tav,qhdir,reterr,/L,/H5,/NOHDF,/D $ ELSE IF h_type EQ 'NC' THEN IDLCR8HDF,ga,sds,tav,qhdir,reterr,/L,/NC,/NOHDF,/D $ ELSE IDLCR8HDF,ga,sds,tav,qhdir,reterr,/L,/NOHDF,/D ENDELSE ENDIF ELSE BEGIN IF o4 EQ '0' THEN BEGIN IF h_type EQ 'H5' THEN IDLCR8HDF,ga,sds,tav,qhdir,reterr,/L,/H5 $ ELSE IF h_type EQ 'NC' THEN IDLCR8HDF,ga,sds,tav,qhdir,reterr,/L,/NC $ ELSE IDLCR8HDF,ga,sds,tav,qhdir,reterr,/L ENDIF ELSE BEGIN IF h_type EQ 'H5' THEN IDLCR8HDF,ga,sds,tav,qhdir,reterr,/L,/H5,/D $ ELSE IF h_type EQ 'NC' THEN IDLCR8HDF,ga,sds,tav,qhdir,reterr,/L,/NC,/D $ ELSE IDLCR8HDF,ga,sds,tav,qhdir,reterr,/L,/D ENDELSE ENDELSE ;Read contents of idlcr8hdf logfile into a string array OPENR,hu,qhdir+logname,/GET_LUN WHILE NOT EOF(hu) DO BEGIN READF,hu,dum & lq=lq+1 ENDWHILE FREE_LUN,hu qh_s2=STRARR(lq) OPENR,hu,qhdir+logname,/GET_LUN,/DELETE READF,hu,qh_s2 FREE_LUN,hu ;Append list_changes information to qh_s2 array IF list_changes[0] NE '' THEN BEGIN qh_s2=[qh_s2[0:5],list_changes,qh_s2[6:N_ELEMENTS(qh_s2)-1]] lq=N_ELEMENTS(qh_s2) ENDIF IF optqh EQ 0 THEN BEGIN ;Edit messages regarding the TAV file types tavi=WHERE(STRMID(qh_s2,0,14) EQ 'Input TAV File',tavcnt) tav_name=STRARR(tavcnt) FOR j=0,tavcnt-1 DO tav_name[j]=STRMID(qh_s2[tavi[j]],15) tav_inputs=['Input TAV file calculated from FILE_GENERATION_DATE:', $ 'Input TAV file calculated from FILE_META_VERSION:', $ 'Input TAV file calculated from FILE_GENERATION_DATE and FILE_META_VERSION:'] CASE 1 OF (~gf_err) AND (~cur_tav) AND (n_tav_chks EQ 1): BEGIN ;TAV files determined from FGD and FMV are the same qh_s2[tavi[0]]=tav_inputs[2]+tav_name[0] END (~gf_err) AND (~cur_tav) AND (n_tav_chks EQ 2): BEGIN ;TAV files determined from FGD and FMV are different FOR j=0,1 DO qh_s2[tavi[j]]=tav_inputs[j]+tav_name[j] END n_tav_chks EQ 2: BEGIN ;TAV files are input TAV (error generated) and one from FMV qh_s2[tavi[1]]=tav_inputs[1]+tav_name[1] END ELSE: ;Only check is with input TAV file so no change needed to qh_s2 ENDCASE IF tavcnt EQ 2 THEN BEGIN tav_err_chk=BYTARR(2) tavrv=STRARR(2) FOR j=0,1 DO BEGIN IF STRMID(tav_name[j],STRLEN(tav_name[j])-8) EQ '_idl.dat' THEN $ tnh=STRMID(tav_name[j],0,STRLEN(tav_name[j])-8) $ ELSE tnh=tav_name[j] tavrv[j]=STRMID(tnh,STRPOS(tnh,'_',/REVERSE_SEARCH)+1,6) ENDFOR ;Check QA using FGD or input TAV for errors FOR j=0,tavi[1]-1 DO $ IF STRMID(STRTRIM(qh_s2[j],2),0,5) EQ 'ERROR' THEN tav_err_chk[0]=1B ;Check QA using FMV TAV for errors FOR j=tavi[1],lq-1 DO $ IF STRMID(STRTRIM(qh_s2[j],2),0,5) EQ 'ERROR' THEN tav_err_chk[1]=1B IF tav_err_chk[1] THEN BEGIN IF tav_err_chk[0] THEN $ qh_s2[tavi[1]+1]=' ERROR: QA errors also generated using '+tavrv[1]+' definitions' $ ELSE BEGIN qh_s2[tavi[1]+1]=' ERROR: QA errors generated using '+tavrv[1]+' definitions.' qh_s2[tavi[1]+1]=qh_s2[tavi[1]+1]+' Please update first FILE_META_VERSION value to '+ $ tavrv[0] ENDELSE ENDIF ELSE BEGIN IF tav_err_chk[0] THEN BEGIN qh_s2[tavi[1]+1]=' ERROR: Definitions based on FILE_META_VERSION='+tavrv[1]+' are out-of-date.' qh_s2[tavi[1]+1]=qh_s2[tavi[1]+1]+' Please update first FILE_META_VERSION value to '+ $ tavrv[0]+' and use these TAV definitions' ENDIF ELSE BEGIN IF qhs1error THEN $ qh_s2[tavi[1]+1]=' ERROR: QA errors also generated using '+tavrv[1]+' definitions' $ ELSE qh_s2[tavi[1]+1]=' INFORMATION: GEOMS file passed using '+tavrv[1]+' definitions' ENDELSE ENDELSE ;Don't want to keep additional mesasages from 2nd TAV check qh_s2=qh_s2[0:tavi[1]+1] lq=N_ELEMENTS(qh_s2) ENDIF ENDIF lc=lc+1 IF (optqh EQ 1) AND (intype3 NE -1) AND (lc EQ 1) THEN BEGIN ;Check whether information and/or error text is present in log files ai=WHERE(STRMID(qh_s1,0,13) EQ ' INFORMATION',acnt) bi=WHERE(STRMID(qh_s2,0,13) EQ ' INFORMATION',bcnt) ci=WHERE(STRMID(qh_s2,0,7) EQ ' ERROR',ccnt) di=WHERE(STRMID(qh_s1,0,7) EQ ' ERROR',dcnt) IF ((acnt EQ 0) AND (bcnt LE 1) AND (newdfv[2] EQ '1')) OR (ccnt NE 0) OR (dcnt NE 0) THEN BEGIN lc=n_lc ;either GEOMS compliant or non-harmonizable so don't create new file FILE_DELETE,hdfdir+'idlcr8ascii.log',/QUIET ;delete idlcr8ascii.log ENDIF ELSE BEGIN ;Requires harmonization - do the following ;1. Extract File Archive Date and FILE_GENERATION_DATE for SOURCE_PRODUCT entry ;2. Reset FILE_GENERATION_DATE for new file ;3. Update DATA_FILE_VERSION according to newdfv values ;4. Determine the new DF and log filenames ;5. Append to or create SOURCE.PRODUCT attribute ;Extract File Archive Date (only use if original FILE_GENERATION_DATE is not present) fi=FILE_INFO(hdfqh[i]) fmin=LONG(fi.mtime/60L) & fsec=LONG(fi.mtime MOD 60L) res=JULIAN_DATE([1970,1,1,0,fmin,fsec]) ;fdate=' with File Archive Date of '+JDF_2_DATETIME(res,/S) fdate=JDF_2_DATETIME(res,/S) ;Look for FILE_GENERATION_DATE mjd2k=SYSTIME(/Julian,/UTC)-2451544.5D mjds=JDF_2_DATETIME(mjd2k,/M,/S) ;returns datetime in ISO8601 format fgi=WHERE(STRMID(STRUPCASE(ga),0,20) EQ 'FILE_GENERATION_DATE',fgcnt) IF fgcnt NE 0 THEN BEGIN ;reset FILE_GENERATION_DATE value res=STRSPLIT(ga[fgi[0]],' =',/EXTRACT,COUNT=nres) IF nres EQ 2 THEN fdate=res[1] ;fdate=' with FILE_GENERATION_DATE='+res[1] ;Reset FILE_GENERATION_DATE with curent Date/Time ga[fgi[0]]='FILE_GENERATION_DATE='+mjds ENDIF ELSE BEGIN ;append FILE_GENERATION_DATE attribute to ga array fgd_att='FILE_GENERATION_DATE='+mjds ga=[ga,fgd_att] ENDELSE ;Determine FILE_ACCESS value for SOURCE_PRODUCT entry source_archive='UNKNOWN' fai=WHERE(STRMID(STRUPCASE(ga),0,11) EQ 'FILE_ACCESS',facnt) IF facnt NE 0 THEN BEGIN ;Extract first value res=STRSPLIT(ga[fai[0]],' =;',/EXTRACT,COUNT=nres) IF nres GE 2 THEN source_archive=STRUPCASE(res[1]) ENDIF ;Update DATA_FILE_VERSION in ga array for input to idlcr8hdf ga[newdfv[0]]=newdfv[1] res=STRSPLIT(newdfv[1],'=',/EXTRACT) dfv=res[1] ;Look for new filename in log file orifname=FILE_BASENAME(hdfqh[i]) newfname=orifname ;default is that filename remains the same fi=WHERE(STRMID(qh_s2,2,32) EQ 'INFORMATION: Filename determined',fcnt) IF fcnt NE 0 THEN BEGIN ;new filename is first entry in the next array res=STRSPLIT(qh_s2[fi[0]+1L],' ',/EXTRACT) newfname=res[0] ENDIF ;Update the DATA_FILE_VERSION portion of the filename and create new log file name CASE 1 of h_type EQ 'H4': h_ext='.hdf' h_type EQ 'NC': h_ext='.nc' ELSE: h_ext='.'+STRLOWCASE(h_type) ;.h5 ENDCASE newfname=STRMID(newfname,0,STRPOS(newfname,'_',/REVERSE_SEARCH)+1)+dfv+h_ext newlname=qhdir+STRMID(newfname,0,STRPOS(newfname,'_',/REVERSE_SEARCH)+1)+dfv+'.log' ;Create SOURCE.PRODUCT entry spdata=source_archive+';'+orifname+';'+fdate ;spdata='File '+newfname+' with FILE_GENERATION_DATE='+mjds+' contains Metadata ' ;spdata=spdata+'and Data that originated from file '+orifname+fdate+'. It has ' ;spdata=spdata+'been modified to comply with existing GEOMS compliance rules ' ;spdata=spdata+'using '+gtver+' in Harmonization mode.' ;Test structure for SOURCE.PRODUCT label pchk=PTR_VALID(sds.va_l) ;Check VAR_NAME values spf=INTARR(2)-1L s_dim=SIZE(pchk,/DIMENSIONS) ;1st-D = number of datasets, 2nd-D = number of atributes FOR j=0,s_dim[0]-1 DO BEGIN FOR k=0,s_dim[1]-1 DO BEGIN IF pchk[j,k] EQ 1 THEN BEGIN vavtest=STRTRIM(STRUPCASE(*sds[j,k].va_v),2) valtest=STRTRIM(STRUPCASE(*sds[j,k].va_l),2) IF valtest EQ 'VAR_NAME' THEN IF vavtest EQ 'SOURCE.PRODUCT' THEN spf=[j,k] ENDIF ENDFOR ENDFOR IF spf[0] NE -1 THEN BEGIN ;SOURCE_PRODUCT is already present so test to see if existing data is of type ;string and is 1-D, if so, append new data value. If not, then cannot harmonize ;dataset so append error to output log dval=*sds[spf[0],0].data dtype=SIZE(dval,/TYPE) n_dim=SIZE(dval,/N_DIMENSIONS) IF (dtype EQ 7) AND (n_dim LE 1) THEN BEGIN ;Existing Dataset is valid ;add new data value dval=[dval,spdata] sds[spf[0],0].data=PTR_NEW(dval) ;update VAR_SIZE and check VAR_DEPEND=INDEPENDENT and VAR_DATA_TYPE=STRING new_vs=N_ELEMENTS(dval) FOR k=0,s_dim[1]-1 DO BEGIN IF pchk[spf[0],k] EQ 1 THEN BEGIN valtest=STRTRIM(STRUPCASE(*sds[spf[0],k].va_l),2) IF valtest EQ 'VAR_SIZE' THEN BEGIN vavtest=LONG(*sds[spf[0],k].va_v)+1L sds[spf[0],k].va_v=PTR_NEW(vavtest) ENDIF ELSE IF valtest EQ 'VAR_DEPEND' THEN BEGIN vavtest=STRTRIM(STRUPCASE(*sds[spf[0],k].va_v),2) IF vavtest NE 'INDEPENDENT' THEN sds[spf[0],k].va_v=PTR_NEW('INDEPENDENT') ENDIF ELSE IF valtest EQ 'VAR_DATA_TYPE' THEN BEGIN vavtest=STRTRIM(STRUPCASE(*sds[spf[0],k].va_v),2) IF vavtest NE 'STRING' THEN sds[spf[0],k].va_v=PTR_NEW('STRING') ENDIF ENDIF ENDFOR newstr=' INFORMATION: New Comment appended to the SOURCE.PRODUCT Dataset' ENDIF ELSE BEGIN ;Cannot Harmonize dataset so append error to log lc=n_lc newstr=' ERROR: Harmonization Information cannot be written to ' newstr=errstr+' the SOURCE_PRODUCT Dataset' ENDELSE ENDIF ELSE BEGIN ;need to create new variable attribute and data ;First update DATA_VARIABLES dvi=WHERE(STRMID(STRUPCASE(ga),0,14) EQ 'DATA_VARIABLES',dvcnt) IF dvcnt NE 0 THEN BEGIN sppos=STRPOS(ga[dvi[0]],'SOURCE.PRODUCT') IF sppos EQ -1 THEN $ ;confirm SOURCE.PRODUCT not present so add ga[dvi[0]]=ga[dvi[0]]+';SOURCE.PRODUCT' ENDIF ;Then create new Variable Attribute and Data splab=['VAR_NAME','VAR_DESCRIPTION','VAR_NOTES','VAR_DEPEND','VAR_SIZE',$ 'VAR_DATA_TYPE','VAR_UNITS','VAR_SI_CONVERSION','VAR_VALID_MIN',$ 'VAR_VALID_MAX','VAR_FILL_VALUE'] vdval='Information relevant to the source history of the Metadata and Data' spval=['SOURCE.PRODUCT',vdval,'','INDEPENDENT','1','STRING','','','','',''] ;Create new structure to hold existing and new Metadata and Data IF N_ELEMENTS(nds) NE 0 THEN PTR_FREE,nds.va_l,nds.va_v,nds.data ;Define the HDF SDS storage structure nds_set={va_l: PTR_NEW(), $ ;Variable Attribute Labels va_v: PTR_NEW(), $ ;Variable Attribute Values data: PTR_NEW()} ;SD data array IF s_dim[1] LT 11 THEN n_dim=11 ELSE n_dim=s_dim[1] nds=REPLICATE(nds_set, s_dim[0]+1L, n_dim) ;Write existing data to new structure FOR j=0,s_dim[0]-1 DO BEGIN FOR k=0,s_dim[1]-1 DO BEGIN IF pchk[j,k] EQ 1 THEN BEGIN vavtest=*sds[j,k].va_v valtest=*sds[j,k].va_l nds[j,k].va_v=PTR_NEW(vavtest) & nds[j,k].va_l=PTR_NEW(valtest) IF k EQ 0 THEN BEGIN dval=*sds[j,k].data & nds[j,k].data=PTR_NEW(dval) ENDIF ENDIF ENDFOR ENDFOR ;Write SOURCE.PRODUCT Metadata and Data to structure FOR k=0,N_ELEMENTS(splab)-1 DO BEGIN nds[s_dim[0],k].va_v=PTR_NEW(spval[k]) nds[s_dim[0],k].va_l=PTR_NEW(splab[k]) IF k EQ 0 THEN nds[s_dim[0],k].data=PTR_NEW(spdata) ENDFOR ;Finally write data back to the sds structure PTR_FREE,sds.va_l,sds.va_v,sds.data pchk=PTR_VALID(nds.va_l) ;Define the HDF SDS storage structure sds_set={va_l: PTR_NEW(), $ ;Variable Attribute Labels va_v: PTR_NEW(), $ ;Variable Attribute Values data: PTR_NEW()} ;SD data array sds=REPLICATE(nds_set, s_dim[0]+1L, n_dim) ;Write existing and new data to original structure FOR j=0,s_dim[0] DO BEGIN FOR k=0,n_dim-1 DO BEGIN IF pchk[j,k] EQ 1 THEN BEGIN vavtest=*nds[j,k].va_v valtest=*nds[j,k].va_l sds[j,k].va_v=PTR_NEW(vavtest) & sds[j,k].va_l=PTR_NEW(valtest) IF k EQ 0 THEN BEGIN dval=*nds[j,k].data & sds[j,k].data=PTR_NEW(dval) ENDIF ENDIF ENDFOR ENDFOR PTR_FREE,nds.va_l,nds.va_v,nds.data newstr=' INFORMATION: SOURCE.PRODUCT Metadata and Data added to the HDF file' ENDELSE ENDELSE ENDIF IF lc EQ n_lc THEN BEGIN ;Final or only run so prepare log output ;get rid of unwanted strings in qh_s1 IF la LE 6 THEN BEGIN ;i.e. no extra comments qh_s1=[''] & la=0 ENDIF ELSE BEGIN ;remove start and end strings qh_s1=qh_s1[3:la-3] ;check for non-blank lines gi=WHERE(qh_s1 NE '',la) IF la NE 0 THEN qh_s1=qh_s1[gi] ENDELSE IF optqh EQ 1 THEN BEGIN ;i.e. idlcr8hdf.log ;get rid of unwanted strings in qh_s2 qh_s2=qh_s2[2:lq-3] lq=lq-4 di=WHERE(STRPOS(qh_s2,dontuse) NE -1,dcnt) IF dcnt EQ 1 THEN BEGIN IF di[0] EQ lq-1 THEN qh_s2=qh_s2[0:di[0]-1] $ ELSE qh_s2=[qh_s2[0:di[0]-1],qh_s2[di[0]+1:lq-1]] lq=lq-1 ENDIF ENDIF IF qh_s2[lq-1] EQ '' THEN BEGIN qh_s2=qh_s2[0:lq-2] & lq=lq-1 ENDIF ;Move Template file identifier line if present ei=WHERE(STRPOS(qh_s2,'Required Metadata Template:') NE -1,ecnt) IF ecnt NE 0 THEN BEGIN IF notc EQ 1 THEN $ ;change Required DATA_TEMPLATE comment in the log file qh_s2[ei[0]]=STRMID(qh_s2[ei[0]],0,STRPOS(qh_s2[ei[0]],':',/REVERSE_SEARCH)+2)+ $ 'DATA_TEMPLATE filename not valid for Harmonized files' di=WHERE(STRPOS(qh_s2,'Input TAV File:') NE -1) IF lq-ei[0] LE 1 THEN qh_s2=[qh_s2[0:di[0]+1],qh_s2[ei[0]],qh_s2[di[0]+2:ei[0]-1]] $ ELSE $ qh_s2=[qh_s2[0:di[0]+1],qh_s2[ei[0]],qh_s2[di[0]+2:ei[0]-1],qh_s2[ei[0]+1:lq-1]] ENDIF IF la NE 0 THEN BEGIN ;combine the log strings IF STRPOS(qh_s1[la-1],'heap structure created') NE -1 THEN $ IF la GT 1 THEN qh_s1=qh_s1[0:la-2] ELSE qh_s1=[''] IF qh_s2[2] NE '' THEN qh_s1=['',qh_s1] qh_str=[qh_s2[0:2],qh_s1] IF qh_s2[3] NE '' THEN si=3 ELSE si=4 IF lq GE 5 THEN qh_str=[qh_str,qh_s2[si:lq-1]] ENDIF ELSE qh_str=qh_s2 ;Append SOURCE.PRODUCT comment to log if present n_str=N_ELEMENTS(qh_str) IF newstr NE '' THEN BEGIN qh_str=[qh_str[0:n_str-2],newstr,qh_str[n_str-1]] ENDIF IF qh_str[n_str-1L] EQ '' THEN qh_str=qh_str[0:n_str-2L] ;Search contents of log file to check for success or failure (transfer to qhlog) ai=WHERE(STRMID(qh_str,0,8) EQ ' ERROR ',acnt) bi=WHERE(STRMID(qh_str,0,8) EQ ' ERROR:',bcnt) ci=WHERE(STRMID(qh_str,0,8) EQ ' NON-ST',ccnt) ;One last check for more than one consecutive empty line (happens when checking two different TAVs (20201119)) qh_strx=[qh_str[0]] FOR j=1,N_ELEMENTS(qh_str)-1 DO BEGIN qh_strm0=qh_str[j] & qh_strm1=qh_str[j-1] IF (qh_strm0 NE '') OR ((qh_strm0 EQ '') AND (qh_strm1 NE '')) THEN qh_strx=[qh_strx,qh_strm0] ENDFOR qh_str=qh_strx ;PRINTF,lu,gttxt FOR j=0,N_ELEMENTS(qh_str)-1 DO PRINTF,lu,qh_str[j] PRINTF,lu,'' IF (acnt NE 0) OR (bcnt NE 0) OR (gf_err) THEN ei=2 $ ELSE IF ccnt NE 0 THEN ei=1 $ ELSE ei=0 PRINTF,lu,gtcode[ei] gt_error_code=FIX(STRMID(gtcode[ei],0,1)) ENDIF ENDWHILE ENDELSE FREE_LUN,lu ;If a new HDF file has been created then rename log file to match new HDF filename IF newlname NE qhlog THEN FILE_MOVE,qhlog,newlname,/OVERWRITE ENDFOR ENDELSE RETURN, gt_error_code END ;Function geoms_tools