---
title: "surface-insitu"
output: rmarkdown::html_vignette
vignette: >
  %\VignetteIndexEntry{surface-insitu}
  %\VignetteEngine{knitr::rmarkdown}
  %\VignetteEncoding{UTF-8}
---



```{r, include = FALSE}
knitr::opts_chunk$set(
  collapse = TRUE,
  comment = "#>"
)
```

```{r setup}
library(rtorf)
library(data.table)
```


```{r read1, eval = F}

cate = c("aircraft-pfp",
         "aircraft-insitu",
         "aircraft-flask",
         "surface-insitu",
         "surface-flask", 
         "surface-pfp",   
         "tower-insitu",  
         "aircore",       
         "shipboard-insitu",
         "shipboard-flask") 

obs <- "Z:/obspack/obspack_ch4_1_GLOBALVIEWplus_v5.1_2023-03-08/data/nc/"
index <- obs_summary(obs = obs, 
                     categories = cate)
```

```
#> Number of files of index: 429
#>               sector     N
#>               <char> <int>
#>  1:     aircraft-pfp    40
#>  2:  aircraft-insitu    15
#>  3:    surface-flask   106
#>  4:   surface-insitu   174
#>  5:   aircraft-flask     4
#>  6:          aircore     1
#>  7:      surface-pfp    33
#>  8:     tower-insitu    51
#>  9:  shipboard-flask     4
#> 10: shipboard-insitu     1
#> 11:    Total sectors   429
#> Detected 190 files with agl
#> Detected 239 files without agl
```

Now we read the `surface-insitu` using the function `obs_read_nc`. 
`solar_time` is  included for surface, so we `TRUE` that argument.

```{r readnc, eval = F}
datasetid <- "surface-insitu"
df <- obs_read_nc(index = index,
                  categories = datasetid,
                  solar_time = TRUE,
                  verbose = TRUE)

```

```
#> Searching surface-insitu...
#> 1: ch4_abt_surface-insitu_6_allvalid.nc
#> 2: ch4_alt_surface-insitu_6_allvalid.nc
#> 3: ch4_amy_surface-insitu_61_allvalid.nc
#> 4: ch4_bck_surface-insitu_6_allvalid.nc
#> 5: ch4_bhd_surface-insitu_15_baseline.nc
#> 6: ch4_bir_surface-insitu_442_allvalid-10magl.nc
#> 7: ch4_bir_surface-insitu_442_allvalid-50magl.nc
#> 8: ch4_bir_surface-insitu_442_allvalid-75magl.nc
#> 9: ch4_blk_surface-insitu_6_allvalid.nc
#> 10: ch4_bra_surface-insitu_6_allvalid.nc
#> 11: ch4_brw_surface-insitu_1_allvalid.nc
#> 12: ch4_buc_surface-insitu_112_allvalid.nc
#> 13: ch4_carl_surface-insitu_60_allvalid-9magl.nc
#> 14: ch4_cbw_surface-insitu_118_allvalid.nc
#> 15: ch4_cby_surface-insitu_6_allvalid.nc
#> 16: ch4_cdl_surface-insitu_6_allvalid.nc
#> 17: ch4_chl_surface-insitu_6_allvalid.nc
#> 18: ch4_chm_surface-insitu_6_allvalid.nc
#> 19: ch4_cmn_surface-insitu_106_allvalid.nc
#> 20: ch4_cmn_surface-insitu_442_allvalid-8magl.nc
#> 21: ch4_cps_surface-insitu_6_allvalid.nc
#> 22: ch4_cpt_surface-insitu_36_marine.nc
#> 23: ch4_cvo_surface-insitu_456_allvalid.nc
#> 24: ch4_dnh_surface-insitu_112_allvalid.nc
#> 25: ch4_egb_surface-insitu_6_allvalid.nc
#> 26: ch4_ena_surface-insitu_64_allvalid-10magl.nc
#> 27: ch4_esp_surface-insitu_6_allvalid.nc
#> 28: ch4_est_surface-insitu_6_allvalid.nc
#> 29: ch4_etl_surface-insitu_6_allvalid.nc
#> 30: ch4_fne_surface-insitu_6_allvalid.nc
#> 31: ch4_fort_surface-insitu_60_allvalid-128magl.nc
#> 32: ch4_fsd_surface-insitu_6_allvalid.nc
#> 33: ch4_gat_surface-insitu_442_allvalid-132magl.nc
#> 34: ch4_gat_surface-insitu_442_allvalid-216magl.nc
#> 35: ch4_gat_surface-insitu_442_allvalid-30magl.nc
#> 36: ch4_gat_surface-insitu_442_allvalid-341magl.nc
#> 37: ch4_gat_surface-insitu_442_allvalid-60magl.nc
#> 38: ch4_ghg03_surface-insitu_112_allvalid.nc
#> 39: ch4_ghg06_surface-insitu_112_allvalid.nc
#> 40: ch4_ghg09_surface-insitu_112_allvalid.nc
#> 41: ch4_ghg10_surface-insitu_112_allvalid.nc
#> 42: ch4_hct_surface-insitu_112_allvalid.nc
#> 43: ch4_hel_surface-insitu_442_allvalid-110magl.nc
#> 44: ch4_hnp_surface-insitu_6_allvalid.nc
#> 45: ch4_hobb_surface-insitu_60_allvalid-91magl.nc
#> 46: ch4_hpb_surface-insitu_442_allvalid-131magl.nc
#> 47: ch4_hpb_surface-insitu_442_allvalid-50magl.nc
#> 48: ch4_hpb_surface-insitu_442_allvalid-93magl.nc
#> 49: ch4_htm_surface-insitu_442_allvalid-150magl.nc
#> 50: ch4_htm_surface-insitu_442_allvalid-30magl.nc
#> 51: ch4_htm_surface-insitu_442_allvalid-70magl.nc
#> 52: ch4_inu_surface-insitu_6_allvalid.nc
#> 53: ch4_inx01_surface-insitu_60_allhours-10magl.nc
#> 54: ch4_inx01_surface-insitu_60_allhours-121magl.nc
#> 55: ch4_inx01_surface-insitu_60_allhours-40magl.nc
#> 56: ch4_inx02_surface-insitu_60_allhours-10magl.nc
#> 57: ch4_inx02_surface-insitu_60_allhours-136magl.nc
#> 58: ch4_inx02_surface-insitu_60_allhours-40magl.nc
#> 59: ch4_inx03_surface-insitu_60_allhours-10magl.nc
#> 60: ch4_inx03_surface-insitu_60_allhours-20magl.nc
#> 61: ch4_inx03_surface-insitu_60_allhours-40magl.nc
#> 62: ch4_inx03_surface-insitu_60_allhours-54magl.nc
#> 63: ch4_inx04_surface-insitu_60_allhours-60magl.nc
#> 64: ch4_inx05_surface-insitu_60_allhours-125magl.nc
#> 65: ch4_inx07_surface-insitu_60_allhours-21magl.nc
#> 66: ch4_inx07_surface-insitu_60_allhours-58magl.nc
#> 67: ch4_inx08_surface-insitu_60_allhours-41magl.nc
#> 68: ch4_inx09_surface-insitu_60_allhours-130magl.nc
#> 69: ch4_inx10_surface-insitu_60_allhours-40magl.nc
#> 70: ch4_inx11_surface-insitu_60_allhours-130magl.nc
#> 71: ch4_inx13_surface-insitu_60_allhours-87magl.nc
#> 72: ch4_inx14_surface-insitu_60_allhours-76magl.nc
#> 73: ch4_inx15_surface-insitu_60_allhours-75magl.nc
#> 74: ch4_ipr_surface-insitu_442_allvalid-100magl.nc
#> 75: ch4_ipr_surface-insitu_442_allvalid-40magl.nc
#> 76: ch4_ipr_surface-insitu_442_allvalid-60magl.nc
#> 77: ch4_izo_surface-insitu_27_allvalid.nc
#> 78: ch4_jfj_surface-insitu_442_allvalid-5magl.nc
#> 79: ch4_jfj_surface-insitu_5_allvalid.nc
#> 80: ch4_jgs_surface-insitu_61_allvalid.nc
#> 81: ch4_jue_surface-insitu_442_allvalid-120magl.nc
#> 82: ch4_jue_surface-insitu_442_allvalid-50magl.nc
#> 83: ch4_jue_surface-insitu_442_allvalid-80magl.nc
#> 84: ch4_kit_surface-insitu_442_allvalid-100magl.nc
#> 85: ch4_kit_surface-insitu_442_allvalid-200magl.nc
#> 86: ch4_kit_surface-insitu_442_allvalid-30magl.nc
#> 87: ch4_kit_surface-insitu_442_allvalid-60magl.nc
#> 88: ch4_kre_surface-insitu_442_allvalid-10magl.nc
#> 89: ch4_kre_surface-insitu_442_allvalid-125magl.nc
#> 90: ch4_kre_surface-insitu_442_allvalid-250magl.nc
#> 91: ch4_kre_surface-insitu_442_allvalid-50magl.nc
#> 92: ch4_krk_surface-insitu_118_allvalid.nc
#> 93: ch4_lew_surface-insitu_112_allvalid.nc
#> 94: ch4_lin_surface-insitu_442_allvalid-10magl.nc
#> 95: ch4_lin_surface-insitu_442_allvalid-2.5magl.nc
#> 96: ch4_lin_surface-insitu_442_allvalid-40magl.nc
#> 97: ch4_lin_surface-insitu_442_allvalid-98magl.nc
#> 98: ch4_llb_surface-insitu_6_allvalid.nc
#> 99: ch4_lmp_surface-insitu_442_allvalid-8magl.nc
#> 100: ch4_lut_surface-insitu_118_allvalid.nc
#> 101: ch4_lut_surface-insitu_442_allvalid-60magl.nc
#> 102: ch4_malj_surface-insitu_60_allvalid-134magl.nc
#> 103: ch4_mlo_surface-insitu_1_allvalid.nc
#> 104: ch4_mnc_surface-insitu_112_allvalid.nc
#> 105: ch4_mnm_surface-insitu_19_representative.nc
#> 106: ch4_ndao_surface-insitu_45_allvalid-21magl.nc
#> 107: ch4_nor_surface-insitu_442_allvalid-100magl.nc
#> 108: ch4_nor_surface-insitu_442_allvalid-32magl.nc
#> 109: ch4_nor_surface-insitu_442_allvalid-58magl.nc
#> 110: ch4_notr_surface-insitu_60_allvalid-91magl.nc
#> 111: ch4_oli_surface-insitu_64_allvalid-10magl.nc
#> 112: ch4_ope_surface-insitu_442_allvalid-10magl.nc
#> 113: ch4_ope_surface-insitu_442_allvalid-120magl.nc
#> 114: ch4_ope_surface-insitu_442_allvalid-50magl.nc
#> 115: ch4_oxk_surface-insitu_442_allvalid-163magl.nc
#> 116: ch4_oxk_surface-insitu_442_allvalid-23magl.nc
#> 117: ch4_oxk_surface-insitu_442_allvalid-90magl.nc
#> 118: ch4_pal_surface-insitu_30_continental.nc
#> 119: ch4_pal_surface-insitu_30_marine.nc
#> 120: ch4_pal_surface-insitu_30_nonlocal.nc
#> 121: ch4_pal_surface-insitu_442_allvalid-12magl.nc
#> 122: ch4_prs_surface-insitu_442_allvalid-10magl.nc
#> 123: ch4_pui_surface-insitu_442_allvalid-47magl.nc
#> 124: ch4_pui_surface-insitu_442_allvalid-84magl.nc
#> 125: ch4_puy_surface-insitu_442_allvalid-10magl.nc
#> 126: ch4_rgl_surface-insitu_442_allvalid-45magl.nc
#> 127: ch4_rgl_surface-insitu_442_allvalid-90magl.nc
#> 128: ch4_ric_surface-insitu_112_allvalid.nc
#> 129: ch4_run_surface-insitu_442_allvalid-6magl.nc
#> 130: ch4_ryo_surface-insitu_19_representative.nc
#> 131: ch4_sac_surface-insitu_442_allvalid-100magl.nc
#> 132: ch4_sac_surface-insitu_442_allvalid-15magl.nc
#> 133: ch4_sac_surface-insitu_442_allvalid-50magl.nc
#> 134: ch4_sgp_surface-insitu_64_allvalid-60magl.nc
#> 135: ch4_sig_surface-insitu_112_allvalid.nc
#> 136: ch4_smr_surface-insitu_442_allvalid-125magl.nc
#> 137: ch4_smr_surface-insitu_442_allvalid-17magl.nc
#> 138: ch4_smr_surface-insitu_442_allvalid-67magl.nc
#> 139: ch4_snj_surface-insitu_112_allvalid.nc
#> 140: ch4_sno_surface-insitu_442_allvalid-20magl.nc
#> 141: ch4_sno_surface-insitu_442_allvalid-50magl.nc
#> 142: ch4_sno_surface-insitu_442_allvalid-85magl.nc
#> 143: ch4_ssl_surface-insitu_442_allvalid-12magl.nc
#> 144: ch4_ssl_surface-insitu_442_allvalid-35magl.nc
#> 145: ch4_ste_surface-insitu_442_allvalid-127magl.nc
#> 146: ch4_ste_surface-insitu_442_allvalid-187magl.nc
#> 147: ch4_ste_surface-insitu_442_allvalid-252magl.nc
#> 148: ch4_ste_surface-insitu_442_allvalid-32magl.nc
#> 149: ch4_ste_surface-insitu_442_allvalid-82magl.nc
#> 150: ch4_svb_surface-insitu_442_allvalid-150magl.nc
#> 151: ch4_svb_surface-insitu_442_allvalid-35magl.nc
#> 152: ch4_svb_surface-insitu_442_allvalid-85magl.nc
#> 153: ch4_syo_surface-insitu_8_representative.nc
#> 154: ch4_tao_surface-insitu_6_allvalid.nc
#> 155: ch4_thd_surface-insitu_112_allvalid.nc
#> 156: ch4_tik_surface-insitu_30_allvalid.nc
#> 157: ch4_toh_surface-insitu_442_allvalid-10magl.nc
#> 158: ch4_toh_surface-insitu_442_allvalid-110magl.nc
#> 159: ch4_toh_surface-insitu_442_allvalid-147magl.nc
#> 160: ch4_toh_surface-insitu_442_allvalid-76magl.nc
#> 161: ch4_tpd_surface-insitu_6_allvalid.nc
#> 162: ch4_trn_surface-insitu_442_allvalid-100magl.nc
#> 163: ch4_trn_surface-insitu_442_allvalid-180magl.nc
#> 164: ch4_trn_surface-insitu_442_allvalid-50magl.nc
#> 165: ch4_trn_surface-insitu_442_allvalid-5magl.nc
#> 166: ch4_uld_surface-insitu_61_allvalid.nc
#> 167: ch4_uny_surface-insitu_112_allvalid.nc
#> 168: ch4_uto_surface-insitu_442_allvalid-57magl.nc
#> 169: ch4_wao_surface-insitu_442_allvalid-10magl.nc
#> 170: ch4_wes_surface-insitu_442_allvalid-14magl.nc
#> 171: ch4_wsa_surface-insitu_6_allvalid.nc
#> 172: ch4_yon_surface-insitu_19_representative.nc
#> 173: ch4_zep_surface-insitu_442_allvalid-15magl.nc
#> 174: ch4_zsf_surface-insitu_442_allvalid-3magl.nc
```

Now we check the data

```{r checkdf, eval = F}
df
```

```
#>           year month   day  hour minute second       time start_time
#>          <int> <int> <int> <int>  <int>  <int>      <int>      <int>
#>       1:  2014     3     6     5     30      0 1394083800 1394082000
#>       2:  2014     3     6     6     30      0 1394087400 1394085600
#>       3:  2014     3     6     7     30      0 1394091000 1394089200
#>       4:  2014     3     6     8     30      0 1394094600 1394092800
#>       5:  2014     3     6     9     30      0 1394098200 1394096400
#>      ---                                                            
#> 8281129:  2021    12    31    19     30      0 1640979000 1640977200
#> 8281130:  2021    12    31    20     30      0 1640982600 1640980800
#> 8281131:  2021    12    31    21     30      0 1640986200 1640984400
#> 8281132:  2021    12    31    22     30      0 1640989800 1640988000
#> 8281133:  2021    12    31    23     30      0 1640993400 1640991600
#>          midpoint_time             datetime time_decimal       value nvalue
#>                  <int>               <char>        <num>       <num>  <int>
#>       1:    1394083800 2014-03-06T05:30:00Z     2014.176 1.93390e-06     53
#>       2:    1394087400 2014-03-06T06:30:00Z     2014.176 1.93930e-06     57
#>       3:    1394091000 2014-03-06T07:30:00Z     2014.176 1.95110e-06     57
#>       4:    1394094600 2014-03-06T08:30:00Z     2014.176 1.96890e-06     57
#>       5:    1394098200 2014-03-06T09:30:00Z     2014.176 1.99260e-06     57
#>      ---                                                                   
#> 8281129:    1640979000 2021-12-31T19:30:00Z     2021.999 1.93855e-06     60
#> 8281130:    1640982600 2021-12-31T20:30:00Z     2022.000 1.93909e-06     60
#> 8281131:    1640986200 2021-12-31T21:30:00Z     2022.000 1.93938e-06     60
#> 8281132:    1640989800 2021-12-31T22:30:00Z     2022.000 1.94089e-06     60
#> 8281133:    1640993400 2021-12-31T23:30:00Z     2022.000 1.94087e-06     26
#>          value_std_dev latitude longitude altitude elevation intake_height
#>                  <num>    <num>     <num>    <num>     <num>         <num>
#>       1:      4.20e-09  49.0114 -122.3353       93        60            33
#>       2:      8.40e-09  49.0114 -122.3353       93        60            33
#>       3:      5.70e-09  49.0114 -122.3353       93        60            33
#>       4:      2.18e-08  49.0114 -122.3353       93        60            33
#>       5:      9.40e-09  49.0114 -122.3353       93        60            33
#>      ---                                                                  
#> 8281129:      4.54e-10  47.4165   10.9796     2669      2666             3
#> 8281130:      3.38e-10  47.4165   10.9796     2669      2666             3
#> 8281131:      7.64e-10  47.4165   10.9796     2669      2666             3
#> 8281132:      4.25e-10  47.4165   10.9796     2669      2666             3
#> 8281133:      3.55e-10  47.4165   10.9796     2669      2666             3
#>          obs_flag obspack_num
#>             <int>       <int>
#>       1:        0     2671976
#>       2:        0     2671977
#>       3:        0     2671978
#>       4:        0     2671979
#>       5:        0     2671980
#>      ---                     
#> 8281129:        0    13341897
#> 8281130:        0    13341898
#> 8281131:        0    13341899
#> 8281132:        0    13341900
#> 8281133:        0    13341901
#>                                                                                               obspack_id
#>                                                                                                   <char>
#>       1:          obspack_ch4_1_GLOBALVIEWplus_v5.1_2023-03-08~ch4_abt_surface-insitu_6_allvalid~2671976
#>       2:          obspack_ch4_1_GLOBALVIEWplus_v5.1_2023-03-08~ch4_abt_surface-insitu_6_allvalid~2671977
#>       3:          obspack_ch4_1_GLOBALVIEWplus_v5.1_2023-03-08~ch4_abt_surface-insitu_6_allvalid~2671978
#>       4:          obspack_ch4_1_GLOBALVIEWplus_v5.1_2023-03-08~ch4_abt_surface-insitu_6_allvalid~2671979
#>       5:          obspack_ch4_1_GLOBALVIEWplus_v5.1_2023-03-08~ch4_abt_surface-insitu_6_allvalid~2671980
#>      ---                                                                                                
#> 8281129: obspack_ch4_1_GLOBALVIEWplus_v5.1_2023-03-08~ch4_zsf_surface-insitu_442_allvalid-3magl~13341897
#> 8281130: obspack_ch4_1_GLOBALVIEWplus_v5.1_2023-03-08~ch4_zsf_surface-insitu_442_allvalid-3magl~13341898
#> 8281131: obspack_ch4_1_GLOBALVIEWplus_v5.1_2023-03-08~ch4_zsf_surface-insitu_442_allvalid-3magl~13341899
#> 8281132: obspack_ch4_1_GLOBALVIEWplus_v5.1_2023-03-08~ch4_zsf_surface-insitu_442_allvalid-3magl~13341900
#> 8281133: obspack_ch4_1_GLOBALVIEWplus_v5.1_2023-03-08~ch4_zsf_surface-insitu_442_allvalid-3magl~13341901
#>          unique_sample_location_num year_st month_st day_st hour_st minute_st
#>                               <int>   <int>    <int>  <int>   <int>     <int>
#>       1:                   32763088    2014        3      5      21         8
#>       2:                   32763089    2014        3      5      22         8
#>       3:                   32763090    2014        3      5      23         8
#>       4:                   32763091    2014        3      6       0         8
#>       5:                   32763092    2014        3      6       1         8
#>      ---                                                                     
#> 8281129:                   49987485    2021       12     31      20        11
#> 8281130:                   49987486    2021       12     31      21        11
#> 8281131:                   49987487    2021       12     31      22        11
#> 8281132:                   49987488    2021       12     31      23        11
#> 8281133:                   49987489    2022        1      1       0        11
#>          second_st          scale site_elevation_unit dataset_project
#>              <int>         <char>              <char>          <char>
#>       1:        58 WMO CH4 X2004A                masl  surface-insitu
#>       2:        58 WMO CH4 X2004A                masl  surface-insitu
#>       3:        58 WMO CH4 X2004A                masl  surface-insitu
#>       4:        58 WMO CH4 X2004A                masl  surface-insitu
#>       5:        58 WMO CH4 X2004A                masl  surface-insitu
#>      ---                                                             
#> 8281129:         1  WMO CH4 X2019                masl  surface-insitu
#> 8281130:         1  WMO CH4 X2019                masl  surface-insitu
#> 8281131:         1  WMO CH4 X2019                masl  surface-insitu
#> 8281132:         1  WMO CH4 X2019                masl  surface-insitu
#> 8281133:         1  WMO CH4 X2019                masl  surface-insitu
#>          dataset_selection_tag                    site_name site_elevation
#>                         <char>                       <char>          <num>
#>       1:              allvalid Abbotsford, British Columbia             60
#>       2:              allvalid Abbotsford, British Columbia             60
#>       3:              allvalid Abbotsford, British Columbia             60
#>       4:              allvalid Abbotsford, British Columbia             60
#>       5:              allvalid Abbotsford, British Columbia             60
#>      ---                                                                  
#> 8281129:        allvalid-3magl Zugspitze / Schneefernerhaus           2666
#> 8281130:        allvalid-3magl Zugspitze / Schneefernerhaus           2666
#> 8281131:        allvalid-3magl Zugspitze / Schneefernerhaus           2666
#> 8281132:        allvalid-3magl Zugspitze / Schneefernerhaus           2666
#> 8281133:        allvalid-3magl Zugspitze / Schneefernerhaus           2666
#>          site_latitude site_longitude site_country site_code site_utc2lst
#>                  <num>          <num>       <char>    <char>        <num>
#>       1:       49.0114      -122.3353       Canada       ABT           -8
#>       2:       49.0114      -122.3353       Canada       ABT           -8
#>       3:       49.0114      -122.3353       Canada       ABT           -8
#>       4:       49.0114      -122.3353       Canada       ABT           -8
#>       5:       49.0114      -122.3353       Canada       ABT           -8
#>      ---                                                                 
#> 8281129:       47.4165        10.9796      Germany       ZSF            0
#> 8281130:       47.4165        10.9796      Germany       ZSF            0
#> 8281131:       47.4165        10.9796      Germany       ZSF            0
#> 8281132:       47.4165        10.9796      Germany       ZSF            0
#> 8281133:       47.4165        10.9796      Germany       ZSF            0
#>          lab_1_abbr dataset_calibration_scale altitude_final type_altitude
#>              <char>                    <char>          <num>         <num>
#>       1:       ECCC            WMO CH4 X2004A             33            NA
#>       2:       ECCC            WMO CH4 X2004A             33            NA
#>       3:       ECCC            WMO CH4 X2004A             33            NA
#>       4:       ECCC            WMO CH4 X2004A             33            NA
#>       5:       ECCC            WMO CH4 X2004A             33            NA
#>      ---                                                                  
#> 8281129:   ICOS-ATC             WMO CH4 X2019              3             0
#> 8281130:   ICOS-ATC             WMO CH4 X2019              3             0
#> 8281131:   ICOS-ATC             WMO CH4 X2019              3             0
#> 8281132:   ICOS-ATC             WMO CH4 X2019              3             0
#> 8281133:   ICOS-ATC             WMO CH4 X2019              3             0
#>          dataset_intake_ht_unit instrument method pressure qcflag temperature
#>                          <char>     <char> <char>    <num> <char>       <num>
#>       1:                   <NA>       <NA>   <NA>       NA   <NA>          NA
#>       2:                   <NA>       <NA>   <NA>       NA   <NA>          NA
#>       3:                   <NA>       <NA>   <NA>       NA   <NA>          NA
#>       4:                   <NA>       <NA>   <NA>       NA   <NA>          NA
#>       5:                   <NA>       <NA>   <NA>       NA   <NA>          NA
#>      ---                                                                     
#> 8281129:                   magl        880   <NA>       NA      O          NA
#> 8281130:                   magl        880   <NA>       NA      O          NA
#> 8281131:                   magl        880   <NA>       NA      O          NA
#> 8281132:                   magl        880   <NA>       NA      O          NA
#> 8281133:                   magl        880   <NA>       NA      O          NA
#>          time_interval     u     v value_unc
#>                  <int> <num> <num>     <num>
#>       1:            NA    NA    NA        NA
#>       2:            NA    NA    NA        NA
#>       3:            NA    NA    NA        NA
#>       4:            NA    NA    NA        NA
#>       5:            NA    NA    NA        NA
#>      ---                                    
#> 8281129:            NA    NA    NA        NA
#> 8281130:            NA    NA    NA        NA
#> 8281131:            NA    NA    NA        NA
#> 8281132:            NA    NA    NA        NA
#> 8281133:            NA    NA    NA        NA
```

Now we can process the data. We first filter for observations
within our spatial domain:

## Checks and definitions

```{r spatial, eval = F}
north <- 80
south <- 10
west <- -170
east <- -50
max_altitude <- 8000
yy <- 2020
evening <- 14

```

We check altitude, intake_height, altitude_final and elevation.
altitude_final is a column from intake_height, added to match
column from obs_read text files.

```{r checkcols, eval = F}
df[, c("altitude", "altitude_final", "intake_height", "elevation",
       "dataset_selection_tag",
              "site_name")]
```

```
#>          altitude altitude_final intake_height elevation dataset_selection_tag
#>             <num>          <num>         <num>     <num>                <char>
#>       1:       93             33            33        60              allvalid
#>       2:       93             33            33        60              allvalid
#>       3:       93             33            33        60              allvalid
#>       4:       93             33            33        60              allvalid
#>       5:       93             33            33        60              allvalid
#>      ---                                                                      
#> 8281129:     2669              3             3      2666        allvalid-3magl
#> 8281130:     2669              3             3      2666        allvalid-3magl
#> 8281131:     2669              3             3      2666        allvalid-3magl
#> 8281132:     2669              3             3      2666        allvalid-3magl
#> 8281133:     2669              3             3      2666        allvalid-3magl
#>                             site_name
#>                                <char>
#>       1: Abbotsford, British Columbia
#>       2: Abbotsford, British Columbia
#>       3: Abbotsford, British Columbia
#>       4: Abbotsford, British Columbia
#>       5: Abbotsford, British Columbia
#>      ---                             
#> 8281129: Zugspitze / Schneefernerhaus
#> 8281130: Zugspitze / Schneefernerhaus
#> 8281131: Zugspitze / Schneefernerhaus
#> 8281132: Zugspitze / Schneefernerhaus
#> 8281133: Zugspitze / Schneefernerhaus
```
The temporal range of data is
```{r range_year, eval = F}
range(df$year)
```

```
#> [1] 1983 2021
```

We also check for dimensions of data

```{r dim_df, eval = F}
dim(df)
```

```
#> [1] 8281133      54
```

## Filters

```{r spatial_temporal_filter, eval = F}
df <- df[year == yy]

df <- df[altitude_final < max_altitude &
           latitude < north &
           latitude > south &
           longitude < east &
           longitude > west]
dim(df)
```

```
#> [1] 274587     54
```

Towers can have observations at different
heights. Here we need to select one site with the observations
registered at the highest height. The column
with the height is named `altitude_final` and the
max altitude was named `max_altitude`.


```{r, eval = F}
dfa <- df[,
          max(altitude_final),
          by = site_code] |> unique()

names(dfa)[2] <- "max_altitude"
dfa
```

```
#>     site_code max_altitude
#>        <char>        <num>
#>  1:       ABT        33.00
#>  2:       BCK        60.00
#>  3:       BRA        35.00
#>  4:       BRW        16.46
#>  5:      CARL         9.00
#>  6:       CBY        12.00
#>  7:       CHL        60.00
#>  8:       CPS        40.00
#>  9:       EGB        25.00
#> 10:       ESP        40.00
#> 11:       EST        50.00
#> 12:       ETL       105.00
#> 13:       FNE        15.00
#> 14:      FORT       128.00
#> 15:       FSD        40.00
#> 16:     GHG06       100.00
#> 17:     GHG09       100.00
#> 18:       HNP        10.00
#> 19:      HOBB        91.00
#> 20:       INU        10.00
#> 21:     INX01       121.00
#> 22:     INX02       136.00
#> 23:     INX07        58.00
#> 24:     INX08        41.00
#> 25:     INX09       130.00
#> 26:     INX10        40.00
#> 27:     INX13        87.00
#> 28:     INX14        76.00
#> 29:       LLB        50.00
#> 30:      MALJ       134.00
#> 31:       MLO        40.00
#> 32:       OLI         0.00
#> 33:       SGP         4.00
#> 34:       THD        90.00
#> 35:       TPD        35.00
#> 36:       WSA         3.00
#>     site_code max_altitude
```


### Key Time

Here we need to start time columns.
The function `obs_addtime` adds time columns
`timeUTC`, `timeUTC_start` which shows the start time
of each observation and `timeUTC_end`
which shows the end time for each observation.


```{r, eval = F}
df2 <- obs_addtime(df)
```

```
#> Adding timeUTC
#> Adding timeUTC_start
#> Adding timeUTC_end
#> Found time_interval
```

Then we need a *key_time* to aggregate data.
This can be done using UTC, solar, or local time.
The normal approach is using afternoon solar or local time.


#### Hierarchy of solar or local time

1) Solar time
2) Local time with columns `site_utc2lst`
3) Local time longitude


> solar time (default)

Here we select the hours of interest and then
aggregate data by year, month and day of solar time.
In this way, we will have one information per day. however
this approach is not appropriate for aircraft
which are aggregated every 10 or 20 seconds. Hence
we need to aggregate data by one time column.
Also, this helps to generate the receptor info 
files including hour, minute and second.
Hence, *we need to add solar or local time column*.

```{r, eval = F}
df2$solar_time <- obs_addstime(df2)

```


> local time with column `site_utc2lst`

Then we need to identify the local time with the function
`add_ltime`. This is important because to identifying
observations in the evening in local time.
`add_ltime` uses two methods, first identify
the time difference with utc by identifying the
metadata column "site_utc2lst". 
If solar time is not available
#now we need to cut solar time for the frequency needed. 
As we will work with 



> local time longitude

If this information
is not available, with the aircrafts for instance,
the local time is calculated with an approximation
based on longitude:


$$
lt = UTC + longitude/15 * 60 * 60
$$
Where $lt$ is the local time, $UTC$ the time,
$longitude$ the coordinate.
Then, the time is cut every two hours.
Now, we identify the local time to select evening hours.



#### Cut time

Now we have they key column time, we can cut it accordingly.

```{r, eval = F}
df2$solar_time_cut <- cut(x = df2$solar_time,
                          breaks = "1 hour") |>
  as.character()

```


How we can check the solar time and the cut solar time.
Please note that solar_time_cut,
the column that it will be used to aggregate data

How we filter for the required solar time, in this case 14.

```{r, eval = F}
df3 <- df2[hour_st %in% evening]
df3[, c("solar_time", "solar_time_cut")]


```

```
#>                 solar_time      solar_time_cut
#>                     <POSc>              <char>
#>     1: 2020-01-01 14:17:19 2020-01-01 14:00:00
#>     2: 2020-01-02 14:16:52 2020-01-02 14:00:00
#>     3: 2020-01-03 14:16:26 2020-01-03 14:00:00
#>     4: 2020-01-04 14:16:00 2020-01-04 14:00:00
#>     5: 2020-01-05 14:15:34 2020-01-05 14:00:00
#>    ---                                        
#> 11473: 2020-03-15 14:20:51 2020-03-15 14:00:00
#> 11474: 2020-03-16 14:21:08 2020-03-16 14:00:00
#> 11475: 2020-03-17 14:21:26 2020-03-17 14:00:00
#> 11476: 2020-03-18 14:21:44 2020-03-18 14:00:00
#> 11477: 2020-03-19 14:22:02 2020-03-19 14:00:00
```


At this point we can calculate the averages of several columns by
the cut time. The function `obs_agg` does this aggregation as shown
in the following lines of code. The argument `gby`
establish the function used to aggregate `cols`. I need to aggregate the data
by date (year, month, date), because it is already filtered by
the hours of interest. Then, I would have 1 observation per day.

As standard, let us define `key_time` as `solar_time`. 
The `obs_agg` function will aggregate the desired data by that column.

```{r, eval = F}
df3$key_time <- df3$solar_time_cut
```


```{r, eval = F}
df4 <- obs_agg(dt = df3,
               cols = c("value",
                        "latitude",
                        "longitude",
                        "site_utc2lst"),
               verbose = T,
               byalt = TRUE)
```

```
#> Selecting by alt
#> Adding time
```

Here we add the column `max_altitude` to identify the max altitude
by site. 

```{r, eval = F}
df4[,
    max_altitude := max(altitude_final),
    by = site_code]
df4[,
    c("site_code",
      "altitude_final",
      "max_altitude")] |> unique()

```

```
#>     site_code altitude_final max_altitude
#>        <char>          <num>        <num>
#>  1:       ABT          33.00        33.00
#>  2:       BCK          60.00        60.00
#>  3:       BRA          35.00        35.00
#>  4:       BRW          16.46        16.46
#>  5:      CARL           9.00         9.00
#>  6:       CBY          12.00        12.00
#>  7:       CHL          60.00        60.00
#>  8:       CPS          40.00        40.00
#>  9:       EGB          25.00        25.00
#> 10:       ESP          40.00        40.00
#> 11:       EST          50.00        50.00
#> 12:       ETL         105.00       105.00
#> 13:       FNE          15.00        15.00
#> 14:      FORT         128.00       128.00
#> 15:       FSD          40.00        40.00
#> 16:     GHG06         100.00       100.00
#> 17:     GHG06          50.00       100.00
#> 18:     GHG09         100.00       100.00
#> 19:     GHG09          50.00       100.00
#> 20:       HNP          10.00        10.00
#> 21:      HOBB          91.00        91.00
#> 22:       INU          10.00        10.00
#> 23:     INX01          10.00       121.00
#> 24:     INX01         121.00       121.00
#> 25:     INX01          40.00       121.00
#> 26:     INX02         136.00       136.00
#> 27:     INX07          21.00        58.00
#> 28:     INX07          58.00        58.00
#> 29:     INX08          41.00        41.00
#> 30:     INX09         130.00       130.00
#> 31:     INX10          40.00        40.00
#> 32:     INX13          87.00        87.00
#> 33:     INX14          76.00        76.00
#> 34:       LLB          50.00        50.00
#> 35:      MALJ         134.00       134.00
#> 36:       MLO          40.00        40.00
#> 37:       OLI           0.00         0.00
#> 38:       SGP           4.00         4.00
#> 39:       THD          90.00        90.00
#> 40:       TPD          35.00        35.00
#> 41:       WSA           3.00         3.00
#>     site_code altitude_final max_altitude
```


## Master

Before generating the receptors list, we have the database with all the required information

```{r rename, eval = F}
master <- df4

```
We may replace missing values with a nine nines. Here is commented

#master[is.na(master)] <- 999999999

We transform the time variables to character and round coordinates with 4 digits

```{r round, eval = F}
master$timeUTC <- as.character(master$timeUTC)
master$local_time <- as.character(master$local_time)
master$latitude <- round(master$latitude, 4)
master$longitude <- round(master$longitude, 4)

```

## Save master
Finally we save the master file

```{r outfile, eval = F}
out <- tempfile()

```

### txt

```{r outtxt, eval = F}
message(paste0(out,"_", datasetid, ".txt\n"))
fwrite(master,
       paste0(out,"_", datasetid, ".txt"),
       sep = " ")
```

```
#> C:\Users\sibarrae\AppData\Local\Temp\Rtmp4k67qL\file531463a43a74_surface-insitu.txt
```

### csv

```{r outcsv, eval = F}
message(paste0(out,"_", datasetid, ".csv\n"))
fwrite(master,
       paste0(out,"_", datasetid, ".csv"),
       sep = ",")
```

```
#> C:\Users\sibarrae\AppData\Local\Temp\Rtmp4k67qL\file531463a43a74_surface-insitu.csv
```

### csvy

CSVY are csv files with a YAML header to include metadata in tabulated
text files

```{r csvy, eval = F}
cat("\nAdding notes in csvy:\n")
notes <- c(paste0("sector: ", datasetid),
           paste0("timespan: ", yy),
           paste0("spatial_limits: north = ", north, ", south = ", south, ", east = ", east, ", west = ", west),
           paste0("altitude: < ", max_altitude),
           paste0("hours: ", evening),
           "local_time: used solar_time")

cat(notes, sep = "\n")

message(paste0(out,"_", datasetid, ".csvy\n"))
obs_write_csvy(dt = master,
               notes = notes,
               out = paste0(out,"_", datasetid, ".csvy"))
```

```
#> Adding notes in csvy:
#> sector: surface-insitu
#> timespan: 2020
#> spatial_limits: north = 80, south = 10, east = -50, west = -170
#> data: Data averaged every 20 seconds
#> altitude: < 8000
#> hours: 14
#> local_time: used solar_time
#> C:\Users\sibarrae\AppData\Local\Temp\Rtmp4k67qL\file531463a43a74_surface-insitu.csvy
```


```{r readcsvy, eval = F}
obs_read_csvy(paste0(out,"_", datasetid, ".csvy"))
```

```
#>  [1] "---"                                                                 
#>  [2] "name: Metadata "                                                     
#>  [3] "sector: surface-insitu"                                              
#>  [4] "timespan: 2020"                                                      
#>  [5] "spatial_limits: north = 80, south = 10, east = -50, west = -170"     
#>  [6] "data: Data averaged every 20 seconds"                                
#>  [7] "altitude: < 8000"                                                    
#>  [8] "hours: 14"                                                           
#>  [9] "local_time: used solar_time"                                         
#> [10] "structure: "                                                         
#> [11] "Classes 'data.table' and 'data.frame':\t11378 obs. of  20 variables:"
#> [12] " $ timeUTC                  : chr  \"2020-01-01 14:00:00\" \".."     
#> [13] " $ site_code                : chr  \"ABT\" \"ABT\" ..."              
#> [14] " $ altitude_final           : num  33 33 33 33 33 ..."               
#> [15] " $ type_altitude            : num  NA NA NA NA NA ..."               
#> [16] " $ lab_1_abbr               : chr  \"ECCC\" \"ECCC\" ..."            
#> [17] " $ dataset_calibration_scale: chr  \"WMO CH4 X2004A\" \"WMO \".."    
#> [18] " $ value                    : num  1.96e-06 1.99e-06 ..."            
#> [19] " $ latitude                 : num  49 49 ..."                        
#> [20] " $ longitude                : num  -122 -122 ..."                    
#> [21] " $ site_utc2lst             : num  -8 -8 -8 -8 -8 ..."               
#> [22] " $ year                     : int  2020 2020 2020 2020 202.."        
#> [23] " $ month                    : int  1 1 1 1 1 ..."                    
#> [24] " $ day                      : chr  \"01\" \"02\" ..."                
#> [25] " $ hour                     : int  14 14 14 14 14 ..."               
#> [26] " $ minute                   : int  0 0 0 0 0 ..."                    
#> [27] " $ second                   : int  0 0 0 0 0 ..."                    
#> [28] " $ time                     : num  1.58e+09 1.58e+09 ..."            
#> [29] " $ time_decimal             : num  2020 2020 ..."                    
#> [30] " $ max_altitude             : num  33 33 33 33 33 ..."               
#> [31] " $ local_time               : chr  NA NA ..."                        
#> [32] " - attr(*, \".internal.selfref\")=<externalptr> "                    
#> [33] "NULL"                                                                
#> [34] "---"
#>                    timeUTC site_code altitude_final type_altitude lab_1_abbr
#>                     <POSc>    <char>          <num>         <int>     <char>
#>     1: 2020-01-01 14:00:00       ABT             33            NA       ECCC
#>     2: 2020-01-02 14:00:00       ABT             33            NA       ECCC
#>     3: 2020-01-03 14:00:00       ABT             33            NA       ECCC
#>     4: 2020-01-04 14:00:00       ABT             33            NA       ECCC
#>     5: 2020-01-05 14:00:00       ABT             33            NA       ECCC
#>    ---                                                                      
#> 11374: 2020-03-15 14:00:00       WSA              3            NA       ECCC
#> 11375: 2020-03-16 14:00:00       WSA              3            NA       ECCC
#> 11376: 2020-03-17 14:00:00       WSA              3            NA       ECCC
#> 11377: 2020-03-18 14:00:00       WSA              3            NA       ECCC
#> 11378: 2020-03-19 14:00:00       WSA              3            NA       ECCC
#>        dataset_calibration_scale      value latitude longitude site_utc2lst
#>                           <char>      <num>    <num>     <num>        <int>
#>     1:            WMO CH4 X2004A 1.9555e-06  49.0114 -122.3353           -8
#>     2:            WMO CH4 X2004A 1.9929e-06  49.0114 -122.3353           -8
#>     3:            WMO CH4 X2004A 1.9471e-06  49.0114 -122.3353           -8
#>     4:            WMO CH4 X2004A 1.9763e-06  49.0114 -122.3353           -8
#>     5:            WMO CH4 X2004A 1.9656e-06  49.0114 -122.3353           -8
#>    ---                                                                     
#> 11374:            WMO CH4 X2004A 1.9610e-06  43.9322  -60.0093           -4
#> 11375:            WMO CH4 X2004A 1.9614e-06  43.9322  -60.0093           -4
#> 11376:            WMO CH4 X2004A 1.9560e-06  43.9322  -60.0093           -4
#> 11377:            WMO CH4 X2004A 1.9663e-06  43.9322  -60.0093           -4
#> 11378:            WMO CH4 X2004A 1.9711e-06  43.9322  -60.0093           -4
#>         year month   day  hour minute second       time time_decimal
#>        <int> <int> <int> <int>  <int>  <int>      <int>        <num>
#>     1:  2020     1     1    14      0      0 1577887200     2020.002
#>     2:  2020     1     2    14      0      0 1577973600     2020.004
#>     3:  2020     1     3    14      0      0 1578060000     2020.007
#>     4:  2020     1     4    14      0      0 1578146400     2020.010
#>     5:  2020     1     5    14      0      0 1578232800     2020.013
#>    ---                                                              
#> 11374:  2020     3    15    14      0      0 1584280800     2020.204
#> 11375:  2020     3    16    14      0      0 1584367200     2020.207
#> 11376:  2020     3    17    14      0      0 1584453600     2020.209
#> 11377:  2020     3    18    14      0      0 1584540000     2020.212
#> 11378:  2020     3    19    14      0      0 1584626400     2020.215
#>        max_altitude local_time
#>               <num>     <lgcl>
#>     1:           33         NA
#>     2:           33         NA
#>     3:           33         NA
#>     4:           33         NA
#>     5:           33         NA
#>    ---                        
#> 11374:            3         NA
#> 11375:            3         NA
#> 11376:            3         NA
#> 11377:            3         NA
#> 11378:            3         NA
```



## Receptors

Now we can do the last step which is generating the receptor list files.
Now we filter selected columns

```{r receptors, eval = F}
receptor <- master[, c("site_code",
                     "year",
                     "month",
                     "day",
                     "hour",
                     "minute",
                     "second",
                     "latitude",
                     "longitude",
                     "altitude_final",
                     "type_altitude",
                     "time_decimal")]

```

We can round altitude also

```{r round_alt, eval = F}
receptor$altitude_final <- round(receptor$altitude_final)
```

Now we can format time variables with two digits

```{r formatrec, eval = F}
receptor <- obs_format(receptor,
                        spf =  c("month", "day",
                                 "hour", "minute", "second"))
```

We have a column that indicate AGL or ASL

```{r aslagl, eval = F}
receptor_agl <- receptor[type_altitude == 0]
receptor_asl <- receptor[type_altitude == 1]
```

Finally, we save the receptors

```{r save_receptors, eval = F}
if(nrow(receptor_agl) > 0) {
  message(paste0(out, "_", datasetid, "_receptor_AGL.txt"), "\n")

  fwrite(x = receptor_agl,
         file = paste0(out, "_", datasetid, "_receptor_AGL.txt"),
         sep = " ")
}

if(nrow(receptor_asl) > 0) {
  message(paste0(out, "_", datasetid, "_receptor_ASL.txt"), "\n")

  fwrite(x = receptor_asl,
         file = paste0(out, "_", datasetid, "receptor_ASL.txt"),
         sep = " ")

}
```

```
#> C:\Users\sibarrae\AppData\Local\Temp\Rtmp4k67qL\file531463a43a74_surface-insitu_receptor_AGL.txt
```


## Plot

Finally, we just plot some data, run it locally


```{r obs_plotsave, fig.width=5, fig.height=3, eval = F, echo = F, message=F, warning=F}
png("../man/figures/obsplot_surfaceinsitu.png", width = 1500, height = 1000, res = 200)
obs_plot(df3, time = "timeUTC", yfactor = 1e9)
dev.off()
```

```{r obs_plot, fig.width=5, fig.height=3, eval = F}
obs_plot(df4, time = "timeUTC", yfactor = 1e9)
```
```
#> Found the following sites: 
#>  [1] ABT   BCK   BRA   BRW   CARL  CBY   CHL   CPS   EGB   ESP   EST   ETL  
#> [13] FNE   FORT  FSD   GHG06 GHG09 HNP   HOBB  INU   INX01 INX02 INX07 INX08
#> [25] INX09 INX10 INX13 INX14 LLB   MALJ  MLO   OLI   SGP   THD   TPD   WSA  
#> Plotting the following sites: 
#> [1] ABT BCK
```
![Time series](https://github.com/ibarraespinosa/rtorf/blob/main/man/figures/obsplot_surfaceinsitu.png?raw=true)


```{r savesf, fig.width=5, fig.height=3, eval = F, echo = F}
library(sf)
dx <- df4[, 
    lapply(.SD, mean),
    .SDcols = "value",
    by = .(latitude, longitude)]
x <- st_as_sf(dx, coords = c("longitude", "latitude"), crs = 4326)
png("../man/figures/obsplot_surfaceinsitu_map.png", width = 1500, height = 1000, res = 200)
plot(x["value"], axes = T, reset = F)
maps::map(add = T)
dev.off()
```


```{r sf, fig.width=5, fig.height=3, eval = F}
library(sf)
dx <- df4[, 
    lapply(.SD, mean),
    .SDcols = "value",
    by = .(latitude, longitude)]
x <- st_as_sf(dx, coords = c("longitude", "latitude"), crs = 4326)
plot(x["value"], axes = T, reset = F)
maps::map(add = T)
```

![Map](https://github.com/ibarraespinosa/rtorf/blob/main/man/figures/obsplot_surfaceinsitu_map.png?raw=true)
