---
title: "tower-insitu"
output: rmarkdown::html_vignette
vignette: >
  %\VignetteIndexEntry{tower-insitu}
  %\VignetteEngine{knitr::rmarkdown}
  %\VignetteEncoding{UTF-8}
---



```{r, include = FALSE}
knitr::opts_chunk$set(
  collapse = TRUE,
  comment = "#>"
)
```

```{r setup}
library(rtorf)
library(data.table)
```


```{r read1, eval = F}

cate = c("aircraft-pfp",
         "aircraft-insitu",
         "aircraft-flask",
         "surface-insitu",
         "surface-flask", 
         "surface-pfp",   
         "tower-insitu",  
         "aircore",       
         "shipboard-insitu",
         "shipboard-flask") 

obs <- "Z:/obspack/obspack_ch4_1_GLOBALVIEWplus_v5.1_2023-03-08/data/nc/"
index <- obs_summary(obs = obs, 
                     categories = cate)
```

```
#> Number of files of index: 429
#>               sector     N
#>               <char> <int>
#>  1:     aircraft-pfp    40
#>  2:  aircraft-insitu    15
#>  3:    surface-flask   106
#>  4:   surface-insitu   174
#>  5:   aircraft-flask     4
#>  6:          aircore     1
#>  7:      surface-pfp    33
#>  8:     tower-insitu    51
#>  9:  shipboard-flask     4
#> 10: shipboard-insitu     1
#> 11:    Total sectors   429
#> Detected 190 files with agl
#> Detected 239 files without agl
```

Now we read the `tower-insitu` using the function `obs_read_nc`. 
`solar_time` is  included for surface, so we `TRUE` that argument.

```{r readnc, eval = F}
datasetid <- "tower-insitu"
df <- obs_read_nc(index = index,
                  categories = datasetid,
                  solar_time = TRUE,
                  verbose = TRUE)

```

```
#> Searching tower-insitu...
#> 1: ch4_azv_tower-insitu_20_allvalid-29magl.nc
#> 2: ch4_azv_tower-insitu_20_allvalid-50magl.nc
#> 3: ch4_brz_tower-insitu_20_allvalid-20magl.nc
#> 4: ch4_brz_tower-insitu_20_allvalid-40magl.nc
#> 5: ch4_brz_tower-insitu_20_allvalid-5magl.nc
#> 6: ch4_brz_tower-insitu_20_allvalid-80magl.nc
#> 7: ch4_bsd_tower-insitu_160_allvalid-108magl.nc
#> 8: ch4_bsd_tower-insitu_160_allvalid-248magl.nc
#> 9: ch4_bsd_tower-insitu_160_allvalid-42magl.nc
#> 10: ch4_crv_tower-insitu_1_allvalid-17magl.nc
#> 11: ch4_crv_tower-insitu_1_allvalid-32magl.nc
#> 12: ch4_crv_tower-insitu_1_allvalid-5magl.nc
#> 13: ch4_dem_tower-insitu_20_allvalid-45magl.nc
#> 14: ch4_dem_tower-insitu_20_allvalid-63magl.nc
#> 15: ch4_dvv_tower-insitu_60_allvalid.nc
#> 16: ch4_gci01_tower-insitu_60_allvalid.nc
#> 17: ch4_gci02_tower-insitu_60_allvalid.nc
#> 18: ch4_gci03_tower-insitu_60_allvalid.nc
#> 19: ch4_gci04_tower-insitu_60_allvalid.nc
#> 20: ch4_hun_tower-insitu_35_allvalid-10magl.nc
#> 21: ch4_hun_tower-insitu_35_allvalid-115magl.nc
#> 22: ch4_hun_tower-insitu_35_allvalid-48magl.nc
#> 23: ch4_hun_tower-insitu_35_allvalid-82magl.nc
#> 24: ch4_igr_tower-insitu_20_allvalid-24magl.nc
#> 25: ch4_igr_tower-insitu_20_allvalid-47magl.nc
#> 26: ch4_krs_tower-insitu_20_allvalid-35magl.nc
#> 27: ch4_krs_tower-insitu_20_allvalid-67magl.nc
#> 28: ch4_lef_tower-insitu_1_allvalid-122magl.nc
#> 29: ch4_lef_tower-insitu_1_allvalid-30magl.nc
#> 30: ch4_lef_tower-insitu_1_allvalid-396magl.nc
#> 31: ch4_mrc_tower-insitu_60_allvalid-south.nc
#> 32: ch4_noy_tower-insitu_20_allvalid-21magl.nc
#> 33: ch4_noy_tower-insitu_20_allvalid-43magl.nc
#> 34: ch4_rgl_tower-insitu_160_allvalid-45magl.nc
#> 35: ch4_rgl_tower-insitu_160_allvalid-90magl.nc
#> 36: ch4_sct_tower-insitu_1_allvalid-305magl.nc
#> 37: ch4_sct_tower-insitu_1_allvalid-31magl.nc
#> 38: ch4_sct_tower-insitu_1_allvalid-61magl.nc
#> 39: ch4_svv_tower-insitu_20_allvalid-27magl.nc
#> 40: ch4_svv_tower-insitu_20_allvalid-52magl.nc
#> 41: ch4_tac_tower-insitu_160_allvalid-100magl.nc
#> 42: ch4_tac_tower-insitu_160_allvalid-185magl.nc
#> 43: ch4_tac_tower-insitu_160_allvalid-54magl.nc
#> 44: ch4_vgn_tower-insitu_20_allvalid-42magl.nc
#> 45: ch4_vgn_tower-insitu_20_allvalid-85magl.nc
#> 46: ch4_wgc_tower-insitu_1_allvalid-30magl.nc
#> 47: ch4_wgc_tower-insitu_1_allvalid-483magl.nc
#> 48: ch4_wgc_tower-insitu_1_allvalid-91magl.nc
#> 49: ch4_wsd_tower-insitu_60_allvalid.nc
#> 50: ch4_yak_tower-insitu_20_allvalid-11magl.nc
#> 51: ch4_yak_tower-insitu_20_allvalid-77magl.nc
```

Now we check the data

```{r checkdf, eval = F}
df
```

```
#>           year month   day  hour minute second       time start_time
#>          <int> <int> <int> <int>  <int>  <int>      <int>      <int>
#>       1:  2009    10     7     5     30      0 1254893400 1254891600
#>       2:  2009    10     7     6     30      0 1254897000 1254895200
#>       3:  2009    10     7     7     30      0 1254900600 1254898800
#>       4:  2009    10     7     8     30      0 1254904200 1254902400
#>       5:  2009    10     7     9     30      0 1254907800 1254906000
#>      ---                                                            
#> 2611288:  2013    12    13     1     30      0 1386898200 1386896400
#> 2611289:  2013    12    13     2     30      0 1386901800 1386900000
#> 2611290:  2013    12    13     3     30      0 1386905400 1386903600
#> 2611291:  2013    12    13     4     30      0 1386909000 1386907200
#> 2611292:  2013    12    13     5     30      0 1386912600 1386910800
#>          midpoint_time             datetime time_decimal time_interval
#>                  <int>               <char>        <num>         <int>
#>       1:    1254893400 2009-10-07T05:30:00Z     2009.765          3600
#>       2:    1254897000 2009-10-07T06:30:00Z     2009.765          3600
#>       3:    1254900600 2009-10-07T07:30:00Z     2009.765          3600
#>       4:    1254904200 2009-10-07T08:30:00Z     2009.765          3600
#>       5:    1254907800 2009-10-07T09:30:00Z     2009.765          3600
#>      ---                                                              
#> 2611288:    1386898200 2013-12-13T01:30:00Z     2013.948          3600
#> 2611289:    1386901800 2013-12-13T02:30:00Z     2013.948          3600
#> 2611290:    1386905400 2013-12-13T03:30:00Z     2013.948          3600
#> 2611291:    1386909000 2013-12-13T04:30:00Z     2013.948          3600
#> 2611292:    1386912600 2013-12-13T05:30:00Z     2013.949          3600
#>              value latitude longitude altitude elevation intake_height obs_flag
#>              <num>    <num>     <num>    <num>     <num>         <num>    <int>
#>       1: 1.907e-06  54.7050   73.0292      139       110            29        1
#>       2: 1.904e-06  54.7050   73.0292      139       110            29        1
#>       3: 1.903e-06  54.7050   73.0292      139       110            29        1
#>       4: 1.906e-06  54.7050   73.0292      139       110            29        1
#>       5: 1.903e-06  54.7050   73.0292      139       110            29        1
#>      ---                                                                       
#> 2611288: 1.938e-06  62.0886  129.3558      341       264            77        1
#> 2611289: 1.933e-06  62.0886  129.3558      341       264            77        1
#> 2611290: 1.943e-06  62.0886  129.3558      341       264            77        1
#> 2611291: 1.949e-06  62.0886  129.3558      341       264            77        1
#> 2611292: 1.947e-06  62.0886  129.3558      341       264            77        1
#>          obspack_num
#>                <int>
#>       1:     5856371
#>       2:     5856372
#>       3:     5856373
#>       4:     5856374
#>       5:     5856375
#>      ---            
#> 2611288:     6664376
#> 2611289:     6664377
#> 2611290:     6664378
#> 2611291:     6664379
#> 2611292:     6664380
#>                                                                                            obspack_id
#>                                                                                                <char>
#>       1: obspack_ch4_1_GLOBALVIEWplus_v5.1_2023-03-08~ch4_azv_tower-insitu_20_allvalid-29magl~5856371
#>       2: obspack_ch4_1_GLOBALVIEWplus_v5.1_2023-03-08~ch4_azv_tower-insitu_20_allvalid-29magl~5856372
#>       3: obspack_ch4_1_GLOBALVIEWplus_v5.1_2023-03-08~ch4_azv_tower-insitu_20_allvalid-29magl~5856373
#>       4: obspack_ch4_1_GLOBALVIEWplus_v5.1_2023-03-08~ch4_azv_tower-insitu_20_allvalid-29magl~5856374
#>       5: obspack_ch4_1_GLOBALVIEWplus_v5.1_2023-03-08~ch4_azv_tower-insitu_20_allvalid-29magl~5856375
#>      ---                                                                                             
#> 2611288: obspack_ch4_1_GLOBALVIEWplus_v5.1_2023-03-08~ch4_yak_tower-insitu_20_allvalid-77magl~6664376
#> 2611289: obspack_ch4_1_GLOBALVIEWplus_v5.1_2023-03-08~ch4_yak_tower-insitu_20_allvalid-77magl~6664377
#> 2611290: obspack_ch4_1_GLOBALVIEWplus_v5.1_2023-03-08~ch4_yak_tower-insitu_20_allvalid-77magl~6664378
#> 2611291: obspack_ch4_1_GLOBALVIEWplus_v5.1_2023-03-08~ch4_yak_tower-insitu_20_allvalid-77magl~6664379
#> 2611292: obspack_ch4_1_GLOBALVIEWplus_v5.1_2023-03-08~ch4_yak_tower-insitu_20_allvalid-77magl~6664380
#>          unique_sample_location_num year_st month_st day_st hour_st minute_st
#>                               <int>   <int>    <int>  <int>   <int>     <int>
#>       1:                   46849218    2009       10      7      10        34
#>       2:                   46849219    2009       10      7      11        34
#>       3:                   46849220    2009       10      7      12        34
#>       4:                   46849221    2009       10      7      13        34
#>       5:                   46849222    2009       10      7      14        34
#>      ---                                                                     
#> 2611288:                   47506934    2013       12     13      10        12
#> 2611289:                   47506935    2013       12     13      11        12
#> 2611290:                   47506936    2013       12     13      12        12
#> 2611291:                   47506937    2013       12     13      13        12
#> 2611292:                   47506938    2013       12     13      14        12
#>          second_st             scale dataset_intake_ht_unit site_elevation_unit
#>              <int>            <char>                 <char>              <char>
#>       1:        46 NIES 94 CH4 scale                   magl                masl
#>       2:        46 NIES 94 CH4 scale                   magl                masl
#>       3:        46 NIES 94 CH4 scale                   magl                masl
#>       4:        46 NIES 94 CH4 scale                   magl                masl
#>       5:        46 NIES 94 CH4 scale                   magl                masl
#>      ---                                                                       
#> 2611288:        48 NIES 94 CH4 scale                   magl                masl
#> 2611289:        48 NIES 94 CH4 scale                   magl                masl
#> 2611290:        48 NIES 94 CH4 scale                   magl                masl
#> 2611291:        48 NIES 94 CH4 scale                   magl                masl
#> 2611292:        48 NIES 94 CH4 scale                   magl                masl
#>          dataset_project dataset_selection_tag site_name site_elevation
#>                   <char>                <char>    <char>          <num>
#>       1:    tower-insitu       allvalid-29magl     Azovo            110
#>       2:    tower-insitu       allvalid-29magl     Azovo            110
#>       3:    tower-insitu       allvalid-29magl     Azovo            110
#>       4:    tower-insitu       allvalid-29magl     Azovo            110
#>       5:    tower-insitu       allvalid-29magl     Azovo            110
#>      ---                                                               
#> 2611288:    tower-insitu       allvalid-77magl   Yakutsk            264
#> 2611289:    tower-insitu       allvalid-77magl   Yakutsk            264
#> 2611290:    tower-insitu       allvalid-77magl   Yakutsk            264
#> 2611291:    tower-insitu       allvalid-77magl   Yakutsk            264
#> 2611292:    tower-insitu       allvalid-77magl   Yakutsk            264
#>          site_latitude site_longitude site_country site_code site_utc2lst
#>                  <num>          <num>       <char>    <char>        <num>
#>       1:       54.7050        73.0292       Russia       AZV            6
#>       2:       54.7050        73.0292       Russia       AZV            6
#>       3:       54.7050        73.0292       Russia       AZV            6
#>       4:       54.7050        73.0292       Russia       AZV            6
#>       5:       54.7050        73.0292       Russia       AZV            6
#>      ---                                                                 
#> 2611288:       62.0886       129.3558       Russia       YAK            9
#> 2611289:       62.0886       129.3558       Russia       YAK            9
#> 2611290:       62.0886       129.3558       Russia       YAK            9
#> 2611291:       62.0886       129.3558       Russia       YAK            9
#> 2611292:       62.0886       129.3558       Russia       YAK            9
#>          lab_1_abbr dataset_calibration_scale altitude_final type_altitude
#>              <char>                    <char>          <num>         <num>
#>       1:       NIES         NIES 94 CH4 scale             29             0
#>       2:       NIES         NIES 94 CH4 scale             29             0
#>       3:       NIES         NIES 94 CH4 scale             29             0
#>       4:       NIES         NIES 94 CH4 scale             29             0
#>       5:       NIES         NIES 94 CH4 scale             29             0
#>      ---                                                                  
#> 2611288:       NIES         NIES 94 CH4 scale             77             0
#> 2611289:       NIES         NIES 94 CH4 scale             77             0
#> 2611290:       NIES         NIES 94 CH4 scale             77             0
#> 2611291:       NIES         NIES 94 CH4 scale             77             0
#> 2611292:       NIES         NIES 94 CH4 scale             77             0
#>          inst_repeatability instrument method nvalue qcflag value_std_dev
#>                       <num>     <char> <char>  <int> <char>         <num>
#>       1:                 NA       <NA>   <NA>     NA   <NA>            NA
#>       2:                 NA       <NA>   <NA>     NA   <NA>            NA
#>       3:                 NA       <NA>   <NA>     NA   <NA>            NA
#>       4:                 NA       <NA>   <NA>     NA   <NA>            NA
#>       5:                 NA       <NA>   <NA>     NA   <NA>            NA
#>      ---                                                                 
#> 2611288:                 NA       <NA>   <NA>     NA   <NA>            NA
#> 2611289:                 NA       <NA>   <NA>     NA   <NA>            NA
#> 2611290:                 NA       <NA>   <NA>     NA   <NA>            NA
#> 2611291:                 NA       <NA>   <NA>     NA   <NA>            NA
#> 2611292:                 NA       <NA>   <NA>     NA   <NA>            NA
#>          value_unc
#>              <num>
#>       1:        NA
#>       2:        NA
#>       3:        NA
#>       4:        NA
#>       5:        NA
#>      ---          
#> 2611288:        NA
#> 2611289:        NA
#> 2611290:        NA
#> 2611291:        NA
#> 2611292:        NA
```

Now we can process the data. We first filter for observations
within our spatial domain:

## Checks and definitions

```{r spatial, eval = F}
north <- 80
south <- 10
west <- -170
east <- -50
max_altitude <- 8000
yy <- 2020
evening <- 14

```

We check altitude, intake_height, altitude_final and elevation.
altitude_final is a column from intake_height, added to match
column from obs_read text files.

```{r checkcols, eval = F}
df[, c("altitude", "altitude_final", "intake_height", "elevation",
       "dataset_selection_tag",
              "site_name")]
```

```
#>          altitude altitude_final intake_height elevation dataset_selection_tag
#>             <num>          <num>         <num>     <num>                <char>
#>       1:      139             29            29       110       allvalid-29magl
#>       2:      139             29            29       110       allvalid-29magl
#>       3:      139             29            29       110       allvalid-29magl
#>       4:      139             29            29       110       allvalid-29magl
#>       5:      139             29            29       110       allvalid-29magl
#>      ---                                                                      
#> 2611288:      341             77            77       264       allvalid-77magl
#> 2611289:      341             77            77       264       allvalid-77magl
#> 2611290:      341             77            77       264       allvalid-77magl
#> 2611291:      341             77            77       264       allvalid-77magl
#> 2611292:      341             77            77       264       allvalid-77magl
#>          site_name
#>             <char>
#>       1:     Azovo
#>       2:     Azovo
#>       3:     Azovo
#>       4:     Azovo
#>       5:     Azovo
#>      ---          
#> 2611288:   Yakutsk
#> 2611289:   Yakutsk
#> 2611290:   Yakutsk
#> 2611291:   Yakutsk
#> 2611292:   Yakutsk
```
The temporal range of data is
```{r range_year, eval = F}
range(df$year)
```

```
#> [1] 2004 2021
```

We also check for dimensions of data

```{r dim_df, eval = F}
dim(df)
```

```
#> [1] 2611292      51
```

## Filters

```{r spatial_temporal_filter, eval = F}
df <- df[year == yy]

df <- df[altitude_final < max_altitude &
           latitude < north &
           latitude > south &
           longitude < east &
           longitude > west]
dim(df)
```

```
#> [1] 96106    51
```

Towers can have observations at different
heights. Here we need to select one site with the observations
registered at the highest height. The column
with the height is named `altitude_final` and the
max altitude was named `max_altitude`.


```{r, eval = F}
dfa <- df[,
          max(altitude_final),
          by = site_code] |> unique()

names(dfa)[2] <- "max_altitude"
dfa
```

```
#>    site_code max_altitude
#>       <char>        <num>
#> 1:       CRV         31.7
#> 2:       LEF        396.0
#> 3:       SCT        304.8
#> 4:       WGC        484.0
```


### Key Time

Here we need to start time columns.
The function `obs_addtime` adds time columns
`timeUTC`, `timeUTC_start` which shows the start time
of each observation and `timeUTC_end`
which shows the end time for each observation.


```{r, eval = F}
df2 <- obs_addtime(df)
```

```
#> Adding timeUTC
#> Adding timeUTC_start
#> Adding timeUTC_end
#> Found time_interval
```

Then we need a *key_time* to aggregate data.
This can be done using UTC, solar, or local time.
The normal approach is using afternoon solar or local time.


#### Hierarchy of solar or local time

1) Solar time
2) Local time with columns `site_utc2lst`
3) Local time longitude


> solar time (default)

Here we select the hours of interest and then
aggregate data by year, month and day of solar time.
In this way, we will have one information per day. however
this approach is not appropriate for aircraft
which are aggregated every 10 or 20 seconds. Hence
we need to aggregate data by one time column.
Also, this helps to generate the receptor info 
files including hour, minute and second.
Hence, *we need to add solar or local time column*.

```{r, eval = F}
df2$solar_time <- obs_addstime(df2)

```


> local time with column `site_utc2lst`

Then we need to identify the local time with the function
`add_ltime`. This is important because to identifying
observations in the evening in local time.
`add_ltime` uses two methods, first identify
the time difference with utc by identifying the
metadata column "site_utc2lst". 
If solar time is not available
#now we need to cut solar time for the frequency needed. 
As we will work with 



> local time longitude

If this information
is not available, with the aircrafts for instance,
the local time is calculated with an approximation
based on longitude:


$$
lt = UTC + longitude/15 * 60 * 60
$$
Where $lt$ is the local time, $UTC$ the time,
$longitude$ the coordinate.
Then, the time is cut every two hours.
Now, we identify the local time to select evening hours.



#### Cut time

Now we have they key column time, we can cut it accordingly.

```{r, eval = F}
df2$solar_time_cut <- cut(x = df2$solar_time,
                          breaks = "1 hour") |>
  as.character()

```


How we can check the solar time and the cut solar time.
Please note that solar_time_cut,
the column that it will be used to aggregate data

How we filter for the required solar time, in this case 14.

```{r, eval = F}
df3 <- df2[hour_st %in% evening]
df3[, c("solar_time", "solar_time_cut")]


```

```
#>                solar_time      solar_time_cut
#>                    <POSc>              <char>
#>    1: 2019-12-31 14:37:48 2019-12-31 14:00:00
#>    2: 2020-01-01 14:35:49 2020-01-01 14:00:00
#>    3: 2020-01-02 14:35:23 2020-01-02 14:00:00
#>    4: 2020-01-03 14:34:57 2020-01-03 14:00:00
#>    5: 2020-01-04 14:34:31 2020-01-04 14:00:00
#>   ---                                        
#> 3676: 2020-12-27 14:22:57 2020-12-27 14:00:00
#> 3677: 2020-12-28 14:22:30 2020-12-28 14:00:00
#> 3678: 2020-12-29 14:22:02 2020-12-29 14:00:00
#> 3679: 2020-12-30 14:21:35 2020-12-30 14:00:00
#> 3680: 2020-12-31 14:21:08 2020-12-31 14:00:00
```


At this point we can calculate the averages of several columns by
the cut time. The function `obs_agg` does this aggregation as shown
in the following lines of code. The argument `gby`
establish the function used to aggregate `cols`. I need to aggregate the data
by date (year, month, date), because it is already filtered by
the hours of interest. Then, I would have 1 observation per day.

As standard, let us define `key_time` as `solar_time`. 
The `obs_agg` function will aggregate the desired data by that column.

```{r, eval = F}
df3$key_time <- df3$solar_time_cut
```


```{r, eval = F}
df4 <- obs_agg(dt = df3,
               cols = c("value",
                        "latitude",
                        "longitude",
                        "site_utc2lst"),
               verbose = T,
               byalt = TRUE)
```

```
#> Selecting by alt
#> Adding time
```

Here we add the column `max_altitude` to identify the max altitude
by site. 

```{r, eval = F}
df4[,
    max_altitude := max(altitude_final),
    by = site_code]
df4[,
    c("site_code",
      "altitude_final",
      "max_altitude")] |> unique()

```

```
#>     site_code altitude_final max_altitude
#>        <char>          <num>        <num>
#>  1:       CRV           17.1         31.7
#>  2:       CRV           31.7         31.7
#>  3:       CRV            4.9         31.7
#>  4:       LEF          122.0        396.0
#>  5:       LEF          396.0        396.0
#>  6:       LEF           30.0        396.0
#>  7:       SCT          304.8        304.8
#>  8:       SCT           31.0        304.8
#>  9:       SCT           61.0        304.8
#> 10:       WGC           30.0        484.0
#> 11:       WGC          484.0        484.0
#> 12:       WGC           89.1        484.0
```


## Master

Before generating the receptors list, we have the database with all the required information

```{r rename, eval = F}
master <- df4

```
We may replace missing values with a nine nines. Here is commented

#master[is.na(master)] <- 999999999

We transform the time variables to character and round coordinates with 4 digits

```{r round, eval = F}
master$timeUTC <- as.character(master$timeUTC)
master$local_time <- as.character(master$local_time)
master$latitude <- round(master$latitude, 4)
master$longitude <- round(master$longitude, 4)

```

## Save master
Finally we save the master file

```{r outfile, eval = F}
out <- tempfile()

```

### txt

```{r outtxt, eval = F}
message(paste0(out,"_", datasetid, ".txt\n"))
fwrite(master,
       paste0(out,"_", datasetid, ".txt"),
       sep = " ")
```

```
#> C:\Users\sibarrae\AppData\Local\Temp\RtmpyqH3NI\file5ef420fc3b21_tower-insitu.txt
```

### csv

```{r outcsv, eval = F}
message(paste0(out,"_", datasetid, ".csv\n"))
fwrite(master,
       paste0(out,"_", datasetid, ".csv"),
       sep = ",")
```

```
#> C:\Users\sibarrae\AppData\Local\Temp\RtmpyqH3NI\file5ef420fc3b21_tower-insitu.csv
```

### csvy

CSVY are csv files with a YAML header to include metadata in tabulated
text files

```{r csvy, eval = F}
cat("\nAdding notes in csvy:\n")
notes <- c(paste0("sector: ", datasetid),
           paste0("timespan: ", yy),
           paste0("spatial_limits: north = ", north, ", south = ", south, ", east = ", east, ", west = ", west),
           paste0("altitude: < ", max_altitude),
           paste0("hours: ", evening),
           "local_time: used solar_time")

cat(notes, sep = "\n")

message(paste0(out,"_", datasetid, ".csvy\n"))
obs_write_csvy(dt = master,
               notes = notes,
               out = paste0(out,"_", datasetid, ".csvy"))
```

```
#> Adding notes in csvy:
#> sector: tower-insitu
#> timespan: 2020
#> spatial_limits: north = 80, south = 10, east = -50, west = -170
#> data: Data averaged every 20 seconds
#> altitude: < 8000
#> hours: 14
#> local_time: used solar_time
#> C:\Users\sibarrae\AppData\Local\Temp\RtmpyqH3NI\file5ef420fc3b21_tower-insitu.csvy
```


```{r readcsvy, eval = F}
obs_read_csvy(paste0(out,"_", datasetid, ".csvy"))
```

```
#>  [1] "---"                                                                
#>  [2] "name: Metadata "                                                    
#>  [3] "sector: tower-insitu"                                               
#>  [4] "timespan: 2020"                                                     
#>  [5] "spatial_limits: north = 80, south = 10, east = -50, west = -170"    
#>  [6] "data: Data averaged every 20 seconds"                               
#>  [7] "altitude: < 8000"                                                   
#>  [8] "hours: 14"                                                          
#>  [9] "local_time: used solar_time"                                        
#> [10] "structure: "                                                        
#> [11] "Classes 'data.table' and 'data.frame':\t3680 obs. of  20 variables:"
#> [12] " $ timeUTC                  : chr  \"2019-12-31 14:00:00\" \".."    
#> [13] " $ site_code                : chr  \"CRV\" \"CRV\" ..."             
#> [14] " $ altitude_final           : num  17.1 31.7 ..."                   
#> [15] " $ type_altitude            : num  0 0 0 0 0 ..."                   
#> [16] " $ lab_1_abbr               : chr  \"NOAA\" \"NOAA\" ..."           
#> [17] " $ dataset_calibration_scale: chr  \"WMO CH4 X2004A\" \"WMO \".."   
#> [18] " $ value                    : num  1.97e-06 1.97e-06 ..."           
#> [19] " $ latitude                 : num  65 65 ..."                       
#> [20] " $ longitude                : num  -148 -148 ..."                   
#> [21] " $ site_utc2lst             : num  -8 -8 -8 -8 -8 ..."              
#> [22] " $ year                     : int  2019 2019 2019 2020 202.."       
#> [23] " $ month                    : int  12 12 12 1 1 ..."                
#> [24] " $ day                      : chr  \"31\" \"31\" ..."               
#> [25] " $ hour                     : int  14 14 14 14 14 ..."              
#> [26] " $ minute                   : int  0 0 0 0 0 ..."                   
#> [27] " $ second                   : int  0 0 0 0 0 ..."                   
#> [28] " $ time                     : num  1.58e+09 1.58e+09 ..."           
#> [29] " $ time_decimal             : num  2020 2020 ..."                   
#> [30] " $ max_altitude             : num  31.7 31.7 ..."                   
#> [31] " $ local_time               : chr  NA NA ..."                       
#> [32] " - attr(*, \".internal.selfref\")=<externalptr> "                   
#> [33] "NULL"                                                               
#> [34] "---"
#>                   timeUTC site_code altitude_final type_altitude lab_1_abbr
#>                    <POSc>    <char>          <num>         <int>     <char>
#>    1: 2019-12-31 14:00:00       CRV           17.1             0       NOAA
#>    2: 2019-12-31 14:00:00       CRV           31.7             0       NOAA
#>    3: 2019-12-31 14:00:00       CRV            4.9             0       NOAA
#>    4: 2020-01-01 14:00:00       CRV           17.1             0       NOAA
#>    5: 2020-01-01 14:00:00       CRV           31.7             0       NOAA
#>   ---                                                                      
#> 3676: 2020-12-30 14:00:00       WGC          484.0             0       NOAA
#> 3677: 2020-12-30 14:00:00       WGC           89.1             0       NOAA
#> 3678: 2020-12-31 14:00:00       WGC           30.0             0       NOAA
#> 3679: 2020-12-31 14:00:00       WGC          484.0             0       NOAA
#> 3680: 2020-12-31 14:00:00       WGC           89.1             0       NOAA
#>       dataset_calibration_scale        value latitude longitude site_utc2lst
#>                          <char>        <num>    <num>     <num>        <int>
#>    1:            WMO CH4 X2004A 1.970275e-06  64.9863 -147.5980           -8
#>    2:            WMO CH4 X2004A 1.967440e-06  64.9863 -147.5980           -8
#>    3:            WMO CH4 X2004A 1.971806e-06  64.9863 -147.5980           -8
#>    4:            WMO CH4 X2004A 1.988251e-06  64.9863 -147.5980           -8
#>    5:            WMO CH4 X2004A 1.987424e-06  64.9863 -147.5980           -8
#>   ---                                                                       
#> 3676:            WMO CH4 X2004A 2.004825e-06  38.2645 -121.4904           -8
#> 3677:            WMO CH4 X2004A 2.238231e-06  38.2645 -121.4904           -8
#> 3678:            WMO CH4 X2004A 2.048590e-06  38.2645 -121.4904           -8
#> 3679:            WMO CH4 X2004A 2.007957e-06  38.2645 -121.4904           -8
#> 3680:            WMO CH4 X2004A 2.048209e-06  38.2645 -121.4904           -8
#>        year month   day  hour minute second       time time_decimal
#>       <int> <int> <int> <int>  <int>  <int>      <int>        <num>
#>    1:  2019    12    31    14      0      0 1577800800     2019.999
#>    2:  2019    12    31    14      0      0 1577800800     2019.999
#>    3:  2019    12    31    14      0      0 1577800800     2019.999
#>    4:  2020     1     1    14      0      0 1577887200     2020.002
#>    5:  2020     1     1    14      0      0 1577887200     2020.002
#>   ---                                                              
#> 3676:  2020    12    30    14      0      0 1609336800     2020.996
#> 3677:  2020    12    30    14      0      0 1609336800     2020.996
#> 3678:  2020    12    31    14      0      0 1609423200     2020.999
#> 3679:  2020    12    31    14      0      0 1609423200     2020.999
#> 3680:  2020    12    31    14      0      0 1609423200     2020.999
#>       max_altitude local_time
#>              <num>     <lgcl>
#>    1:         31.7         NA
#>    2:         31.7         NA
#>    3:         31.7         NA
#>    4:         31.7         NA
#>    5:         31.7         NA
#>   ---                        
#> 3676:        484.0         NA
#> 3677:        484.0         NA
#> 3678:        484.0         NA
#> 3679:        484.0         NA
#> 3680:        484.0         NA
```



## Receptors

Now we can do the last step which is generating the receptor list files.
Now we filter selected columns

```{r receptors, eval = F}
receptor <- master[, c("site_code",
                     "year",
                     "month",
                     "day",
                     "hour",
                     "minute",
                     "second",
                     "latitude",
                     "longitude",
                     "altitude_final",
                     "type_altitude",
                     "time_decimal")]

```

We can round altitude also

```{r round_alt, eval = F}
receptor$altitude_final <- round(receptor$altitude_final)
```

Now we can format time variables with two digits

```{r formatrec, eval = F}
receptor <- obs_format(receptor,
                        spf =  c("month", "day",
                                 "hour", "minute", "second"))
```

We have a column that indicate AGL or ASL

```{r aslagl, eval = F}
receptor_agl <- receptor[type_altitude == 0]
receptor_asl <- receptor[type_altitude == 1]
```

Finally, we save the receptors

```{r save_receptors, eval = F}
if(nrow(receptor_agl) > 0) {
  message(paste0(out, "_", datasetid, "_receptor_AGL.txt"), "\n")

  fwrite(x = receptor_agl,
         file = paste0(out, "_", datasetid, "_receptor_AGL.txt"),
         sep = " ")
}

if(nrow(receptor_asl) > 0) {
  message(paste0(out, "_", datasetid, "_receptor_ASL.txt"), "\n")

  fwrite(x = receptor_asl,
         file = paste0(out, "_", datasetid, "receptor_ASL.txt"),
         sep = " ")

}
```

```
#> C:\Users\sibarrae\AppData\Local\Temp\RtmpyqH3NI\file5ef420fc3b21_tower-insitu_receptor_AGL.txt
```


## Plot

Finally, we just plot some data, run it locally


```{r obs_plotsave, fig.width=5, fig.height=3, eval = F, echo = F, message=F, warning=F}
png("../man/figures/obsplot_towerinsitu.png", width = 1500, height = 1000, res = 200)
obs_plot(df3, time = "timeUTC", yfactor = 1e9)
dev.off()
```

```{r obs_plot, fig.width=5, fig.height=3, eval = F}
obs_plot(df4, time = "timeUTC", yfactor = 1e9)
```
```
#> Found the following sites: 
#> [1] CRV LEF SCT WGC
#> Plotting the following sites: 
#> [1] CRV LEF
#> png 
#>   2
```
![Time series](https://github.com/ibarraespinosa/rtorf/blob/main/man/figures/obsplot_towerinsitu.png?raw=true)


```{r savesf, fig.width=5, fig.height=3, eval = F, echo = F}
library(sf)
dx <- df4[, 
    lapply(.SD, mean),
    .SDcols = "value",
    by = .(latitude, longitude)]
x <- st_as_sf(dx, coords = c("longitude", "latitude"), crs = 4326)
png("../man/figures/obsplot_towerinsitu_map.png", width = 1500, height = 1000, res = 200)
plot(x["value"], axes = T, reset = F)
maps::map(add = T)
dev.off()
```


```{r sf, fig.width=5, fig.height=3, eval = F}
library(sf)
dx <- df4[, 
    lapply(.SD, mean),
    .SDcols = "value",
    by = .(latitude, longitude)]
x <- st_as_sf(dx, coords = c("longitude", "latitude"), crs = 4326)
plot(x["value"], axes = T, reset = F)
maps::map(add = T)
```

![Map](https://github.com/ibarraespinosa/rtorf/blob/main/man/figures/obsplot_towerinsitu_map.png?raw=true)
