Skip to content

Commit

Permalink
Merge branch 'master' into useTerra
Browse files Browse the repository at this point in the history
  • Loading branch information
brownag authored Jan 15, 2022
2 parents f50761e + cd091d5 commit 07060e9
Show file tree
Hide file tree
Showing 127 changed files with 1,379 additions and 635 deletions.
2 changes: 1 addition & 1 deletion DESCRIPTION
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
Package: soilDB
Type: Package
Title: Soil Database Interface
Version: 2.6.11
Version: 2.6.13
Authors@R: c(person(given="Dylan", family="Beaudette", role = c("aut"), email = "dylan.beaudette@usda.gov"),
person(given="Jay", family="Skovlin", role = c("aut")),
person(given="Stephen", family="Roecker", role = c("aut")),
Expand Down
11 changes: 11 additions & 0 deletions NEWS.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,14 @@
# soilDB 2.6.13 (2022-01-11)
* `fetchSDA_spatial` now supports `by.col` `"areaname"`, `"mlraoffice"`, and `"mouagencyresp"`; thanks to suggestion by Jay Skovlin
* `fetchNASIS` fix for multiple site observation records with surface fragments; thanks to bug report from Brianna Wegner
* `waterYearDay()` use format and timezone for start date conversion

# soilDB 2.6.12 (2022-01-07)
* `get_SDA_property()` now works with mixed component and horizon-level properties; thanks to Matthieu Stigler for the bug report
* Added `get_SDV_legend_elements()` for fetching and parsing XML for Soil Data Viewer / Web Soil Survey-style symbol themes for soil interpretations from Soil Data Access
* `fetchNASIS()` pedon and component geomorphic summaries now include columns for landscape, microfeature, microrelief, 2D/3D morphometry, and slope shape
* `fetchNASIS('pedons')` now uses `simplifyFragmentData()` for surface fragments

# soilDB 2.6.11 (2021-12-21)
* `fetchSDA()` handle NULL component-level results with an informative error
* `fetchSDA()` now (again) returns mapunit/legend-level information via `get_mapunit_from_NASIS()`
Expand Down
1 change: 1 addition & 0 deletions R/ISSR800.R
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
#' @param quiet logical, passed to \code{download.file} to enable / suppress URL and progress bar for download.
#'
#' @details \code{aoi} should be specified as a \code{Spatial*}, \code{RasterLayer}, \code{SpatRaster}/\code{SpatVector}, \code{sf}, \code{sfc}, or \code{bbox} object or a \code{list} containing:

#'
#' \describe{
#' \item{\code{aoi}}{bounding-box specified as (xmin, ymin, xmax, ymax) e.g. c(-114.16, 47.65, -114.08, 47.68)}
Expand Down
79 changes: 53 additions & 26 deletions R/SDA_properties.R
Original file line number Diff line number Diff line change
Expand Up @@ -168,15 +168,15 @@ get_SDA_property <-
"Min/Max",
"Dominant Component (Numeric)",
"Dominant Condition",
"None", "None_Horizon")
"None")
method <- match.arg(toupper(method), toupper(labels))

# determine column name prefix/suffix for method
suffixes <- c('_dom_comp_cat',
'_wtd_avg',
'_min_max',
'_dom_comp_num',
'_dom_cond', '', 'chorizon_')
'_dom_cond', '')
modifier <- suffixes[match(method, toupper(labels))]

# return list with method and modifier
Expand Down Expand Up @@ -305,7 +305,7 @@ get_SDA_property <-

## alternate: just assume they are either all component or all horizon column names
# message('assuming `property` is a vector of component OR horizon-level column names')
agg_property <- property
agg_property <- tolower(property)

} else {

Expand All @@ -318,10 +318,44 @@ get_SDA_property <-

method <- toupper(method)

if (method == "NONE")
if (all(agg_property %in% colnames(suppressMessages(SDA_query("SELECT TOP 1 * FROM chorizon")))))
method <- "NONE_HORIZON"

if (method == "NONE") {
# dput(colnames(suppressMessages(SDA_query("SELECT TOP 1 * FROM chorizon")))) # without cokey
is_hz <- agg_property %in% c("hzname", "desgndisc", "desgnmaster", "desgnmasterprime", "desgnvert",
"hzdept_l", "hzdept_r", "hzdept_h", "hzdepb_l", "hzdepb_r", "hzdepb_h",
"hzthk_l", "hzthk_r", "hzthk_h", "fraggt10_l", "fraggt10_r",
"fraggt10_h", "frag3to10_l", "frag3to10_r", "frag3to10_h", "sieveno4_l",
"sieveno4_r", "sieveno4_h", "sieveno10_l", "sieveno10_r", "sieveno10_h",
"sieveno40_l", "sieveno40_r", "sieveno40_h", "sieveno200_l",
"sieveno200_r", "sieveno200_h", "sandtotal_l", "sandtotal_r",
"sandtotal_h", "sandvc_l", "sandvc_r", "sandvc_h", "sandco_l",
"sandco_r", "sandco_h", "sandmed_l", "sandmed_r", "sandmed_h",
"sandfine_l", "sandfine_r", "sandfine_h", "sandvf_l", "sandvf_r",
"sandvf_h", "silttotal_l", "silttotal_r", "silttotal_h", "siltco_l",
"siltco_r", "siltco_h", "siltfine_l", "siltfine_r", "siltfine_h",
"claytotal_l", "claytotal_r", "claytotal_h", "claysizedcarb_l",
"claysizedcarb_r", "claysizedcarb_h", "om_l", "om_r", "om_h",
"dbtenthbar_l", "dbtenthbar_r", "dbtenthbar_h", "dbthirdbar_l",
"dbthirdbar_r", "dbthirdbar_h", "dbfifteenbar_l", "dbfifteenbar_r",
"dbfifteenbar_h", "dbovendry_l", "dbovendry_r", "dbovendry_h",
"partdensity", "ksat_l", "ksat_r", "ksat_h", "awc_l", "awc_r",
"awc_h", "wtenthbar_l", "wtenthbar_r", "wtenthbar_h", "wthirdbar_l",
"wthirdbar_r", "wthirdbar_h", "wfifteenbar_l", "wfifteenbar_r",
"wfifteenbar_h", "wsatiated_l", "wsatiated_r", "wsatiated_h",
"lep_l", "lep_r", "lep_h", "ll_l", "ll_r", "ll_h", "pi_l", "pi_r",
"pi_h", "aashind_l", "aashind_r", "aashind_h", "kwfact", "kffact",
"caco3_l", "caco3_r", "caco3_h", "gypsum_l", "gypsum_r", "gypsum_h",
"sar_l", "sar_r", "sar_h", "ec_l", "ec_r", "ec_h", "cec7_l",
"cec7_r", "cec7_h", "ecec_l", "ecec_r", "ecec_h", "sumbases_l",
"sumbases_r", "sumbases_h", "ph1to1h2o_l", "ph1to1h2o_r", "ph1to1h2o_h",
"ph01mcacl2_l", "ph01mcacl2_r", "ph01mcacl2_h", "freeiron_l",
"freeiron_r", "freeiron_h", "feoxalate_l", "feoxalate_r", "feoxalate_h",
"extracid_l", "extracid_r", "extracid_h", "extral_l", "extral_r",
"extral_h", "aloxalate_l", "aloxalate_r", "aloxalate_h", "pbray1_l",
"pbray1_r", "pbray1_h", "poxalate_l", "poxalate_r", "poxalate_h",
"ph2osoluble_l", "ph2osoluble_r", "ph2osoluble_h", "ptotal_l",
"ptotal_r", "ptotal_h", "excavdifcl", "excavdifms", "chkey")
}

FUN <- toupper(FUN)

# check FUN arg for min max method
Expand Down Expand Up @@ -504,31 +538,24 @@ paste0(sprintf("#last_step2.%s", property), collapse = ", ")))
paste0(sapply(agg_property, .property_dominant_condition_category), collapse = ", "),
where_clause),

# NO AGGREGATION (component properties)
# NO AGGREGATION
"NONE" = sprintf("SELECT areasymbol, musym, muname, mu.mukey/1 AS mukey,
c.compname AS compname, c.comppct_r AS comppct_r, c.majcompflag AS majcompflag,
c.cokey AS cokey,
%s
c.cokey AS cokey, %s %s%s %s
FROM legend AS l
INNER JOIN mapunit AS mu ON mu.lkey = l.lkey AND %s
INNER JOIN component AS c ON c.mukey = mu.mukey
ORDER BY areasymbol, musym, muname, mu.mukey, c.comppct_r DESC, c.cokey",
paste0(sapply(agg_property, function(x) sprintf("c.%s AS %s", x, x)), collapse = ", "),
where_clause),

# NO AGGREGATION (horizon properties)
"NONE_HORIZON" = sprintf("SELECT areasymbol, musym, muname, mu.mukey/1 AS mukey,
c.cokey AS cokey, ch.chkey AS chkey,
%s
ORDER BY areasymbol, musym, muname, mu.mukey, c.comppct_r DESC, c.cokey%s",
ifelse(any(is_hz), "ch.chkey AS chkey,
c.compname AS compname, c.comppct_r AS comppct_r, c.majcompflag AS majcompflag,
ch.hzdept_r AS hzdept_r, ch.hzdepb_r AS hzdepb_r,
%s
FROM legend AS l
INNER JOIN mapunit AS mu ON mu.lkey = l.lkey AND %s
INNER JOIN component AS c ON c.mukey = mu.mukey
INNER JOIN chorizon AS ch ON ch.cokey = c.cokey
ORDER BY areasymbol, musym, muname, mu.mukey, c.comppct_r DESC, c.cokey, hzdept_r",
paste0(sapply(agg_property, function(x) sprintf("ch.%s AS %s", x, x)), collapse = ", "),
where_clause)
ch.hzdept_r AS hzdept_r, ch.hzdepb_r AS hzdepb_r,", ""),
paste0(sapply(agg_property[!is_hz], function(x) sprintf("c.%s AS %s", x, x)), collapse = ", "),
ifelse(any(is_hz) & !all(is_hz), ",", ""),
paste0(sapply(agg_property[is_hz], function(x) sprintf("ch.%s AS %s", x, x)), collapse = ", "),
where_clause,
ifelse(any(is_hz), "INNER JOIN chorizon AS ch ON ch.cokey = c.cokey", ""),
ifelse(any(is_hz), ", hzdept_r", ""))
)

}
116 changes: 116 additions & 0 deletions R/SoilDataViewer.R
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
#' Get Soil Data Viewer Attribute Information
#'
#' @param WHERE WHERE clause for query of Soil Data Access `sdvattribute` table
#' @param alpha transparency value applied in calculation of hexadecimal color. Default: `255` (opaque).
#' @param notratedcolor Used to add 'Not rated' color entries where applicable. Default: `"#FFFFFF00"` (transparent white).
#' @param simplify Return a data.frame when `WHERE` is length 1? Return a list with 1 element per legend when `WHERE` is length > `1`? Default: `TRUE`
#'
#' @return A list with a data.frame element for each element of `where` containing `"attributekey"`, `"attributename"`, `"attributetype"`, `"attributetablename"`, `"attributecolumnname"`, `"attributedescription"`, `"nasisrulename"`, `"label"`, `"order"`, `"value"`, `"lower_value"`, `"upper_value"`,`"red"`, `"green"`, `"blue"` and `"hex"` columns.
#' @export
#'
#' @importFrom xml2 read_xml as_list
get_SDV_legend_elements <- function(WHERE,
alpha = 255,
notratedcolor = rgb(1, 1, 1, 0),
simplify = TRUE) {

y <- lapply(WHERE, function(ak) {

x <- SDA_query(paste0("SELECT attributekey, attributename, attributetype,
attributetablename, attributecolumnname,
attributedescription, maplegendxml,
nasisrulename, notratedphrase
FROM sdvattribute WHERE ", ak))

if (inherits(x, 'try-error'))
stop(paste0("Invalid WHERE clause: ", ak), call. = FALSE)

lapply(1:nrow(x), function(i) {
.process_SDV_legend_elements(x[i, ],
alpha = alpha,
notratedcolor = notratedcolor)
})
})

if ((length(y) == 1) && length(y[[1]]) == 1 && simplify) {
return(y[[1]][[1]])
} else if (simplify) return(do.call('c', y))
y
}

.process_SDV_legend_elements <- function(x,
alpha = 255,
notratedcolor = rgb(1, 1, 1, 0)) {

# parse map legend symbology/elements
x2 <- xml2::as_list(xml2::read_xml(x$maplegendxml))

# just returning the Legend_Elements as a data.frame
res <- do.call('rbind', lapply(x2$Map_Legend$Legend_Elements,
function(z){

# handle single value labels versus upper/lower bounds
val <- attr(z, 'value')
attvalue <- data.frame(
value = val,
lower_value = NA[length(val)],
upper_value = NA[length(val)]
)
if (nrow(attvalue) == 0) {
attvalue <- data.frame(
value = NA,
lower_value = attr(z, 'lower_value'),
upper_value = attr(z, 'upper_value')
)
}
d <- data.frame(
attributekey = x$attributekey,
attributename = x$attributename,
attributetype = x$attributetype,
attributetablename = x$attributetablename,
attributecolumnname = x$attributecolumnname,
attributedescription = x$attributedescription,
nasisrulename = x$nasisrulename,
label = attr(z, 'label'),
order = attr(z, 'order'))
d <- cbind(d, attvalue)
d2 <- data.frame(
red = attr(z$Color, 'red'),
green = attr(z$Color, 'green'),
blue = attr(z$Color, 'blue')
)
if (nrow(d2) == 0) {
d2 <- data.frame(red = NA,
green = NA,
blue = NA,
hex = notratedcolor)
} else {
newcolor <- rgb(
red = d2$red,
green = d2$green,
blue = d2$blue,
alpha = alpha,
maxColorValue = 255
)
newcolor <- ifelse(length(newcolor) == 0, NA, newcolor)
d2$hex <- newcolor
}

cbind(d, d2)
}))
if (!is.na(x$notratedphrase) && x$notratedphrase != ""){
dnr <- res[1,]
dnr$value <- gsub("^not", "Not", x$notratedphrase)
dnr$label <- dnr$value
dnr$order = 0
dnr$red = NA
dnr$green = NA
dnr$blue = NA
dnr$hex <- notratedcolor
res <- rbind(res, dnr)
}
rownames(res) <- NULL
type.convert(res, as.is = TRUE)
}


16 changes: 5 additions & 11 deletions R/fetchNASIS.R
Original file line number Diff line number Diff line change
Expand Up @@ -39,17 +39,9 @@
#' - [fetchNASIS Components Tutorial](http://ncss-tech.github.io/AQP/soilDB/NASIS-component-data.html)
#'
#' @aliases fetchNASIS get_phorizon_from_NASIS_db
#' get_component_copm_data_from_NASIS_db
#' get_component_horizon_data_from_NASIS_db
#' get_component_correlation_data_from_NASIS_db
#' get_component_cogeomorph_data_from_NASIS_db
#' get_component_esd_data_from_NASIS_db
#' get_component_otherveg_data_from_NASIS_db get_copedon_from_NASIS_db
#' get_legend_from_NASISget_lmuaoverlap_from_NASIS get_mapunit_from_NASIS
#' get_projectmapunit_from_NASIS get_component_diaghz_from_NASIS_db
#' get_mutext_from_NASIS_db get_phfmp_from_NASIS_db get_RMF_from_NASIS_db
#' get_phfmp_from_NASIS_db get_RMF_from_NASIS_db
#' get_concentrations_from_NASIS_db
#' get_cotext_from_NASIS_db
#'
#' @param from determines what objects should fetched? ('pedons' | 'components' | 'pedon_report')
#' @param url string specifying the url for the NASIS pedon_report (default:
#' `NULL`)
Expand All @@ -71,7 +63,9 @@
#' @param dsn Optional: path to local SQLite database containing NASIS
#' table structure; default: `NULL`
#' @return A SoilProfileCollection object
#' @seealso `get_component_data_from_NASIS()`
#' @author D. E. Beaudette, J. M. Skovlin, S.M. Roecker, A.G. Brown
#'
#' @export fetchNASIS
fetchNASIS <- function(from = 'pedons',
url = NULL,
Expand Down Expand Up @@ -113,7 +107,7 @@ fetchNASIS <- function(from = 'pedons',

if (from == 'components') {
# pass arguments through
res <- .fetchNASIS_components(SS = TRUE,
res <- .fetchNASIS_components(SS = SS,
rmHzErrors = rmHzErrors,
nullFragsAreZero = nullFragsAreZero,
fill = fill,
Expand Down
4 changes: 2 additions & 2 deletions R/fetchNASIS_components.R
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
f.comp <- get_component_data_from_NASIS_db(SS = SS, stringsAsFactors = stringsAsFactors, dsn = dsn, nullFragsAreZero = nullFragsAreZero)
f.chorizon <- get_component_horizon_data_from_NASIS_db(SS = SS, fill = fill, dsn = dsn, nullFragsAreZero = nullFragsAreZero)
f.copm <- get_component_copm_data_from_NASIS_db(SS = SS, stringsAsFactors = stringsAsFactors, dsn = dsn)
f.cogeomorph <- get_component_cogeomorph_data_from_NASIS_db(SS = SS, dsn = dsn)
f.cogeomorph <- get_component_cogeomorph_data_from_NASIS_db2(SS = SS, dsn = dsn)
f.otherveg <- get_component_otherveg_data_from_NASIS_db(SS = SS, dsn = dsn)
f.ecosite <- get_component_esd_data_from_NASIS_db(SS = SS, stringsAsFactors = stringsAsFactors, dsn = dsn)
f.diaghz <- get_component_diaghz_from_NASIS_db(SS = SS, dsn = dsn)
Expand Down Expand Up @@ -66,7 +66,7 @@
# add site data to object
site(f.chorizon) <- f.comp # left-join via coiid

## 2017-3-13: short-circuts need testing, consider pre-marking mistakes before parsing
## 2017-3-13: short-circuits need testing, consider pre-marking mistakes before parsing
## 2021-10-28: TODO: harmonize strategies for .formatXXXXString methods and ID variables
.SD <- NULL
.BY <- NULL
Expand Down
35 changes: 19 additions & 16 deletions R/fetchNASIS_pedons.R
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,10 @@
# add site data to object
# remove 'pedon_id' column from site_data
site_data$pedon_id <- NULL


# TODO: duplicating surface fine gravel column with old name for backward compatibility
site_data$surface_fgravel <- site_data$surface_fine_gravel

# left-join via peiid
# < 0.1 second for ~ 4k pedons
site(hz_data) <- site_data
Expand Down Expand Up @@ -207,20 +210,20 @@
hz_data$total_art_pct <- ifelse(is.na(hz_data$total_art_pct), 0, hz_data$total_art_pct)
}

## TODO: convert this to simplifyFragmentData
# add surface frag summary
sfs <- extended_data$surf_frag_summary

# optionally convert NA fragvol to 0
if (nullFragsAreZero) {
sfs <- as.data.frame(
cbind(sfs[, 1, drop = FALSE],
lapply(sfs[, -1], function(i) ifelse(is.na(i), 0, i))
), stringsAsFactors = FALSE)
}

# add surf. frag summary to @site
site(hz_data) <- sfs
## 2021-11-05: converted surface frag summary to simplifyFragmentData() in get_site_data_from_NASIS_db()
# # add surface frag summary
# sfs <- extended_data$surf_frag_summary
#
# # optionally convert NA fragvol to 0
# if (nullFragsAreZero) {
# sfs <- as.data.frame(
# cbind(sfs[, 1, drop = FALSE],
# lapply(sfs[, -1], function(i) ifelse(is.na(i), 0, i))
# ), stringsAsFactors = FALSE)
# }
#
# # add surf. frag summary to @site
# site(hz_data) <- sfs

# load diagnostic horizons into @diagnostic:
# supress warnings: diagnostic_hz() <- is noisy when not all profiles have diagnostic hz data
Expand All @@ -238,7 +241,7 @@
by = list(peiid = ed.lf$peiid)]

if (ncol(lf) > 1)
site(hz_data) <- as.data.frame(lf[,c("peiid","landform_string")])
site(hz_data) <- as.data.frame(lf[,c("peiid","landform_string","landscape_string","microfeature_string", "geomicrorelief_string")])

ed.pm <- data.table::as.data.table(extended_data$pm)
pm <- ed.pm[, .formatParentMaterialString(.SD, uid = .BY$siteiid, name.sep = ' & '),
Expand Down
Loading

0 comments on commit 07060e9

Please sign in to comment.