#8 remove prefix

DESCRIPTION

@@ -1,9 +1,9 @@
 Package: thredds
 Title: Crawler for Navigating THREDDS Catalogs
-Description: Provides a crawler for programmatically navigating THREDDS Data Server (<https://www.unidata.ucar.edu/software/tds/>) 
+Description: Provides a crawler for programmatically navigating THREDDS Data Server (<https://www.unidata.ucar.edu/software/tds>) 
   catalogs, and access dataset metadata and resources.
-Version: 0.1-4
-Date: 2023-09-01
+Version: 0.1-5
+Date: 2026-09-04
 Depends:
     R (>= 3.0)
 Imports:
@@ -18,7 +18,7 @@ Authors@R: c(
   person("Ben", "Tupper", role = c("aut"), email = "btupper@bigelow.org"),
   person("Emmanuel", "Blondel", role = c("aut", "cre"), email = "emmanuel.blondel1@gmail.com", comment = c(ORCID = "0000-0002-5870-5762")),
   person("Bigelow Laboratory for Ocean Sciences", role = c("cph"), email = "info@bigelow.org"))
-URL: https://github.com/BigelowLab/thredds, https://www.unidata.ucar.edu/software/tds/
+URL: https://github.com/BigelowLab/thredds, https://www.unidata.ucar.edu/software/tds
 License: MIT + file LICENSE
 Encoding: UTF-8
-RoxygenNote: 7.2.3
+RoxygenNote: 7.3.3

R/CatalogNode.R

@@ -17,31 +17,29 @@
 #' @examples
 #'  \donttest{
 #'   library(thredds)
-#'   top_uri <- 'https://oceandata.sci.gsfc.nasa.gov/opendap/catalog.xml'
+#'   top_uri <- 'https://psl.noaa.gov/thredds/catalog/Datasets/godas/catalog.xml'
 #'   Top <- thredds::CatalogNode$new(top_uri)
 #'   
 #'   #to browse catalogue
 #'   #Top$browse()
 #'   
-#'   #go down in 'MODISA' catalog
-#'   L3 <- Top$get_catalogs("MODISA")[["MODISA"]]$get_catalogs()[[1]]
-#'   
-#'   #see what's available for 2009
-#'   catalog2009 <- L3$get_catalogs("2009")[[1]]
-#'   
-#'   #get catalog for 2009-01-20 
-#'   doy <- format(as.Date("2009-01-20"), "%m%d")
-#'   catalog20 <- catalog2009$get_catalogs(doy)[[doy]]
+#'   #go down in 'Derived' catalog
+#'   cat_name = "Derived"
+#'   subcat <- Top$get_catalogs(cat_name)[[cat_name]]
 #'   
 #'   #get dataset node
-#'   chl <- catalog20$get_datasets("AQUA_MODIS.20090120.L3m.DAY.CHL.chlor_a.4km.nc")[[1]]
+#'   ds_name = "salt.mon.ltm.1991-2020.nc"
+#'   ds <- subcat$get_datasets(ds_name)[[1]]
 #'   
 #'   #retrieve the relative URL, and add it to the base URL for the service.
 #'   #Somewhat awkwardly, the relative URL comes prepended with a path separator, so we 
 #'   #use straight up `paste0` to append to the base_uri.
 #'   #if(require("ncdf4")){
-#'   #  base_uri <- "https://oceandata.sci.gsfc.nasa.gov:443/opendap"
-#'   #  uri <- paste0(base_uri, chl[["AQUA_MODIS.20090120.L3m.DAY.CHL.chlor_a.4km.nc"]]$url)
+#'   #  base_uri = paste0(
+#'   #   "https://", gsub("^(https?://)?([^/]+).*", "\\\2", subcat$url), 
+#'   #   subcat$list_services()$odap[["base"]]
+#'   #  )
+#'   #  uri <- paste0(base_uri, ds$url)
 #'   #  NC <- ncdf4::nc_open(uri)
 #'   #}
 #'  }

R/thredds-package.R

@@ -1,7 +1,6 @@
 #' @name thredds
 #' @aliases thredds-package
 #' @aliases thredds
-#' @docType package
 #' 
 #' @title thredds: Crawler for Navigating THREDDS Catalogs
 #'
@@ -11,4 +10,4 @@
 #' @import rlang
 #' @import httr
 #' @importFrom magrittr %>%
-NULL
+"_PACKAGE"

README.md

@@ -3,10 +3,10 @@
 [![Build Status](https://github.com/BigelowLab/thredds/actions/workflows/r-cmd-check.yml/badge.svg?branch=master)](https://github.com/BigelowLab/thredds/actions/workflows/r-cmd-check.yml)
 [![CRAN_Status_Badge](https://www.r-pkg.org/badges/version/thredds)](https://cran.r-project.org/package=thredds)
 [![cran checks](https://badges.cranchecks.info/worst/thredds.svg)](https://cran.r-project.org/web/checks/check_results_thredds.html)
-[![Github_Status_Badge](https://img.shields.io/badge/Github-0.1--4-blue.svg)](https://github.com/BigelowLab/thredds)
+[![Github_Status_Badge](https://img.shields.io/badge/Github-0.1--5-blue.svg)](https://github.com/BigelowLab/thredds)
 [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.6027224.svg)](https://doi.org/10.5281/zenodo.6027224)
 
-[THREDDS](https://www.unidata.ucar.edu/software/tds/) catalogs
+[THREDDS](https://www.unidata.ucar.edu/software/tds) catalogs
 are well described.  This package provides only client-side functionality where the user provides
 prior knowledge about how the catalog is organized, as on the server side the provider has some 
 latitude in how to design the catalog system.
@@ -15,7 +15,7 @@ latitude in how to design the catalog system.
 A user's workflow likely is to fetch a top-level catalog, then drill down to a particular sub-catalog
 by hop-skipping through lightweight catalog references.  Often, but not always these
 catalogs are organized around date (a year of observation, a month of observation, etc) or
-a data source ("MODISA" vs "MODIST"), etc. Catalogs may contain references to other catalogs or 
+a data source, etc. Catalogs may contain references to other catalogs or 
 to datasets (often OPeNDAP resources.)
 
 This package replaces [threddscrawler](https://github.com/BigelowLab/threddscrawler) which is
@@ -36,105 +36,75 @@ is based upon [xml2](https://CRAN.R-project.org/package=xml2), and uses
 
 #### Installation
 
-It is easy to install with [devtools](https://CRAN.R-project.org/package=devtools)
+It is easy to install, either from CRAN:
+
+```R
+install.packages("remotes")
+```
+
+or with [remotes](https://CRAN.R-project.org/package=remotes):
+
 ```R
-library(devtools)
+library(remotes)
 install_github("BigelowLab/thredds")
 ```
 
-#### An example from [OBPG](https://oceancolor.gsfc.nasa.gov/)
+#### An example from [OBPG](https://psl.noaa.gov/)
 
-Start with this [page](https://oceandata.sci.gsfc.nasa.gov/opendap/) and it's [XML companion](https://oceandata.sci.gsfc.nasa.gov/opendap/catalog.xml). We find a top level
+Start with this [page](https://psl.noaa.gov/thredds/catalog/Datasets/catalog.html) and it's [XML companion](https://psl.noaa.gov/thredds/catalog/Datasets/catalog.xml). We find a top level
 catalog with a number of sub-catalogs.
 
-```
+```R
 library(ncdf4)
 library(thredds)
-top_uri <- 'https://oceandata.sci.gsfc.nasa.gov/opendap/catalog.xml'
+top_uri <- 'https://psl.noaa.gov/thredds/catalog/Datasets/catalog.xml'
 Top <- thredds::CatalogNode$new(top_uri, prefix = "thredds")
 Top
 # CatalogNode (R6): 
-#   verbose: FALSE    tries: 3    namespace prefix: thredds
-#   url: https://oceandata.sci.gsfc.nasa.gov/opendap/catalog.xml
-#   services [3]: OPeNDAP HTTPServer WCS
-#   catalogRefs [14]: CZCS MERIS MODISA ... SeaWiFS VIIRS VIIRSJ1
-#   datasets [1]: /
+#   verbose: FALSE    tries: 3    namespace prefix: d1
+#   url: https://psl.noaa.gov/thredds/catalog/Datasets/catalog.xml
+#   services [4]: Compound OpenDAP HTTPServer NetcdfSubset
+#   catalogRefs [71]: 20thC_ReanV2 20thC_ReanV2c 20thC_ReanV3 ... snowcover udel.airt.precip uninterp_OLR
+#   datasets [1]: Datasets
 
 Top$browse()
 ```
 
-We'll drill down into `MODISA` which only contains one sub-catalog, `L3SMI` - 
-the gridded level 3 standard mapped image. Knowing that we'll actually chain the
-methods to get the contents of the L3SMI catalog, where thinsg get interesting. Note
-the get_catalogs *always* returns a list, so you must index into it if you want just one result.
+We'll drill down into `godas` sub-catalog, and then the `Derived` sub-catalog to find derived datasets.
 
-```
-L3 <- Top$get_catalogs("MODISA")[["MODISA"]]$get_catalogs()
-L3
-# $L3SMI
-# CatalogNode (R6): 
-#   verbose: FALSE    tries: 3    namespace prefix: thredds
-#   url: https://oceandata.sci.gsfc.nasa.gov/opendap/MODISA/L3SMI/catalog.xml
-#   services [3]: OPeNDAP HTTPServer WCS
-#   catalogRefs [22]: 2002 2003 2004 ... 2021 2022 2023
-#   datasets [1]: /MODISA/L3SMI
-L3[[1]]$browse()
-```
-
-Let's drill down into 2009, and see what is available on January 20.
-
-```
-catalog2009 <- L3[[1]]$get_catalogs("2009")
-# $`2009`
-# CatalogNode (R6): 
-#   verbose: FALSE    tries: 3    namespace prefix: thredds
-#   url: https://oceandata.sci.gsfc.nasa.gov/opendap/MODISA/L3SMI/2009/catalog.xml
-#   services [3]: OPeNDAP HTTPServer WCS
-#   catalogRefs [365]: 0101 0102 0103 ... 1229 1230 1231
-#   datasets [1]: /MODISA/L3SMI/2009
-```
-
-Hmmm. We have to conver '2009-01-20' to a three digit day of year (or 4 digit mmdd if looking for SST).
-
-```
-doy <- format(as.Date("2009-01-20"), "%m%d")
-doy
-# "0120"
-```
-
-Ehem, I suppose I could have thought of that without help.  
-
-```
-catalog20 <- catalog2009[['2009']]$get_catalogs(doy)
-# $`0120`
+```R
+cat_name = "godas"
+subcat <- Top$get_catalogs(cat_name)[[cat_name]]
+dev <- subcat$get_catalogs("Derived")[["Derived"]]
 # CatalogNode (R6): 
-#   verbose: FALSE    tries: 3    namespace prefix: thredds
-#   url: https://oceandata.sci.gsfc.nasa.gov/opendap/MODISA/L3SMI/2009/0120/catalog.xml
-#   services [3]: OPeNDAP HTTPServer WCS
+#   verbose: FALSE    tries: 3    namespace prefix: d1
+#   url: https://psl.noaa.gov/thredds/catalog/Datasets/godas/Derived/catalog.xml
+#   services [4]: Compound OpenDAP HTTPServer NetcdfSubset
 #   catalogRefs [0]: none
-#   datasets [100]: AQUA_MODIS.20090120.L3m.DAY.CHL.chlor_a.4km.nc AQUA_MODIS.20090120.L3m.DAY.CHL.chlor_a.9km.nc ... AQUA_MODIS.20090120.L3m.DAY.SST4.sst4.4km.nc AQUA_MODIS.20090120.L3m.DAY.SST4.sst4.9km.nc
+#   datasets [25]: dbss_obil.mon.ltm.1991-2020.nc dbss_obil.mon.ltm.nc ... vflx.mon.ltm.1991-2020.nc vflx.mon.ltm.nc
+
+dev$browse()
 ```
 
-Let's did out just the 9km chlor_a data for that day.
+Let's fetch a dataset and get its NC file.
 
-```
-chl <- catalog20[[doy]]$get_datasets("AQUA_MODIS.20090120.L3m.DAY.CHL.chlor_a.4km.nc")
-# $AQUA_MODIS.20090120.L3m.DAY.CHL.chlor_a.4km.nc
+```R
+ds = dev$get_datasets("dbss_obil.mon.ltm.nc")[[1]]
 # DatasetNode (R6): 
-#   verbose: FALSE    tries: 3    namespace prefix: thredds
-#   url: /MODISA/L3SMI/2009/0120/AQUA_MODIS.20090120.L3m.DAY.CHL.chlor_a.4km.nc
-#   name: AQUA_MODIS.20090120.L3m.DAY.CHL.chlor_a.4km.nc
-#   dataSize: 14194791
-#   date: 2022-07-25T16:41:08Z
+#   verbose: FALSE    tries: 3    namespace prefix: d1
+#   url: Datasets/godas/Derived/dbss_obil.mon.ltm.nc
+#   name: dbss_obil.mon.ltm.nc
+#   dataSize: 7.549
+#   date: 2022-01-28T20:33:42.909Z
 ```
 
 Now we need only retrieve the relative URL, and add it to the base URL for the service.
-Somewhat awkwardly, the relaive URL comes prepended with a path separator, so we 
-use straight up `paste0` to append to the base_uri.
+Somewhat awkwardly, the relative URL comes prepended with a path separator, so we 
+use straight up `paste0` to append to the `base_uri`.
 
-```
-base_uri <- "https://oceandata.sci.gsfc.nasa.gov:443/opendap"
-uri <- paste0(base_uri, chl[["AQUA_MODIS.20090120.L3m.DAY.CHL.chlor_a.4km.nc"]]$url)
+```R
+base_uri = paste0("https://", gsub("^(https?://)?([^/]+).*", "\\2", dev$url), dev$list_services()$odap[["base"]])
+uri <- paste0(base_uri, ds$url)
 NC <- ncdf4::nc_open(uri)
 ```
 
@@ -152,7 +122,7 @@ for searching the thredds genealogy - in this case there isn't any so the defaul
 Even though it is the default, we specify it explicitly for clarity. Also, note that this catalog 
 hase changed over time, so the example may be out of date.
 
-```
+```R
 library(ncdf4)
 library(thredds)
 uri = "https://opendap.co-ops.nos.noaa.gov/thredds/catalog/NOAA/GOMOFS/MODELS/catalog.xml"
@@ -178,87 +148,82 @@ returned, even if just one is requested. Also, note that the `"name` attribute i
 empty string.  In lieu of `name` we then take the first non-empty instance of 
 `title`, `ID`, `urlPath`, and finally `href`.
 
-```
+```R
 top$get_catalog_names()
-# "2020""
+# "2026""
 
-cata = top$get_catalogs(index = "2020")
+cata = top$get_catalogs(index = "2026")
 cata
-# $`2020`
+# $`2026`
 # CatalogNode (R6): 
 #   verbose: FALSE    tries: 3    namespace prefix: d1
-#   url: https://opendap.co-ops.nos.noaa.gov/thredds/catalog/NOAA/GOMOFS/MODELS/2020/catalog.xml
-#   children: service dataset
-#   services [4]: Compound OPENDAP HTTPServer WMS
-#   catalogRefs [3]: 09 08 07
-#   datasets [0]: none
+#   url: https://opendap.co-ops.nos.noaa.gov/thredds/catalog/NOAA/GOMOFS/MODELS/2026/catalog.xml
+#   services [4]: compound OpenDAP HTTPServer WMS
+#   catalogRefs [2]: 09 08
+#   datasets [1]: 2026/
 
 ```
 
 Note that this is a Catalog - a pointer to other catalogs and/or datasets. It looks like 
-Jcatalogs for July, Aug and Sep of 2020. Let's get September.
+there are catalogs for few months of 2026. Let's get September.
 
 
-```
-Months <- cata[["2020"]]$get_catalogs("09")
+```R
+Months <- cata[["2026"]]$get_catalogs("09")
 Months
 # $`09`
 # CatalogNode (R6): 
 #   verbose: FALSE    tries: 3    namespace prefix: d1
-#   url: https://opendap.co-ops.nos.noaa.gov/thredds/catalog/NOAA/GOMOFS/MODELS/2020/09/catalog.xml
-#   children: service dataset
-#   services [4]: Compound OPENDAP HTTPServer WMS
-#   catalogRefs [28]: 28 27 26 ... 03 02 01
-#   datasets [0]: none
+#   url: https://opendap.co-ops.nos.noaa.gov/thredds/catalog/NOAA/GOMOFS/MODELS/2026/09/catalog.xml
+#   services [4]: compound OpenDAP HTTPServer WMS
+#   catalogRefs [4]: 04 03 02 01
+#   datasets [1]: 2026/09/
 ```
 
-So, they appear to be listed by day.  So, let's get the most recent...
+Let's take one day of data:
 
-```
-Recent = Months[["09"]]$get_catalogs("28")
+```R
+Recent = Months[["09"]]$get_catalogs("04")
 Recent
-# $`28`
+$`04`
 # CatalogNode (R6): 
 #   verbose: FALSE    tries: 3    namespace prefix: d1
-#   url: https://opendap.co-ops.nos.noaa.gov/thredds/catalog/NOAA/GOMOFS/MODELS/2020/09/28/catalog.xml
-#   children: service dataset
-#   services [4]: Compound OPENDAP HTTPServer WMS
+#   url: https://opendap.co-ops.nos.noaa.gov/thredds/catalog/NOAA/GOMOFS/MODELS/2026/09/04/catalog.xml
+#   services [4]: compound OpenDAP HTTPServer WMS
 #   catalogRefs [0]: none
-#   datasets [396]: nos.gomofs.stations.nowcast.20200928.t12z.nc nos.gomofs.stations.nowcast.20200928.t06z.nc ... # nos.gomofs.2ds.f001.20200928.t06z.nc nos.gomofs.2ds.f001.20200928.t00z.nc
+#   datasets [396]: gomofs.t12z.20260904.stations.nowcast.nc gomofs.t12z.20260904.stations.forecast.nc ... gomofs.t00z.20260904.2ds.f002.nc gomofs.t00z.20260904.2ds.f001.nc
 ```
 
 Note that we are down to a level without any further catalogs, but instead we have 396 datasets.
 Datasets hold the relative file specification for the resource it identifies.  Let's retrieve the
 dataset for the second item listed.
 
-```
-nowcast <- Recent[['28']]$get_datasets('nos.gomofs.stations.nowcast.20200928.t06z.nc')
+```R
+nowcast <- Recent[['04']]$get_datasets('gomofs.t12z.20260904.stations.nowcast.nc')
 nowcast
-  $nos.gomofs.stations.nowcast.20200928.t06z.nc
-  DatasetNode (R6): 
-    verbose: FALSE    tries: 3    namespace prefix: d1
-    url: NOAA/GOMOFS/MODELS/2020/09/28/nos.gomofs.stations.nowcast.20200928.t06z.nc
-    children: dataSize date
-    name: nos.gomofs.stations.nowcast.20200928.t06z.nc
-    dataSize: 4.871
-    date: 2020-09-28T07:25:47Z
+# $gomofs.t12z.20260904.stations.nowcast.nc
+# DatasetNode (R6): 
+#   verbose: FALSE    tries: 3    namespace prefix: d1
+#   url: NOAA/GOMOFS/MODELS/2026/09/04/gomofs.t12z.20260904.stations.nowcast.nc
+#   name: gomofs.t12z.20260904.stations.nowcast.nc
+#   dataSize: 4.87
+#   date: 2026-09-04T13:10:01Z
 ```
 
 If we know the URL for the base service, then we append the relative URL to that.
 
-```
-base_uri <- 'https://opendap.co-ops.nos.noaa.gov/thredds/dodsC'
-nowcast_uri <- file.path(base_uri, nowcast[['nos.gomofs.stations.nowcast.20200928.t06z.nc']]$url)
-NC <- ncdf4::nc_open(nowcast_uri)
+```R
+base_uri = paste0("https://", gsub("^(https?://)?([^/]+).*", "\\2", Recent[['04']]$url), Recent[['04']]$list_services()$dapService[["base"]])
+uri <- paste0(base_uri, nowcast[[1]]$url)
+NC <- ncdf4::nc_open(uri)
 ```
 
 
 ### Note on searching within a prefixed namespace
 
-A given implementation of a THREDDS catalog system may rely upon an [XML namespace](https://en.wikipedia.org/wiki/XML_namespace) with a prefix. We have encountered these: [d1](https://opendap.co-ops.nos.noaa.gov/thredds/catalog/NOAA/GOMOFS/MODELS/catalog.xml) and [thredds](https://oceandata.sci.gsfc.nasa.gov/opendap/catalog.xml).
+A given implementation of a THREDDS catalog system may rely upon an [XML namespace](https://en.wikipedia.org/wiki/XML_namespace) with a prefix.
 
-
-```
+```R
 uri = "https://opendap.co-ops.nos.noaa.gov/thredds/catalog/NOAA/GOMOFS/MODELS/catalog.xml"
 thredds::get_xml_ns(uri)
 # d1    <-> http://www.unidata.ucar.edu/namespaces/thredds/InvCatalog/v1.0