diff --git a/.github/workflows/update-gbif-catalogs.yml b/.github/workflows/update-gbif-catalogs.yml
new file mode 100644
index 00000000000..a9d8583df8d
--- /dev/null
+++ b/.github/workflows/update-gbif-catalogs.yml
@@ -0,0 +1,65 @@
+name: Update GBIF catalogs
+
+on:
+ workflow_dispatch:
+ schedule:
+ - cron: '0 6 1 * *'
+
+permissions:
+ contents: write
+ pull-requests: write
+
+concurrency:
+ group: update-gbif-catalogs
+ cancel-in-progress: true
+
+jobs:
+ update:
+ name: Generate GBIF catalogs
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+
+ - name: Set up Node.js
+ uses: actions/setup-node@v4
+ with:
+ node-version-file: specifyweb/frontend/js_src/package.json
+ cache: npm
+ cache-dependency-path: specifyweb/frontend/js_src/package-lock.json
+
+ - name: Install frontend dependencies
+ working-directory: specifyweb/frontend/js_src
+ run: npm ci
+
+ - name: Generate GBIF catalogs
+ working-directory: specifyweb/frontend/js_src
+ run: npm run dwca:catalog
+
+ - name: Check for catalog changes
+ id: changes
+ run: |
+ if git diff --quiet -- \
+ specifyweb/frontend/js_src/lib/components/DwcaDefinition/data/gbifCores.json \
+ specifyweb/frontend/js_src/lib/components/DwcaDefinition/data/gbifExtensions.json; then
+ echo 'changed=false' >> "$GITHUB_OUTPUT"
+ else
+ echo 'changed=true' >> "$GITHUB_OUTPUT"
+ fi
+
+ - name: Open update pull request if catalogs changed
+ if: steps.changes.outputs.changed == 'true'
+ uses: peter-evans/create-pull-request@v7
+ with:
+ commit-message: 'chore(dwca): update GBIF catalogs'
+ title: 'Update GBIF catalogs'
+ body: |
+ Updates the generated GBIF core and extension catalogs from the GBIF registry.
+
+ This pull request was created automatically by the monthly catalog refresh.
+ branch: automation/update-gbif-catalogs
+ delete-branch: true
+ add-paths: |
+ specifyweb/frontend/js_src/lib/components/DwcaDefinition/data/gbifCores.json
+ specifyweb/frontend/js_src/lib/components/DwcaDefinition/data/gbifExtensions.json
diff --git a/specifyweb/backend/export/dwca.py b/specifyweb/backend/export/dwca.py
index 1ad6722e9ea..b9d6e54db7e 100644
--- a/specifyweb/backend/export/dwca.py
+++ b/specifyweb/backend/export/dwca.py
@@ -15,6 +15,7 @@
from specifyweb.backend.stored_queries.execution import query_to_csv
from specifyweb.backend.stored_queries.queryfield import QueryField, EphemeralField
from specifyweb.backend.stored_queries.models import session_context
+from specifyweb.specify.datamodel import datamodel
logger = logging.getLogger(__name__)
ET.register_namespace('eml', 'eml://ecoinformatics.org/eml-2.1.1')
@@ -22,6 +23,28 @@
class DwCAException(Exception):
pass
+OCCURRENCE_CORE_ROW_TYPE = 'http://rs.tdwg.org/dwc/terms/Occurrence'
+OCCURRENCE_IDENTIFIER_TERM = 'http://rs.tdwg.org/dwc/terms/occurrenceID'
+CORE_IDENTIFIER_TERMS = {
+ 'http://rs.tdwg.org/dwc/terms/Event': 'http://rs.tdwg.org/dwc/terms/eventID',
+ 'http://rs.tdwg.org/dwc/terms/Taxon': 'http://rs.tdwg.org/dwc/terms/taxonID',
+}
+CORE_TABLE_NAMES = {
+ 'http://rs.tdwg.org/dwc/terms/Event': 'collectingevent',
+ 'http://rs.tdwg.org/dwc/terms/Taxon': 'taxon',
+}
+
+
+def get_core_identifier_term(row_type):
+ return CORE_IDENTIFIER_TERMS.get(row_type, OCCURRENCE_IDENTIFIER_TERM)
+
+
+def get_core_table(row_type):
+ return datamodel.get_table_strict(
+ CORE_TABLE_NAMES.get(row_type, 'collectionobject')
+ )
+
+
# from https://stackoverflow.com/a/17402424
def prettify(elem):
"""Return a pretty-printed XML string for the Element.
@@ -34,7 +57,12 @@ class Stanza(namedtuple('Stanza', 'is_core row_type constant_fields export_field
"Represents either a core or extension definition."
@classmethod
def from_xml(cls, node):
- queries = [Query.from_xml(query_node) for query_node in node.find('queries')]
+ if 'rowType' not in node.attrib:
+ raise DwCAException(_("Definition is missing a row type."))
+ queries_node = node.find('queries')
+ if queries_node is None:
+ raise DwCAException(_("Definition doesn't include any queries."))
+ queries = [Query.from_xml(query_node) for query_node in queries_node]
export_fields, id_field_idx = cls.get_export_fields(queries)
constant_fields = [ConstantField.from_xml(fn) for fn in node.findall('field')]
@@ -113,16 +141,21 @@ class Query(namedtuple('Query', 'tableid file_name query_fields')):
"""
@classmethod
def from_xml(cls, query_node):
- return cls(
- tableid = int(query_node.attrib['contextTableId']),
+ if 'contextTableId' not in query_node.attrib or 'name' not in query_node.attrib:
+ raise DwCAException(_("Query is missing its table or file name."))
+ try:
+ return cls(
+ tableid = int(query_node.attrib['contextTableId']),
- file_name = query_node.attrib['name'],
+ file_name = query_node.attrib['name'],
- query_fields = [
- QueryDefField.from_xml(field_node)
- for field_node in query_node
- ],
- )
+ query_fields = [
+ QueryDefField.from_xml(field_node)
+ for field_node in query_node
+ ],
+ )
+ except ValueError as error:
+ raise DwCAException(_("Query contains invalid attributes.")) from error
def get_export_fields(self):
return tuple(
@@ -182,10 +215,7 @@ def from_xml(cls, node):
def make_dwca(collection, user, definition, output_file, eml=None):
output_dir = mkdtemp()
try:
- element_tree = ET.fromstring(definition)
-
- core_stanza = Stanza.from_xml(element_tree.find('core'))
- extension_stanzas = [Stanza.from_xml(node) for node in element_tree.findall('extension')]
+ core_stanza, extension_stanzas = validate_definition(definition)
output_node = ET.Element('archive')
output_node.set('xmlns', "http://rs.tdwg.org/dwc/text/")
@@ -206,7 +236,13 @@ def make_dwca(collection, user, definition, output_file, eml=None):
core_ids = set()
def collect_ids(row):
- core_ids.add(row[core_stanza.id_field_idx + 1])
+ core_id = row[core_stanza.id_field_idx + 1]
+ if core_id in core_ids:
+ raise DwCAException(_(
+ "The core query returned duplicate occurrenceID values. "
+ "Each core row must have a unique occurrenceID."
+ ))
+ core_ids.add(core_id)
return True
with session_context() as session:
@@ -229,6 +265,59 @@ def filter_ids(row):
finally:
shutil.rmtree(output_dir)
+
+def validate_stanzas(core_stanza, extension_stanzas):
+ if core_stanza is None or not core_stanza.is_core:
+ raise DwCAException(_("Definition must include exactly one core."))
+
+ core_row_type = getattr(core_stanza, 'row_type', OCCURRENCE_CORE_ROW_TYPE)
+ core_identifier_term = get_core_identifier_term(core_row_type)
+ core_id = core_stanza.export_fields[core_stanza.id_field_idx]
+ if core_id.term != core_identifier_term:
+ raise DwCAException(_(
+ "The core identifier must use the identifier term for its core row type."
+ ))
+ if any(field.term is None for field in core_stanza.export_fields):
+ raise DwCAException(_("Every displayed core field must have a term."))
+ for stanza in extension_stanzas:
+ extension_id = stanza.export_fields[stanza.id_field_idx]
+ if extension_id.term != core_identifier_term:
+ raise DwCAException(_(
+ "Every extension identifier must use the core identifier term."
+ ))
+ if any(field.term is None for field in stanza.export_fields):
+ raise DwCAException(_("Every displayed extension field must have a term."))
+
+
+def validate_definition(definition):
+ try:
+ element_tree = ET.fromstring(definition)
+ except (ET.ParseError, TypeError) as error:
+ raise DwCAException(_("Definition is not valid XML.")) from error
+
+ cores = element_tree.findall('core')
+ if len(cores) != 1:
+ raise DwCAException(_("Definition must include exactly one core."))
+ try:
+ core_stanza = Stanza.from_xml(cores[0])
+ core_table_id = get_core_table(core_stanza.row_type).tableId
+ extension_stanzas = [
+ Stanza.from_xml(node) for node in element_tree.findall('extension')
+ ]
+ except ValueError as error:
+ raise DwCAException(_("Definition contains invalid query attributes.")) from error
+ if any(
+ query.tableid != core_table_id
+ for stanza in [core_stanza, *extension_stanzas]
+ for query in stanza.queries
+ ):
+ raise DwCAException(_(
+ "All Darwin Core queries must use the base table associated with the core "
+ "row type."
+ ))
+ validate_stanzas(core_stanza, extension_stanzas)
+ return core_stanza, extension_stanzas
+
def write_eml(source, output_path, pub_date=None, package_id=None):
if pub_date is None:
pub_date = date.today()
diff --git a/specifyweb/backend/export/tests.py b/specifyweb/backend/export/tests.py
index 501deb776c1..7304a231c98 100644
--- a/specifyweb/backend/export/tests.py
+++ b/specifyweb/backend/export/tests.py
@@ -5,8 +5,12 @@
Replace this with more appropriate tests for your application.
"""
+from collections import namedtuple
from django.test import TestCase
+from specifyweb.specify.datamodel import datamodel
+
+from .dwca import DwCAException, ExportField, validate_definition, validate_stanzas
class SimpleTest(TestCase):
def test_basic_addition(self):
@@ -14,3 +18,50 @@ def test_basic_addition(self):
Tests that 1 + 1 always equals 2.
"""
self.assertEqual(1 + 1, 2)
+
+class DwcaValidationTest(TestCase):
+ @staticmethod
+ def stanza(is_core, fields, index=0):
+ return namedtuple('TestStanza', 'is_core export_fields id_field_idx')(
+ is_core, fields, index
+ )
+
+ def test_requires_matching_extension_identifier(self):
+ core = self.stanza(
+ True,
+ [ExportField(0, 'http://rs.tdwg.org/dwc/terms/occurrenceID', True)],
+ )
+ extension = self.stanza(False, [ExportField(0, 'eventID', True)])
+ with self.assertRaises(DwCAException):
+ validate_stanzas(core, [extension])
+
+ def test_allows_multiple_extensions(self):
+ occurrence_id = 'http://rs.tdwg.org/dwc/terms/occurrenceID'
+ core = self.stanza(True, [ExportField(0, occurrence_id, True)])
+ extension = self.stanza(False, [ExportField(0, occurrence_id, True)])
+ validate_stanzas(core, [extension, extension])
+
+ def test_uses_core_row_type_base_table(self):
+ collecting_event_id = datamodel.get_table_strict('collectingevent').tableId
+ collection_object_id = datamodel.get_table_strict('collectionobject').tableId
+ definition = f'''
+