feat(dwca): add visual editor

.github/workflows/update-gbif-catalogs.yml

@@ -0,0 +1,65 @@
+name: Update GBIF catalogs
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 6 1 * *'
+
+permissions:
+  contents: write
+  pull-requests: write
+
+concurrency:
+  group: update-gbif-catalogs
+  cancel-in-progress: true
+
+jobs:
+  update:
+    name: Generate GBIF catalogs
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version-file: specifyweb/frontend/js_src/package.json
+          cache: npm
+          cache-dependency-path: specifyweb/frontend/js_src/package-lock.json
+
+      - name: Install frontend dependencies
+        working-directory: specifyweb/frontend/js_src
+        run: npm ci
+
+      - name: Generate GBIF catalogs
+        working-directory: specifyweb/frontend/js_src
+        run: npm run dwca:catalog
+
+      - name: Check for catalog changes
+        id: changes
+        run: |
+          if git diff --quiet -- \
+            specifyweb/frontend/js_src/lib/components/DwcaDefinition/data/gbifCores.json \
+            specifyweb/frontend/js_src/lib/components/DwcaDefinition/data/gbifExtensions.json; then
+            echo 'changed=false' >> "$GITHUB_OUTPUT"
+          else
+            echo 'changed=true' >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Open update pull request if catalogs changed
+        if: steps.changes.outputs.changed == 'true'
+        uses: peter-evans/create-pull-request@v7
+        with:
+          commit-message: 'chore(dwca): update GBIF catalogs'
+          title: 'Update GBIF catalogs'
+          body: |
+            Updates the generated GBIF core and extension catalogs from the GBIF registry.
+
+            This pull request was created automatically by the monthly catalog refresh.
+          branch: automation/update-gbif-catalogs
+          delete-branch: true
+          add-paths: |
+            specifyweb/frontend/js_src/lib/components/DwcaDefinition/data/gbifCores.json
+            specifyweb/frontend/js_src/lib/components/DwcaDefinition/data/gbifExtensions.json

specifyweb/backend/export/dwca.py

@@ -15,13 +15,36 @@
 from specifyweb.backend.stored_queries.execution import query_to_csv
 from specifyweb.backend.stored_queries.queryfield import QueryField, EphemeralField
 from specifyweb.backend.stored_queries.models import session_context
+from specifyweb.specify.datamodel import datamodel
 
 logger = logging.getLogger(__name__)
 ET.register_namespace('eml', 'eml://ecoinformatics.org/eml-2.1.1')
 
 class DwCAException(Exception):
     pass
 
+OCCURRENCE_CORE_ROW_TYPE = 'http://rs.tdwg.org/dwc/terms/Occurrence'
+OCCURRENCE_IDENTIFIER_TERM = 'http://rs.tdwg.org/dwc/terms/occurrenceID'
+CORE_IDENTIFIER_TERMS = {
+    'http://rs.tdwg.org/dwc/terms/Event': 'http://rs.tdwg.org/dwc/terms/eventID',
+    'http://rs.tdwg.org/dwc/terms/Taxon': 'http://rs.tdwg.org/dwc/terms/taxonID',
+}
+CORE_TABLE_NAMES = {
+    'http://rs.tdwg.org/dwc/terms/Event': 'collectingevent',
+    'http://rs.tdwg.org/dwc/terms/Taxon': 'taxon',
+}
+
+
+def get_core_identifier_term(row_type):
+    return CORE_IDENTIFIER_TERMS.get(row_type, OCCURRENCE_IDENTIFIER_TERM)
+
+
+def get_core_table(row_type):
+    return datamodel.get_table_strict(
+        CORE_TABLE_NAMES.get(row_type, 'collectionobject')
+    )
+
+
 # from https://stackoverflow.com/a/17402424
 def prettify(elem):
     """Return a pretty-printed XML string for the Element.
@@ -34,7 +57,12 @@ class Stanza(namedtuple('Stanza', 'is_core row_type constant_fields export_field
     "Represents either a core or extension definition."
     @classmethod
     def from_xml(cls, node):
-        queries = [Query.from_xml(query_node) for query_node in node.find('queries')]
+        if 'rowType' not in node.attrib:
+            raise DwCAException(_("Definition is missing a row type."))
+        queries_node = node.find('queries')
+        if queries_node is None:
+            raise DwCAException(_("Definition doesn't include any queries."))
+        queries = [Query.from_xml(query_node) for query_node in queries_node]
         export_fields, id_field_idx = cls.get_export_fields(queries)
         constant_fields = [ConstantField.from_xml(fn) for fn in node.findall('field')]
 
@@ -113,16 +141,21 @@ class Query(namedtuple('Query', 'tableid file_name query_fields')):
     """
     @classmethod
     def from_xml(cls, query_node):
-        return cls(
-            tableid = int(query_node.attrib['contextTableId']),
+        if 'contextTableId' not in query_node.attrib or 'name' not in query_node.attrib:
+            raise DwCAException(_("Query is missing its table or file name."))
+        try:
+            return cls(
+                tableid = int(query_node.attrib['contextTableId']),
 
-            file_name = query_node.attrib['name'],
+                file_name = query_node.attrib['name'],
 
-            query_fields = [
-                QueryDefField.from_xml(field_node)
-                for field_node in query_node
-            ],
-        )
+                query_fields = [
+                    QueryDefField.from_xml(field_node)
+                    for field_node in query_node
+                ],
+            )
+        except ValueError as error:
+            raise DwCAException(_("Query contains invalid attributes.")) from error
 
     def get_export_fields(self):
         return tuple(
@@ -182,10 +215,7 @@ def from_xml(cls, node):
 def make_dwca(collection, user, definition, output_file, eml=None):
     output_dir = mkdtemp()
     try:
-        element_tree = ET.fromstring(definition)
-
-        core_stanza = Stanza.from_xml(element_tree.find('core'))
-        extension_stanzas = [Stanza.from_xml(node) for node in element_tree.findall('extension')]
+        core_stanza, extension_stanzas = validate_definition(definition)
 
         output_node = ET.Element('archive')
         output_node.set('xmlns', "http://rs.tdwg.org/dwc/text/")
@@ -206,7 +236,13 @@ def make_dwca(collection, user, definition, output_file, eml=None):
 
         core_ids = set()
         def collect_ids(row):
-            core_ids.add(row[core_stanza.id_field_idx + 1])
+            core_id = row[core_stanza.id_field_idx + 1]
+            if core_id in core_ids:
+                raise DwCAException(_(
+                    "The core query returned duplicate occurrenceID values. "
+                    "Each core row must have a unique occurrenceID."
+                ))
+            core_ids.add(core_id)
             return True
 
         with session_context() as session:
@@ -229,6 +265,59 @@ def filter_ids(row):
     finally:
         shutil.rmtree(output_dir)
 
+
+def validate_stanzas(core_stanza, extension_stanzas):
+    if core_stanza is None or not core_stanza.is_core:
+        raise DwCAException(_("Definition must include exactly one core."))
+
+    core_row_type = getattr(core_stanza, 'row_type', OCCURRENCE_CORE_ROW_TYPE)
+    core_identifier_term = get_core_identifier_term(core_row_type)
+    core_id = core_stanza.export_fields[core_stanza.id_field_idx]
+    if core_id.term != core_identifier_term:
+        raise DwCAException(_(
+            "The core identifier must use the identifier term for its core row type."
+        ))
+    if any(field.term is None for field in core_stanza.export_fields):
+        raise DwCAException(_("Every displayed core field must have a term."))
+    for stanza in extension_stanzas:
+        extension_id = stanza.export_fields[stanza.id_field_idx]
+        if extension_id.term != core_identifier_term:
+            raise DwCAException(_(
+                "Every extension identifier must use the core identifier term."
+            ))
+        if any(field.term is None for field in stanza.export_fields):
+            raise DwCAException(_("Every displayed extension field must have a term."))
+
+
+def validate_definition(definition):
+    try:
+        element_tree = ET.fromstring(definition)
+    except (ET.ParseError, TypeError) as error:
+        raise DwCAException(_("Definition is not valid XML.")) from error
+
+    cores = element_tree.findall('core')
+    if len(cores) != 1:
+        raise DwCAException(_("Definition must include exactly one core."))
+    try:
+        core_stanza = Stanza.from_xml(cores[0])
+        core_table_id = get_core_table(core_stanza.row_type).tableId
+        extension_stanzas = [
+            Stanza.from_xml(node) for node in element_tree.findall('extension')
+        ]
+    except ValueError as error:
+        raise DwCAException(_("Definition contains invalid query attributes.")) from error
+    if any(
+        query.tableid != core_table_id
+        for stanza in [core_stanza, *extension_stanzas]
+        for query in stanza.queries
+    ):
+        raise DwCAException(_(
+            "All Darwin Core queries must use the base table associated with the core "
+            "row type."
+        ))
+    validate_stanzas(core_stanza, extension_stanzas)
+    return core_stanza, extension_stanzas
+
 def write_eml(source, output_path, pub_date=None, package_id=None):
     if pub_date is None:
         pub_date = date.today()

specifyweb/backend/export/tests.py

@@ -5,12 +5,63 @@
 Replace this with more appropriate tests for your application.
 """
 
+from collections import namedtuple
 from django.test import TestCase
 
+from specifyweb.specify.datamodel import datamodel
+
+from .dwca import DwCAException, ExportField, validate_definition, validate_stanzas
 
 class SimpleTest(TestCase):
     def test_basic_addition(self):
         """
         Tests that 1 + 1 always equals 2.
         """
         self.assertEqual(1 + 1, 2)
+
+class DwcaValidationTest(TestCase):
+    @staticmethod
+    def stanza(is_core, fields, index=0):
+        return namedtuple('TestStanza', 'is_core export_fields id_field_idx')(
+            is_core, fields, index
+        )
+
+    def test_requires_matching_extension_identifier(self):
+        core = self.stanza(
+            True,
+            [ExportField(0, 'http://rs.tdwg.org/dwc/terms/occurrenceID', True)],
+        )
+        extension = self.stanza(False, [ExportField(0, 'eventID', True)])
+        with self.assertRaises(DwCAException):
+            validate_stanzas(core, [extension])
+
+    def test_allows_multiple_extensions(self):
+        occurrence_id = 'http://rs.tdwg.org/dwc/terms/occurrenceID'
+        core = self.stanza(True, [ExportField(0, occurrence_id, True)])
+        extension = self.stanza(False, [ExportField(0, occurrence_id, True)])
+        validate_stanzas(core, [extension, extension])
+
+    def test_uses_core_row_type_base_table(self):
+        collecting_event_id = datamodel.get_table_strict('collectingevent').tableId
+        collection_object_id = datamodel.get_table_strict('collectionobject').tableId
+        definition = f'''
+            <archive>
+              <core rowType="http://rs.tdwg.org/dwc/terms/Event">
+                <queries>
+                  <query contextTableId="{collecting_event_id}" name="event.csv">
+                    <id stringId="1.collectingevent.guid" isRelFld="false"
+                        oper="11" value="" isNot="false"
+                        term="http://rs.tdwg.org/dwc/terms/eventID" />
+                  </query>
+                </queries>
+              </core>
+            </archive>
+        '''
+        validate_definition(definition)
+
+        invalid_definition = definition.replace(
+            f'contextTableId="{collecting_event_id}"',
+            f'contextTableId="{collection_object_id}"',
+        )
+        with self.assertRaises(DwCAException):
+            validate_definition(invalid_definition)

specifyweb/backend/export/views.py

@@ -14,7 +14,7 @@
 from django.views.decorators.cache import never_cache
 from django.views.decorators.http import require_POST
 
-from .dwca import make_dwca, prettify
+from .dwca import DwCAException, make_dwca, prettify, validate_definition
 from .extract_query import extract_query as extract
 from .feed import FEED_DIR, get_feed_resource, update_feed
 from specifyweb.backend.context.app_resource import get_app_resource
@@ -118,7 +118,14 @@ def export(request):
 
     eml_resource = request.POST.get('metadata', None)
 
-    definition, _, __ = get_app_resource(collection, user, dwca_resource)
+    resolved_definition = get_app_resource(collection, user, dwca_resource)
+    if resolved_definition is None:
+        return HttpResponseBadRequest('DwCA definition resource was not found')
+    definition, _, __ = resolved_definition
+    try:
+        validate_definition(definition)
+    except DwCAException as error:
+        return HttpResponseBadRequest(str(error))
 
     if eml_resource is not None:
         eml, _, __ = get_app_resource(collection, user, eml_resource)

specifyweb/frontend/js_src/lib/components/AppResources/Editor.tsx

@@ -167,7 +167,9 @@ export function AppResourceEditor({
   const isEditingForm =
     typeof toResource(resource, 'SpViewSetObj') === 'object';
   // When editing a form, don't render the page inside of <form> to avoid #3357
-  const renderInForm = !isEditingForm;
+  const renderInForm =
+    !isEditingForm &&
+    appResource.get('mimeType') !== 'application/vnd.specify.dwca+xml';
 
   const headerButtons = (
     <div className="flex flex-wrap gap-3">

specifyweb/frontend/js_src/lib/components/AppResources/TabDefinitions.tsx

@@ -20,6 +20,7 @@ import type {
   SpViewSetObj,
 } from '../DataModel/types';
 import { RssExportFeedEditor } from '../ExportFeed';
+import { DwcaDefinitionEditor } from '../DwcaDefinition/DwcaDefinition';
 import { exportFeedSpec } from '../ExportFeed/spec';
 import { ExpressSearchConfigResourceEditor } from '../ExpressSearchConfig/ExpressSearchConfigEditor';
 import { FieldFormattersEditor } from '../FieldFormatters/Editor';
@@ -168,6 +169,10 @@ export const visualAppResourceEditors = f.store<
     visual: RssExportFeedEditor,
     xml: generateXmlEditor(exportFeedSpec),
   },
+  dwcaDefinition: {
+    visual: DwcaDefinitionEditor,
+    xml: generateXmlEditor(undefined),
+  },
   expressSearchConfig: {
     visual: ExpressSearchConfigResourceEditor,
     xml: AppResourceTextEditor,

specifyweb/frontend/js_src/lib/components/AppResources/__tests__/AppResourcesFilters.test.tsx

@@ -61,6 +61,7 @@ describe('AppResourcesFilters', () => {
         'dataEntryTables',
         'dataObjectFormatters',
         'defaultUserPreferences',
+        'dwcaDefinition',
         'expressSearchConfig',
         'interactionsTables',
         'label',

specifyweb/frontend/js_src/lib/components/AppResources/__tests__/CreateAppResource.test.tsx

@@ -58,7 +58,7 @@ describe('CreateAppResource', () => {
 
     //  This is a lot more cleaner than the inner HTML
     expect(getByRole('dialog').textContent).toMatchInlineSnapshot(
-      `"Select Resource TypeTypeDocumentationLabelDocumentationReportDocumentationDefault User PreferencesDocumentationLeaflet LayersDocumentationRSS Export FeedDocumentationExpress Search ConfigDocumentationType SearchesDocumentationWeb LinksDocumentationField FormattersDocumentationRecord FormattersDocumentationData Entry TablesDocumentationInteractions TablesDocumentationOther XML ResourceOther JSON ResourceOther Properties ResourceOther ResourceCancel"`
+      `"Select Resource TypeTypeDocumentationLabelDocumentationReportDocumentationDefault User PreferencesDocumentationLeaflet LayersDocumentationRSS Export FeedDocumentationDarwin Core definitionExpress Search ConfigDocumentationType SearchesDocumentationWeb LinksDocumentationField FormattersDocumentationRecord FormattersDocumentationData Entry TablesDocumentationInteractions TablesDocumentationOther XML ResourceOther JSON ResourceOther Properties ResourceOther ResourceCancel"`
     );
   });