diff --git a/docs/translations/releases.adoc.pot b/docs/translations/releases.adoc.pot index 1a598e5e10..a7a6129890 100644 --- a/docs/translations/releases.adoc.pot +++ b/docs/translations/releases.adoc.pot @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" -"POT-Creation-Date: 2024-12-18 13:26+0000\n" +"POT-Creation-Date: 2024-12-18 16:49+0000\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" diff --git a/pom.xml b/pom.xml index 1d468f9a85..65be59813f 100644 --- a/pom.xml +++ b/pom.xml @@ -47,7 +47,7 @@ 1.16.18 0.60 2.21 - 2.6 + 2.7 0.67 1.4 @@ -63,11 +63,13 @@ 4.4.0 2.3.33 0.6.16.1-gbif + 0.6.16.2-gbif 2.11.0 20.0 4.2.3 4.5.14 4.4.16 + 2.15.1 2.14.0 2.14 3.12.1.GA @@ -532,6 +534,11 @@ datapackage-java ${frictionless.datapackage-java.version} + + io.frictionlessdata + tableschema-java + ${frictionless.tableschema-java.version} + @@ -678,6 +685,11 @@ gson ${gson.version} + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + diff --git a/renovate.json b/renovate.json index fd8730dc24..b7bf89c255 100644 --- a/renovate.json +++ b/renovate.json @@ -27,6 +27,10 @@ { "matchPackageNames": ["com.fasterxml.jackson.dataformat:jackson-dataformat-yaml"], "automerge": false + }, + { + "matchPackageNames": ["com.fasterxml.jackson.core:jackson-databind"], + "automerge": false } ], "ignorePaths": ["package/**", "docs/**"], diff --git a/src/main/java/org/gbif/ipt/action/manage/ResourceFileAction.java b/src/main/java/org/gbif/ipt/action/manage/ResourceFileAction.java new file mode 100644 index 0000000000..c027595ba6 --- /dev/null +++ b/src/main/java/org/gbif/ipt/action/manage/ResourceFileAction.java @@ -0,0 +1,135 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gbif.ipt.action.manage; + +import org.gbif.ipt.config.AppConfig; +import org.gbif.ipt.config.Constants; +import org.gbif.ipt.config.DataDir; +import org.gbif.ipt.service.admin.RegistrationManager; +import org.gbif.ipt.service.manage.ResourceManager; +import org.gbif.ipt.struts2.SimpleTextProvider; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.InputStream; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.google.inject.Inject; +import lombok.Getter; + +/** + * The Action responsible for serving datadir resource files. + */ +public class ResourceFileAction extends ManagerBaseAction { + + // logging + private static final Logger LOG = LogManager.getLogger(ResourceFileAction.class); + + private static final long serialVersionUID = -3304799051086050164L; + + private final DataDir dataDir; + @Getter + private InputStream inputStream; + @Getter + protected File data; + @Getter + protected String mimeType = "text/plain"; + @Getter + protected String filename; + + @Inject + public ResourceFileAction(SimpleTextProvider textProvider, AppConfig cfg, RegistrationManager registrationManager, + DataDir dataDir, ResourceManager resourceManager) { + super(textProvider, cfg, registrationManager, resourceManager); + this.dataDir = dataDir; + } + + /** + * Handles metadata file download request. + * + * @return Struts2 result string + */ + public String metadata() { + if (resource == null) { + return NOT_FOUND; + } + + boolean isDataPackageResource = resource.getDataPackageIdentifier() != null; + // construct download filename + StringBuilder sb = new StringBuilder(); + + // serve file + if (isDataPackageResource) { + if (Constants.COL_DP.equals(resource.getCoreType())) { + data = dataDir.resourceDatapackageMetadataFile(resource.getShortname(), resource.getCoreType()); + mimeType = "text/yaml"; + sb.append("metadata-").append(resource.getShortname()); + sb.append(".yaml"); + } else { + data = dataDir.resourceDatapackageMetadataFile(resource.getShortname(), resource.getCoreType()); + mimeType = "application/json"; + sb.append("datapackage-").append(resource.getShortname()); + sb.append(".json"); + } + } else { + data = dataDir.resourceEmlFile(resource.getShortname()); + mimeType = "text/xml"; + sb.append("eml-").append(resource.getShortname()); + sb.append(".xml"); + } + + filename = sb.toString(); + return execute(); + } + + @Override + public String execute() { + // make sure we have a download filename + if (data == null) { + return NOT_FOUND; + } else if (filename == null) { + filename = data.getName(); + } + try { + inputStream = new FileInputStream(data); + // Set a Last-Modified header, even on 304 Not Modified responses. + // Round to the nearest second, as HTTP doesn't support milliseconds. + long lastModified = 1000 * ((data.lastModified() + 500) / 1000); + response.setDateHeader("Last-Modified", lastModified); + + // see if we have a conditional get with If-Modified-Since header + try { + long since = req.getDateHeader("If-Modified-Since"); + if (since >= lastModified) { + return NOT_MODIFIED; + } + } catch (IllegalArgumentException e) { + // headers might not be formed correctly, swallow + LOG.warn("Conditional get with If-Modified-Since header couldn't be interpreted", e); + } + } catch (FileNotFoundException e) { + LOG.warn("Data dir file not found", e); + return NOT_FOUND; + } + return SUCCESS; + } + + @Override + public void prepare() { + super.prepare(); + } +} diff --git a/src/main/java/org/gbif/ipt/task/GenerateDataPackage.java b/src/main/java/org/gbif/ipt/task/GenerateDataPackage.java index 40a35ed026..9819361542 100644 --- a/src/main/java/org/gbif/ipt/task/GenerateDataPackage.java +++ b/src/main/java/org/gbif/ipt/task/GenerateDataPackage.java @@ -806,6 +806,7 @@ private void addMetadata() throws GeneratorException, InterruptedException { } } catch (Exception e) { + addMessage(Level.ERROR, e.getMessage()); throw new GeneratorException("Problem occurred while adding metadata file to data package folder", e); } // final reporting diff --git a/src/main/java/org/gbif/ipt/validation/EmlValidator.java b/src/main/java/org/gbif/ipt/validation/EmlValidator.java index 866a61f07b..57eb91daf9 100644 --- a/src/main/java/org/gbif/ipt/validation/EmlValidator.java +++ b/src/main/java/org/gbif/ipt/validation/EmlValidator.java @@ -235,6 +235,7 @@ public void validate(BaseAction action, Resource resource, @Nullable MetadataSec String strippedDescription = Optional.ofNullable(eml.getDescription()) .map(d -> d.replaceAll("<[^>]*>", "")) // get rid of tags + .map(d -> d.replace(" ", " ")) // replace   with a space .map(String::trim) .orElse(""); @@ -251,7 +252,12 @@ public void validate(BaseAction action, Resource resource, @Nullable MetadataSec } else { try { Eml stubValidationEml = getStubEml(); - stubValidationEml.setDescription(eml.getDescription()); + + String descriptionWithNbspReplaced = Optional.ofNullable(eml.getDescription()) + .map(d -> d.replace(" ", " ")) // replace   with a space + .orElse(""); + + stubValidationEml.setDescription(descriptionWithNbspReplaced); String emlString = IptEmlWriter.writeEmlAsString(stubValidationEml); emlProfileValidator.validate(emlString); } catch (InvalidEmlException e) { @@ -541,7 +547,12 @@ public void validate(BaseAction action, Resource resource, @Nullable MetadataSec } else { try { Eml stubValidationEml = getStubEml(); - stubValidationEml.setAcknowledgements(eml.getAcknowledgements()); + + String acknowledgementsWithNbspReplaced = Optional.ofNullable(eml.getAcknowledgements()) + .map(d -> d.replace(" ", " ")) // replace   with a space + .orElse(""); + + stubValidationEml.setAcknowledgements(acknowledgementsWithNbspReplaced); String emlString = IptEmlWriter.writeEmlAsString(stubValidationEml); emlProfileValidator.validate(emlString); } catch (InvalidEmlException e) { @@ -713,7 +724,12 @@ public void validate(BaseAction action, Resource resource, @Nullable MetadataSec } else { try { Eml stubValidationEml = getStubEml(); - stubValidationEml.setGettingStarted(eml.getPurpose()); + + String purposeWithNbspReplaced = Optional.ofNullable(eml.getPurpose()) + .map(d -> d.replace(" ", " ")) // replace   with a space + .orElse(""); + + stubValidationEml.setGettingStarted(purposeWithNbspReplaced); String emlString = IptEmlWriter.writeEmlAsString(stubValidationEml); emlProfileValidator.validate(emlString); } catch (InvalidEmlException e) { @@ -725,7 +741,12 @@ public void validate(BaseAction action, Resource resource, @Nullable MetadataSec try { Eml stubValidationEml = getStubEml(); - stubValidationEml.setGettingStarted(eml.getGettingStarted()); + + String gettingStartedWithNbspReplaced = Optional.ofNullable(eml.getGettingStarted()) + .map(d -> d.replace(" ", " ")) // replace   with a space + .orElse(""); + + stubValidationEml.setGettingStarted(gettingStartedWithNbspReplaced); String emlString = IptEmlWriter.writeEmlAsString(stubValidationEml); emlProfileValidator.validate(emlString); } catch (InvalidEmlException e) { @@ -737,7 +758,12 @@ public void validate(BaseAction action, Resource resource, @Nullable MetadataSec try { Eml stubValidationEml = getStubEml(); - stubValidationEml.setIntroduction(eml.getIntroduction()); + + String introductionWithNbspReplaced = Optional.ofNullable(eml.getIntroduction()) + .map(d -> d.replace(" ", " ")) // replace   with a space + .orElse(""); + + stubValidationEml.setIntroduction(introductionWithNbspReplaced); String emlString = IptEmlWriter.writeEmlAsString(stubValidationEml); emlProfileValidator.validate(emlString); } catch (InvalidEmlException e) { diff --git a/src/main/resources/ApplicationResources_en.properties b/src/main/resources/ApplicationResources_en.properties index 6ade9cf70e..d43a6daec8 100644 --- a/src/main/resources/ApplicationResources_en.properties +++ b/src/main/resources/ApplicationResources_en.properties @@ -552,7 +552,6 @@ portal.resource.other=Other Metadata portal.resource.spatial.coordinates=Coordinates portal.resource.spatial.coordinatePrecision=Coordinate precision portal.resource.taxonomic.taxonId=Taxon id -portal.resource.taxonomic.taxonIdReference=Taxon id reference portal.resource.taxonomic.scientificName=Scientific name portal.resource.taxonomic.taxonRank=Taxon rank portal.resource.taxonomic.vernacularNames=Vernacular names diff --git a/src/main/resources/struts-manage.xml b/src/main/resources/struts-manage.xml index 132fd69ece..7de086cbdf 100644 --- a/src/main/resources/struts-manage.xml +++ b/src/main/resources/struts-manage.xml @@ -187,6 +187,16 @@ ${baseURL}/manage/resource.do?r=${resource.shortname} + + + ${mimeType} + UTF-8 + inputStream + filename="${filename}" + 1024 + + + ${baseURL}/manage/resource.do?r=${resource.shortname} ${baseURL}/manage/resource.do?r=${resource.shortname} diff --git a/src/main/webapp/WEB-INF/pages/manage/dp-metadata/camtrap/basic.ftl b/src/main/webapp/WEB-INF/pages/manage/dp-metadata/camtrap/basic.ftl index ef0e7c1e98..f2d084b7d4 100644 --- a/src/main/webapp/WEB-INF/pages/manage/dp-metadata/camtrap/basic.ftl +++ b/src/main/webapp/WEB-INF/pages/manage/dp-metadata/camtrap/basic.ftl @@ -401,6 +401,8 @@ allowClear: true, theme: 'bootstrap4' }); + + makeSureResourceParameterIsPresentInURL('${resource.shortname}'); }); <#assign currentMenu="manage"/> @@ -413,6 +415,8 @@
+ +
diff --git a/src/main/webapp/WEB-INF/pages/manage/dp-metadata/camtrap/citation.ftl b/src/main/webapp/WEB-INF/pages/manage/dp-metadata/camtrap/citation.ftl index b930896e49..01e270ed94 100644 --- a/src/main/webapp/WEB-INF/pages/manage/dp-metadata/camtrap/citation.ftl +++ b/src/main/webapp/WEB-INF/pages/manage/dp-metadata/camtrap/citation.ftl @@ -38,6 +38,8 @@ $("#generateOn").show(); $("#generateOff").hide(); }); + + makeSureResourceParameterIsPresentInURL('${resource.shortname}'); }); <#assign currentMenu="manage"/> @@ -50,6 +52,8 @@
+ +
diff --git a/src/main/webapp/WEB-INF/pages/manage/dp-metadata/camtrap/geographic.ftl b/src/main/webapp/WEB-INF/pages/manage/dp-metadata/camtrap/geographic.ftl index 7bba978aaf..0733c1a758 100644 --- a/src/main/webapp/WEB-INF/pages/manage/dp-metadata/camtrap/geographic.ftl +++ b/src/main/webapp/WEB-INF/pages/manage/dp-metadata/camtrap/geographic.ftl @@ -75,6 +75,8 @@ }); $("#re-infer-link").on('click', displayProcessing); + + makeSureResourceParameterIsPresentInURL('${resource.shortname}'); }); <#assign currentMenu="manage"/> @@ -87,6 +89,8 @@
+ +
diff --git a/src/main/webapp/WEB-INF/pages/manage/dp-metadata/camtrap/keywords.ftl b/src/main/webapp/WEB-INF/pages/manage/dp-metadata/camtrap/keywords.ftl index 8c94e3c9bc..8190656ba8 100644 --- a/src/main/webapp/WEB-INF/pages/manage/dp-metadata/camtrap/keywords.ftl +++ b/src/main/webapp/WEB-INF/pages/manage/dp-metadata/camtrap/keywords.ftl @@ -73,6 +73,8 @@ // scroll to the element $('body, html').animate({scrollTop: pos}); } + + makeSureResourceParameterIsPresentInURL('${resource.shortname}'); }); <#assign currentMenu="manage"/> @@ -85,6 +87,8 @@
+ +
diff --git a/src/main/webapp/WEB-INF/pages/manage/dp-metadata/camtrap/other.ftl b/src/main/webapp/WEB-INF/pages/manage/dp-metadata/camtrap/other.ftl index 4043ce65a2..8e4483d1d3 100644 --- a/src/main/webapp/WEB-INF/pages/manage/dp-metadata/camtrap/other.ftl +++ b/src/main/webapp/WEB-INF/pages/manage/dp-metadata/camtrap/other.ftl @@ -217,6 +217,8 @@ allowClear: true, theme: 'bootstrap4' }); + + makeSureResourceParameterIsPresentInURL('${resource.shortname}'); }); <#assign currentMenu="manage"/> @@ -229,6 +231,8 @@
+ +
diff --git a/src/main/webapp/WEB-INF/pages/manage/dp-metadata/camtrap/project.ftl b/src/main/webapp/WEB-INF/pages/manage/dp-metadata/camtrap/project.ftl index 9cd4758f09..bb163b3579 100644 --- a/src/main/webapp/WEB-INF/pages/manage/dp-metadata/camtrap/project.ftl +++ b/src/main/webapp/WEB-INF/pages/manage/dp-metadata/camtrap/project.ftl @@ -43,6 +43,8 @@ allowClear: true, theme: 'bootstrap4' }); + + makeSureResourceParameterIsPresentInURL('${resource.shortname}'); }); <#assign currentMenu="manage"/> @@ -56,6 +58,8 @@
+ +
diff --git a/src/main/webapp/WEB-INF/pages/manage/dp-metadata/camtrap/taxonomic.ftl b/src/main/webapp/WEB-INF/pages/manage/dp-metadata/camtrap/taxonomic.ftl index 028dfccc56..148d05255b 100644 --- a/src/main/webapp/WEB-INF/pages/manage/dp-metadata/camtrap/taxonomic.ftl +++ b/src/main/webapp/WEB-INF/pages/manage/dp-metadata/camtrap/taxonomic.ftl @@ -300,6 +300,8 @@ }); $("#re-infer-link").on('click', displayProcessing); + + makeSureResourceParameterIsPresentInURL('${resource.shortname}'); }); <#assign currentMenu="manage"/> @@ -312,6 +314,8 @@
+ +
diff --git a/src/main/webapp/WEB-INF/pages/manage/dp-metadata/camtrap/temporal.ftl b/src/main/webapp/WEB-INF/pages/manage/dp-metadata/camtrap/temporal.ftl index db976f3963..262cba68e2 100644 --- a/src/main/webapp/WEB-INF/pages/manage/dp-metadata/camtrap/temporal.ftl +++ b/src/main/webapp/WEB-INF/pages/manage/dp-metadata/camtrap/temporal.ftl @@ -47,6 +47,8 @@ }); $("#re-infer-link").on('click', displayProcessing); + + makeSureResourceParameterIsPresentInURL('${resource.shortname}'); }); <#assign currentMenu="manage"/> @@ -59,6 +61,8 @@
+ +
diff --git a/src/main/webapp/WEB-INF/pages/manage/dp-metadata/frictionless/basic.ftl b/src/main/webapp/WEB-INF/pages/manage/dp-metadata/frictionless/basic.ftl index 610223bb87..dae7358f66 100644 --- a/src/main/webapp/WEB-INF/pages/manage/dp-metadata/frictionless/basic.ftl +++ b/src/main/webapp/WEB-INF/pages/manage/dp-metadata/frictionless/basic.ftl @@ -229,6 +229,8 @@ // scroll to the element $('body, html').animate({scrollTop: pos}); } + + makeSureResourceParameterIsPresentInURL('${resource.shortname}'); }); <#assign currentMenu="manage"/> diff --git a/src/main/webapp/WEB-INF/pages/manage/eml/acknowledgements.ftl b/src/main/webapp/WEB-INF/pages/manage/eml/acknowledgements.ftl index b3ea3ec8e9..7b6f3e31e2 100644 --- a/src/main/webapp/WEB-INF/pages/manage/eml/acknowledgements.ftl +++ b/src/main/webapp/WEB-INF/pages/manage/eml/acknowledgements.ftl @@ -52,6 +52,8 @@ // Submit the form this.submit(); }); + + makeSureResourceParameterIsPresentInURL('${resource.shortname}'); }); <#assign currentMenu="manage"/> @@ -74,6 +76,8 @@
+ +
diff --git a/src/main/webapp/WEB-INF/pages/manage/eml/additional.ftl b/src/main/webapp/WEB-INF/pages/manage/eml/additional.ftl index 175872bcfb..48dc3165a9 100644 --- a/src/main/webapp/WEB-INF/pages/manage/eml/additional.ftl +++ b/src/main/webapp/WEB-INF/pages/manage/eml/additional.ftl @@ -115,6 +115,8 @@ hideProcessing(); } + + makeSureResourceParameterIsPresentInURL('${resource.shortname}'); }); <#assign currentMetadataPage = "additional"/> diff --git a/src/main/webapp/WEB-INF/pages/manage/eml/additionalDescription.ftl b/src/main/webapp/WEB-INF/pages/manage/eml/additionalDescription.ftl index a2afa79531..55ce1a824b 100644 --- a/src/main/webapp/WEB-INF/pages/manage/eml/additionalDescription.ftl +++ b/src/main/webapp/WEB-INF/pages/manage/eml/additionalDescription.ftl @@ -123,6 +123,8 @@ // Submit the form this.submit(); }); + + makeSureResourceParameterIsPresentInURL('${resource.shortname}'); }); <#assign currentMenu="manage"/> @@ -145,6 +147,8 @@
+ +
diff --git a/src/main/webapp/WEB-INF/pages/manage/eml/basic.ftl b/src/main/webapp/WEB-INF/pages/manage/eml/basic.ftl index f2f26306e8..055974a902 100644 --- a/src/main/webapp/WEB-INF/pages/manage/eml/basic.ftl +++ b/src/main/webapp/WEB-INF/pages/manage/eml/basic.ftl @@ -271,6 +271,8 @@ // Submit the form this.submit(); }); + + makeSureResourceParameterIsPresentInURL('${resource.shortname}'); });