Reduce Schema.org jsonld for datasets with many files

doc/release-notes/12118-scalable-schema-org.md

@@ -0,0 +1,6 @@
+# Scalable Schema.org Export
+
+The Schema.org export (available as an export format and embedded as JSON-LD in dataset page headers) has been improved to better handle datasets with many files.
+
+- For datasets with a large number of files, the export can now use a more scalable `SearchAction` structure instead of listing every individual file in the `distribution` field. This ensures that the metadata remains readable by Google and other search engines without exceeding size limits.
+- A new configuration option `dataverse.exports.schema-dot-org.max-files-for-download-entries` has been added to control the threshold for this behavior. By default, all files are included.

doc/sphinx-guides/source/api/native-api.rst

@@ -2401,6 +2401,9 @@ The standard has further evolved into a format called Croissant. For details, se
 
 The ``schema.org`` format changed after Dataverse 6.4 as well. Previously its content type was "application/json" but now it is "application/ld+json".
 
+As of Dataverse v6.12, the ``schema.org`` format may be configured to replace the ``distribution`` field (listing individual files) with a ``potentialAction`` field (providing a template for file downloads) for datasets with many files, which can reduce the size of the export and avoid errors when scanned by Google. See :ref:`dataverse.exports.schema-dot-org.max-files-for-download-entries` in the Installation Guide for details.
+
+
 List Files in a Dataset
 ~~~~~~~~~~~~~~~~~~~~~~~
 

doc/sphinx-guides/source/installation/config.rst

@@ -3920,6 +3920,19 @@ Instead of Croissant, use the legacy format (Schema.org JSON-LD) in the head of
 
 Can also be set via any `supported MicroProfile Config API source`_, e.g. the environment variable ``DATAVERSE_LEGACY_SCHEMAORG_IN_HTML_HEAD``.
 
+.. _dataverse.exports.schema-dot-org.max-files-for-download-entries:
+
+dataverse.exports.schema-dot-org.max-files-for-download-entries
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+To improve scalability for datasets with many files, you can limit the number of files included in the Schema.org JSON-LD output (both in the export format and the HTML header). When the number of files in a dataset version exceeds this threshold, individual file entries in the ``distribution`` field are replaced with a more compact ``potentialAction`` structure that search engines like Google can use to discover how to download files.
+
+By default, all files are included (effectively ``Integer.MAX_VALUE``).
+
+``./asadmin create-jvm-options '-Ddataverse.exports.schema-dot-org.max-files-for-download-entries=1000'``
+
+Can also be set via any `supported MicroProfile Config API source`_, e.g. the environment variable ``DATAVERSE_EXPORTS_SCHEMA_DOT_ORG_MAX_FILES_FOR_DOWNLOAD_ENTRIES``.
+
 .. dataverse.ldn
 
 Linked Data Notifications (LDN) Allowed Hosts

src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java

@@ -2095,30 +2095,70 @@ public String getJsonLd() {
         }
 
         List<FileMetadata> fileMetadatasSorted = getFileMetadatasSorted();
+
         if (fileMetadatasSorted != null && !fileMetadatasSorted.isEmpty()) {
-            JsonArrayBuilder fileArray = Json.createArrayBuilder();
-            String dataverseSiteUrl = SystemConfig.getDataverseSiteUrlStatic();
-            for (FileMetadata fileMetadata : fileMetadatasSorted) {
-                JsonObjectBuilder fileObject = NullSafeJsonBuilder.jsonObjectBuilder();
-                String filePidUrlAsString = null;
-                GlobalId gid = fileMetadata.getDataFile().getGlobalId();
-                filePidUrlAsString = gid != null ? gid.asURL() : null;
-                fileObject.add("@type", "DataDownload");
-                fileObject.add("name", fileMetadata.getLabel());
-                fileObject.add("encodingFormat", fileMetadata.getDataFile().getContentType());
-                fileObject.add("contentSize", fileMetadata.getDataFile().getFilesize());
-                fileObject.add("description", MarkupChecker.stripAllTags(fileMetadata.getDescription()));
-                fileObject.add("@id", filePidUrlAsString);
-                fileObject.add("identifier", filePidUrlAsString);
-                boolean hideFilesBoolean = JvmSettings.HIDE_SCHEMA_DOT_ORG_DOWNLOAD_URLS.lookupOptional(Boolean.class).orElse(false);
-                if (!hideFilesBoolean) {
-                    String nullDownloadType = null;
+            Integer maxFilesForDownloadEntries = JvmSettings.EXPORTS_SCHEMA_DOT_ORG_MAX_FILES_FOR_DOWNLOAD_ENTRIES.lookupOptional(Integer.class).orElse(Integer.MAX_VALUE);
+            if (fileMetadatasSorted.size() <= maxFilesForDownloadEntries) {
+
+                JsonArrayBuilder fileArray = Json.createArrayBuilder();
+                String dataverseSiteUrl = SystemConfig.getDataverseSiteUrlStatic();
+                for (FileMetadata fileMetadata : fileMetadatasSorted) {
+                    JsonObjectBuilder fileObject = NullSafeJsonBuilder.jsonObjectBuilder();
+                    String filePidUrlAsString = null;
+                    GlobalId gid = fileMetadata.getDataFile().getGlobalId();
+                    filePidUrlAsString = gid != null ? gid.asURL() : null;
+                    fileObject.add("@type", "DataDownload");
+                    fileObject.add("name", fileMetadata.getLabel());
+                    fileObject.add("encodingFormat", fileMetadata.getDataFile().getContentType());
+                    fileObject.add("contentSize", fileMetadata.getDataFile().getFilesize());
+                    fileObject.add("description", MarkupChecker.stripAllTags(fileMetadata.getDescription()));
+                    fileObject.add("@id", filePidUrlAsString);
+                    fileObject.add("identifier", filePidUrlAsString);
+                    boolean hideFilesBoolean = JvmSettings.HIDE_SCHEMA_DOT_ORG_DOWNLOAD_URLS.lookupOptional(Boolean.class).orElse(false);
+                    if (!hideFilesBoolean) {
+                        String nullDownloadType = null;
                     fileObject.add("contentUrl", dataverseSiteUrl + FileUtil.getFileDownloadUrlPath(nullDownloadType, fileMetadata.getDataFile().getId(), false, fileMetadata.getId(), null));
+                    }
+                    fileArray.add(fileObject);
+                }
+                job.add("distribution", fileArray);
+            } else {
+                // If we have too many files, we don't include individual distribution entries
+                // but we can still provide a search action for file downloads
+                StringBuilder valuePattern = new StringBuilder();
+                boolean first = true;
+
+                // Build the pattern of all file IDs joined with OR operator
+                for (FileMetadata fileMetadata : fileMetadatas) {
+                    if (!first) {
+                        valuePattern.append("|");
+                    } else {
+                        first = false;
+                    }
+                    valuePattern.append(fileMetadata.getDataFile().getId());
                 }
-                fileArray.add(fileObject);
+
+                // Create the potentialAction object
+                JsonObjectBuilder potentialAction = Json.createObjectBuilder()
+                        .add("@type", "SearchAction")
+                        .add("target", Json.createObjectBuilder()
+                                .add("@type", "EntryPoint")
+                                .add("contentType", Json.createArrayBuilder().add("*/*"))
+                                .add("urlTemplate", SystemConfig.getDataverseSiteUrlStatic() + "/api/access/datafile/{fileId}")
+                                .add("description", "Download each file from the dataset based on file id")
+                                .add("httpMethod", Json.createArrayBuilder().add("GET")))
+                        .add("query-input", Json.createArrayBuilder()
+                                .add(Json.createObjectBuilder()
+                                        .add("@type", "PropertyValueSpecification")
+                                        .add("valueName", "fileId")
+                                        .add("description", "Id of the desired file")
+                                        .add("valueRequired", true)
+                                        .add("valuePattern", "(" + valuePattern.toString() + ")")));
+
+                job.add("potentialAction", potentialAction);
             }
-            job.add("distribution", fileArray);
         }
+
         jsonLd = job.build().toString();
 
         //Most fields above should be stripped/sanitized but, since this is output in the dataset page as header metadata, do a final sanitize step to make sure

src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java

@@ -318,6 +318,12 @@ public enum JvmSettings {
     SCOPE_COARNOTIFY(SCOPE_LINKEDDATANOTIFICATION, "coar-notify"),
     SCOPE_COARNOTIFY_RELATIONSHIP_ANNOUNCEMENT(SCOPE_COARNOTIFY, "relationship-announcement"),
     COARNOTIFY_RELATIONSHIP_ANNOUNCEMENT_NOTIFY_SUPERUSERS_ONLY(SCOPE_COARNOTIFY_RELATIONSHIP_ANNOUNCEMENT, "notify-superusers-only"),
+
+    //EXPORTS
+    SCOPE_EXPORTS(PREFIX, "exports"),
+    SCOPE_EXPORTS_SCHEMA_DOT_ORG(SCOPE_EXPORTS, "schema-dot-org"),
+    EXPORTS_SCHEMA_DOT_ORG_MAX_FILES_FOR_DOWNLOAD_ENTRIES(SCOPE_EXPORTS_SCHEMA_DOT_ORG, "max-files-for-download-entries"),
+
     ;
 
     private static final String SCOPE_SEPARATOR = ".";