Skip to content

Commit 5766a6e

Browse files
kcondonkcondon
authored
Merge pull request #3521 from IQSS/4.6-tweaks-for-harvesting-from-partners
4.6 tweaks for harvesting from partners
2 parents bace17e + f684e64 commit 5766a6e

6 files changed

Lines changed: 278 additions & 65 deletions

File tree

‎src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java‎

Lines changed: 178 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ public class ImportDDIServiceBean {
4747
public static final String NAMING_PROTOCOL_DOI = "doi";
4848
public static final String AGENCY_HANDLE = "handle";
4949
public static final String AGENCY_DOI = "DOI";
50+
public static final String AGENCY_DARA = "dara"; // da|ra - http://www.da-ra.de/en/home/
5051
public static final String REPLICATION_FOR_TYPE = "replicationFor";
5152
public static final String VAR_WEIGHTED = "wgtd";
5253
public static final String VAR_INTERVAL_CONTIN = "contin";
@@ -91,6 +92,7 @@ public class ImportDDIServiceBean {
9192
public static final String NOTE_SUBJECT_LOCKSS_PERM = "LOCKSS Permission";
9293

9394
public static final String NOTE_TYPE_REPLICATION_FOR = "DVN:REPLICATION_FOR";
95+
private static final String HARVESTED_FILE_STORAGE_PREFIX = "http://";
9496
private XMLInputFactory xmlInputFactory = null;
9597

9698
@EJB CustomFieldServiceBean customFieldService;
@@ -241,18 +243,28 @@ private void processCodeBook(ImportType importType, XMLStreamReader xmlr, Datase
241243
if (event == XMLStreamConstants.START_ELEMENT) {
242244
if (xmlr.getLocalName().equals("docDscr")) {
243245
processDocDscr(xmlr, datasetDTO);
244-
}
245-
else if (xmlr.getLocalName().equals("stdyDscr")) {
246+
} else if (xmlr.getLocalName().equals("stdyDscr")) {
246247
processStdyDscr(importType, xmlr, datasetDTO);
247-
}
248-
else if (xmlr.getLocalName().equals("fileDscr") && !isMigrationImport(importType)) {
248+
} else if (xmlr.getLocalName().equals("otherMat") && (isNewImport(importType) || isHarvestWithFilesImport(importType)) ) {
249+
processOtherMat(xmlr, datasetDTO, filesMap);
250+
} else if (xmlr.getLocalName().equals("fileDscr") && isHarvestWithFilesImport(importType)) {
251+
// If this is a harvesting import, we'll attempt to extract some minimal
252+
// file-level metadata information from the fileDscr sections as well.
253+
// TODO: add more info here... -- 4.6
254+
processFileDscrMinimal(xmlr, datasetDTO, filesMap);
255+
} else if (xmlr.getLocalName().equals("fileDscr") && isNewImport(importType)) {
256+
// this is a "full" fileDscr section - Dataverses use it
257+
// to encode *tabular* files only. It will contain the information
258+
// about variables, observations, etc. It will be complemented
259+
// by a number of <var> entries in the dataDscr section.
260+
// Dataverses do not use this section for harvesting exports, since
261+
// we don't harvest tabular metadata. And all the "regular"
262+
// file-level metadata is encoded in otherMat sections.
263+
// The goal is to one day be able to import such tabular
264+
// metadata using the direct (non-harvesting) import API.
249265
// EMK TODO: add this back in for ImportType.NEW
250266
//processFileDscr(xmlr, datasetDTO, filesMap);
251-
252-
}
253-
else if (xmlr.getLocalName().equals("otherMat") && (isNewImport(importType) || isHarvestWithFilesImport(importType)) ) {
254-
processOtherMat(xmlr, datasetDTO, filesMap);
255-
}
267+
}
256268

257269
} else if (event == XMLStreamConstants.END_ELEMENT) {
258270
if (xmlr.getLocalName().equals("codeBook")) return;
@@ -432,12 +444,23 @@ else if (xmlr.getLocalName().equals("relStdy")) {
432444
private void processCitation(ImportType importType, XMLStreamReader xmlr, DatasetDTO datasetDTO) throws XMLStreamException, ImportException {
433445
DatasetVersionDTO dvDTO = datasetDTO.getDatasetVersion();
434446
MetadataBlockDTO citation=datasetDTO.getDatasetVersion().getMetadataBlocks().get("citation");
447+
boolean distStatementProcessed = false;
435448
for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) {
436449
if (event == XMLStreamConstants.START_ELEMENT) {
437450
if (xmlr.getLocalName().equals("titlStmt")) processTitlStmt(xmlr, datasetDTO);
438451
else if (xmlr.getLocalName().equals("rspStmt")) processRspStmt(xmlr,citation);
439452
else if (xmlr.getLocalName().equals("prodStmt")) processProdStmt(xmlr,citation);
440-
else if (xmlr.getLocalName().equals("distStmt")) processDistStmt(xmlr,citation);
453+
else if (xmlr.getLocalName().equals("distStmt")) {
454+
if (distStatementProcessed) {
455+
// We've already encountered one Distribution Statement in
456+
// this citation, we'll just skip any consecutive ones.
457+
// This is a defensive check against duplicate distStmt
458+
// in some DDIs (notably, from ICPSR)
459+
} else {
460+
processDistStmt(xmlr,citation);
461+
distStatementProcessed = true;
462+
}
463+
}
441464
else if (xmlr.getLocalName().equals("serStmt")) processSerStmt(xmlr,citation);
442465
else if (xmlr.getLocalName().equals("verStmt")) processVerStmt(importType, xmlr,dvDTO);
443466
else if (xmlr.getLocalName().equals("notes")) {
@@ -882,11 +905,23 @@ private void processAnlyInfo(XMLStreamReader xmlr, MetadataBlockDTO socialScienc
882905

883906
private void processDataColl(XMLStreamReader xmlr, DatasetVersionDTO dvDTO) throws XMLStreamException {
884907
MetadataBlockDTO socialScience =getSocialScience(dvDTO);
908+
909+
String collMode = "";
910+
String timeMeth = "";
911+
String weight = "";
912+
885913
for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) {
886914
if (event == XMLStreamConstants.START_ELEMENT) {
887915
//timeMethod
888916
if (xmlr.getLocalName().equals("timeMeth")) {
889-
socialScience.getFields().add(FieldDTO.createPrimitiveFieldDTO("timeMethod", parseText( xmlr, "timeMeth" )));
917+
String thisValue = parseText( xmlr, "timeMeth" );
918+
if (!StringUtil.isEmpty(thisValue)) {
919+
if (!"".equals(timeMeth)) {
920+
timeMeth = timeMeth.concat(", ");
921+
}
922+
timeMeth = timeMeth.concat(thisValue);
923+
}
924+
//socialScience.getFields().add(FieldDTO.createPrimitiveFieldDTO("timeMethod", parseText( xmlr, "timeMeth" )));
890925
} else if (xmlr.getLocalName().equals("dataCollector")) {
891926
socialScience.getFields().add(FieldDTO.createPrimitiveFieldDTO("dataCollector", parseText( xmlr, "dataCollector" )));
892927
// frequencyOfDataCollection
@@ -903,7 +938,14 @@ private void processDataColl(XMLStreamReader xmlr, DatasetVersionDTO dvDTO) thro
903938
socialScience.getFields().add(FieldDTO.createPrimitiveFieldDTO("deviationsFromSampleDesign", parseText( xmlr, "deviat" )));
904939
// collectionMode
905940
} else if (xmlr.getLocalName().equals("collMode")) {
906-
socialScience.getFields().add(FieldDTO.createPrimitiveFieldDTO("collectionMode", parseText( xmlr, "collMode" )));
941+
String thisValue = parseText( xmlr, "collMode" );
942+
if (!StringUtil.isEmpty(thisValue)) {
943+
if (!"".equals(collMode)) {
944+
collMode = collMode.concat(", ");
945+
}
946+
collMode = collMode.concat(thisValue);
947+
}
948+
//socialScience.getFields().add(FieldDTO.createPrimitiveFieldDTO("collectionMode", parseText( xmlr, "collMode" )));
907949
//researchInstrument
908950
} else if (xmlr.getLocalName().equals("resInstru")) {
909951
socialScience.getFields().add(FieldDTO.createPrimitiveFieldDTO("researchInstrument", parseText( xmlr, "resInstru" )));
@@ -916,12 +958,30 @@ private void processDataColl(XMLStreamReader xmlr, DatasetVersionDTO dvDTO) thro
916958
} else if (xmlr.getLocalName().equals("ConOps")) {
917959
socialScience.getFields().add(FieldDTO.createPrimitiveFieldDTO("controlOperations", parseText( xmlr, "ConOps" )));
918960
} else if (xmlr.getLocalName().equals("weight")) {
919-
socialScience.getFields().add(FieldDTO.createPrimitiveFieldDTO("weighting", parseText( xmlr, "weight" )));
961+
String thisValue = parseText( xmlr, "weight" );
962+
if (!StringUtil.isEmpty(thisValue)) {
963+
if (!"".equals(weight)) {
964+
weight = weight.concat(", ");
965+
}
966+
weight = weight.concat(thisValue);
967+
}
968+
//socialScience.getFields().add(FieldDTO.createPrimitiveFieldDTO("weighting", parseText( xmlr, "weight" )));
920969
} else if (xmlr.getLocalName().equals("cleanOps")) {
921970
socialScience.getFields().add(FieldDTO.createPrimitiveFieldDTO("cleaningOperations", parseText( xmlr, "cleanOps" )));
922-
}
971+
}
923972
} else if (event == XMLStreamConstants.END_ELEMENT) {
924-
if (xmlr.getLocalName().equals("dataColl")) return;
973+
if (xmlr.getLocalName().equals("dataColl")) {
974+
if (!StringUtil.isEmpty(timeMeth)) {
975+
socialScience.getFields().add(FieldDTO.createPrimitiveFieldDTO("timeMethod", timeMeth));
976+
}
977+
if (!StringUtil.isEmpty(collMode)) {
978+
socialScience.getFields().add(FieldDTO.createPrimitiveFieldDTO("collectionMode", collMode));
979+
}
980+
if (!StringUtil.isEmpty(weight)) {
981+
socialScience.getFields().add(FieldDTO.createPrimitiveFieldDTO("weighting", weight));
982+
}
983+
return;
984+
}
925985
}
926986
}
927987
}
@@ -1242,6 +1302,16 @@ private void processTitlStmt(XMLStreamReader xmlr, DatasetDTO datasetDTO) throws
12421302
parseStudyIdHandle( parseText(xmlr), datasetDTO );
12431303
} else if ( AGENCY_DOI.equals( xmlr.getAttributeValue(null, "agency") ) ) {
12441304
parseStudyIdDOI( parseText(xmlr), datasetDTO );
1305+
} else if ( AGENCY_DARA.equals( xmlr.getAttributeValue(null, "agency"))) {
1306+
/*
1307+
da|ra - "Registration agency for social and economic data"
1308+
(http://www.da-ra.de/en/home/)
1309+
ICPSR uses da|ra to register their DOIs; so they have agency="dara"
1310+
in their IDNo entries.
1311+
Also, their DOIs are formatted differently, without the
1312+
hdl: prefix.
1313+
*/
1314+
parseStudyIdDoiICPSRdara( parseText(xmlr), datasetDTO );
12451315
} else {
12461316
HashSet<FieldDTO> set = new HashSet<>();
12471317
addToSet(set,"otherIdAgency", xmlr.getAttributeValue(null, "agency"));
@@ -1325,16 +1395,23 @@ private Object parseTextNew(XMLStreamReader xmlr, String endTag) throws XMLStrea
13251395
if (event == XMLStreamConstants.CHARACTERS) {
13261396
returnString += xmlr.getText().trim().replace('\n',' ');
13271397
} else if (event == XMLStreamConstants.START_ELEMENT) {
1328-
if (xmlr.getLocalName().equals("p")) {
1329-
returnString += "<p>" + parseText(xmlr, "p") + "</p>";
1330-
} else if (xmlr.getLocalName().equals("emph")) {
1331-
returnString += "<em>" + parseText(xmlr, "emph") + "</em>";
1332-
} else if (xmlr.getLocalName().equals("hi")) {
1333-
returnString += "<strong>" + parseText(xmlr, "hi") + "</strong>";
1398+
if (xmlr.getLocalName().equals("p") || xmlr.getLocalName().equals("br") || xmlr.getLocalName().equals("head")) {
1399+
returnString += "<p>" + parseText(xmlr, xmlr.getLocalName()) + "</p>";
1400+
} else if (xmlr.getLocalName().equals("emph") || xmlr.getLocalName().equals("em") || xmlr.getLocalName().equals("i")) {
1401+
returnString += "<em>" + parseText(xmlr, xmlr.getLocalName()) + "</em>";
1402+
} else if (xmlr.getLocalName().equals("hi") || xmlr.getLocalName().equals("b")) {
1403+
returnString += "<strong>" + parseText(xmlr, xmlr.getLocalName()) + "</strong>";
13341404
} else if (xmlr.getLocalName().equals("ExtLink")) {
13351405
String uri = xmlr.getAttributeValue(null, "URI");
13361406
String text = parseText(xmlr, "ExtLink").trim();
13371407
returnString += "<a href=\"" + uri + "\">" + ( StringUtil.isEmpty(text) ? uri : text) + "</a>";
1408+
} else if (xmlr.getLocalName().equals("a") || xmlr.getLocalName().equals("A")) {
1409+
String uri = xmlr.getAttributeValue(null, "URI");
1410+
if (StringUtil.isEmpty(uri)) {
1411+
uri = xmlr.getAttributeValue(null, "HREF");
1412+
}
1413+
String text = parseText(xmlr, xmlr.getLocalName()).trim();
1414+
returnString += "<a href=\"" + uri + "\">" + ( StringUtil.isEmpty(text) ? uri : text) + "</a>";
13381415
} else if (xmlr.getLocalName().equals("list")) {
13391416
returnString += parseText_list(xmlr);
13401417
} else if (xmlr.getLocalName().equals("citation")) {
@@ -1343,6 +1420,8 @@ private Object parseTextNew(XMLStreamReader xmlr, String endTag) throws XMLStrea
13431420
} else {
13441421
returnString += parseText_citation(xmlr);
13451422
}
1423+
} else if (xmlr.getLocalName().equals("txt")) {
1424+
returnString += parseText(xmlr);
13461425
} else {
13471426
throw new EJBException("ERROR occurred in mapDDI (parseText): tag not yet supported: <" + xmlr.getLocalName() + ">" );
13481427
}
@@ -1373,7 +1452,7 @@ private String parseText_list (XMLStreamReader xmlr) throws XMLStreamException {
13731452

13741453
// check type
13751454
String listType = xmlr.getAttributeValue(null, "type");
1376-
if ("bulleted".equals(listType) ){
1455+
if ("bulleted".equals(listType) || listType == null){
13771456
listString = "<ul>\n";
13781457
listCloseTag = "</ul>";
13791458
} else if ("ordered".equals(listType) ) {
@@ -1524,6 +1603,31 @@ private void parseStudyIdDOI(String _id, DatasetDTO datasetDTO) throws ImportExc
15241603

15251604
datasetDTO.setIdentifier(_id.substring(index2+1));
15261605
}
1606+
1607+
private void parseStudyIdDoiICPSRdara(String _id, DatasetDTO datasetDTO) throws ImportException{
1608+
/*
1609+
dara/ICPSR DOIs are formatted without the hdl: prefix; for example -
1610+
10.3886/ICPSR06635.v1
1611+
so we assume that everything before the last "/" is the authority,
1612+
and everything past it - the identifier:
1613+
*/
1614+
1615+
int index = _id.lastIndexOf('/');
1616+
1617+
if (index == -1) {
1618+
throw new ImportException("Error parsing ICPSR/dara DOI IdNo: "+_id+". '/' not found in string");
1619+
}
1620+
1621+
if (index == _id.length() - 1) {
1622+
throw new ImportException("Error parsing ICPSR/dara DOI IdNo: "+_id+" ends with '/'");
1623+
}
1624+
1625+
datasetDTO.setAuthority(_id.substring(0, index));
1626+
datasetDTO.setProtocol("doi");
1627+
datasetDTO.setDoiSeparator("/");
1628+
1629+
datasetDTO.setIdentifier(_id.substring(index+1));
1630+
}
15271631
// Helper methods
15281632
private MetadataBlockDTO getCitation(DatasetVersionDTO dvDTO) {
15291633
return dvDTO.getMetadataBlocks().get("citation");
@@ -1609,6 +1713,58 @@ private void processOtherMat(XMLStreamReader xmlr, DatasetDTO datasetDTO, Map fi
16091713
}
16101714
}
16111715

1716+
// this method is for attempting to extract the minimal amount of file-level
1717+
// metadata from an ICPSR-supplied DDI. (they use the "fileDscr" instead of
1718+
// "otherMat" for general file metadata; the only field they populate is
1719+
// "fileName". -- 4.6
1720+
1721+
private void processFileDscrMinimal(XMLStreamReader xmlr, DatasetDTO datasetDTO, Map filesMap) throws XMLStreamException {
1722+
FileMetadataDTO fmdDTO = new FileMetadataDTO();
1723+
1724+
if (datasetDTO.getDatasetVersion().getFileMetadatas() == null) {
1725+
datasetDTO.getDatasetVersion().setFileMetadatas(new ArrayList<>());
1726+
}
1727+
datasetDTO.getDatasetVersion().getFileMetadatas().add(fmdDTO);
1728+
1729+
DataFileDTO dfDTO = new DataFileDTO();
1730+
dfDTO.setContentType("data/various-formats"); // reserved ICPSR content type identifier
1731+
fmdDTO.setDataFile(dfDTO);
1732+
1733+
for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) {
1734+
if (event == XMLStreamConstants.START_ELEMENT) {
1735+
if (xmlr.getLocalName().equals("fileName")) {
1736+
// this is the file name:
1737+
String label = parseText(xmlr);
1738+
// do some cleanup:
1739+
int col = label.lastIndexOf(':');
1740+
if ( col > -1) {
1741+
if (col < label.length() - 1) {
1742+
label = label.substring(col+1);
1743+
} else {
1744+
label = label.replaceAll(":", "");
1745+
}
1746+
}
1747+
label = label.replaceAll("[#;<>\\?\\|\\*\"]", "");
1748+
label = label.replaceAll("/", "-");
1749+
// strip leading blanks:
1750+
label = label.replaceFirst("^[ \t]*", "");
1751+
fmdDTO.setLabel(label);
1752+
}
1753+
} else if (event == XMLStreamConstants.END_ELEMENT) {
1754+
if (xmlr.getLocalName().equals("fileDscr")) {
1755+
if (fmdDTO.getLabel() == null || fmdDTO.getLabel().trim().equals("") ) {
1756+
fmdDTO.setLabel("harvested file");
1757+
}
1758+
if (StringUtil.isEmpty(fmdDTO.getDataFile().getStorageIdentifier())) {
1759+
fmdDTO.getDataFile().setStorageIdentifier(HARVESTED_FILE_STORAGE_PREFIX);
1760+
}
1761+
1762+
return;
1763+
}
1764+
}
1765+
}
1766+
}
1767+
16121768
private void processFileDscr(XMLStreamReader xmlr, DatasetDTO datasetDTO, Map filesMap) throws XMLStreamException {
16131769
FileMetadataDTO fmdDTO = new FileMetadataDTO();
16141770

0 commit comments

Comments
 (0)