Merge pull request #6 from arunaengine/feat/remote-subcrate-main

Feat/remote subcrate handling

Author: mburridge96

Cargo.lock

@@ -1,7 +1,7 @@
 [package]
 name = "ro-crate-rs"
 version = "0.4.12"
-edition = "2021"
+edition = "2024"
 repository = "https://github.com/intbio-ncl/ro-crate-rs"
 authors = ["Matt Burridge <m.burridge1@newcastle.ac.uk>"]
 autoexamples = false
@@ -31,6 +31,11 @@ dirs = "6.0.0"
 polars = { version = "0.52.0", features = ["parquet"] }
 log = "0.4.29"
 thiserror = "2.0.17"
+sha1 = "0.10.6"
+sha2 = "0.10.9"
+hex = "0.4.3"
+bytes = "1.11.0"
+regex = "1.12.2"
 
 [lib]
 name = "rocraters"
@@ -39,6 +44,7 @@ doctest = false
 
 [features]
 parquet = ["dep:polars"]
+subcrate-resolution = ["dep:sha1","dep:sha2","dep:hex","dep:bytes","dep:regex"]
 
 [dependencies]
 serde = { workspace = true }
@@ -54,6 +60,13 @@ dirs = { workspace = true }
 polars = { workspace = true, optional = true }
 thiserror = { workspace = true }
 log = { workspace = true }
+sha1 = { workspace = true, optional = true }
+sha2 = { workspace = true, optional = true }
+hex = { workspace = true, optional = true }
+bytes = { workspace = true, optional = true }
+regex = { workspace = true, optional = true }
+
 
 [dev-dependencies]
 tempfile = "3.24"
+mockito = "1.7.1"

Cargo.toml

@@ -1,5 +1,3 @@
-Working in Alpha
-
 # Overview 
 
 ro-crate-rs is a rust library, python library and CLI tool for interfacing with [Research Object Crates](https://www.researchobject.org/ro-crate/). 
@@ -59,5 +57,5 @@ full python implementation.
 
 # Compatability 
 
-RO-Crate v1.1 only
+RO-Crate v1.1 and v1.2
 

README.md

@@ -4,7 +4,7 @@ use rocraters::ro_crate::constraints::*;
 use rocraters::ro_crate::contextual_entity::ContextualEntity;
 use rocraters::ro_crate::data_entity::DataEntity;
 use rocraters::ro_crate::metadata_descriptor::MetadataDescriptor;
-use rocraters::ro_crate::read::{crate_path, read_crate, CrateReadError};
+use rocraters::ro_crate::read::{CrateReadError, crate_path, read_crate};
 use rocraters::ro_crate::rocrate::GraphVector;
 use rocraters::ro_crate::rocrate::{RoCrate, RoCrateContext};
 use rocraters::ro_crate::root::RootDataEntity;

benches/create_and_write.rs

@@ -166,7 +166,7 @@ fn main() {
                     println!("{}", to_string_pretty(&rocrate).unwrap());
                 } else {
                     match to_string_pretty(&rocrate) {
-                        Ok(json_ld) => {
+                        Ok(_json_ld) => {
                             let mut table = json_to_table(&json!(&rocrate.graph)).into_table();
                             table.with(Style::modern_rounded());
                             if read_crate_command.fit {
@@ -275,7 +275,7 @@ fn main() {
             ValidateCommand::Basic(basic) => {
                 let crate_name = crate_path(&basic.target_crate);
                 match read_crate(&crate_name, 2) {
-                    Ok(rocrate) => println!("Crate Valid"),
+                    Ok(_rocrate) => println!("Crate Valid"),
                     Err(e) => println!("Crate not valid: {:?}", e),
                 }
             }

cli/src/main.rs

@@ -179,7 +179,7 @@ impl RoCrateContext {
             RoCrateContext::ExtendedContext(context) => {
                 for x in context {
                     match x {
-                        ContextItem::EmbeddedContext(ref mut embedded) => {
+                        ContextItem::EmbeddedContext(embedded) => {
                             let urn_found = embedded
                                 .get("@base")
                                 .is_some_and(|value| value.starts_with("urn:uuid:"));

src/ro_crate/context.rs

@@ -6,8 +6,8 @@
 use crate::ro_crate::constraints::*;
 use crate::ro_crate::modify::*;
 use serde::{
-    de::{self, MapAccess, Visitor},
     Deserialize, Deserializer, Serialize, Serializer,
+    de::{self, MapAccess, Visitor},
 };
 use std::collections::HashMap;
 use std::fmt;

src/ro_crate/contextual_entity.rs

@@ -14,9 +14,18 @@ use crate::ro_crate::graph_vector::GraphVector;
 use crate::ro_crate::rocrate::RoCrate;
 use log::warn;
 use polars::prelude::*;
+use thiserror::Error;
 use std::collections::HashMap;
 use std::path::PathBuf;
 
+#[derive(Debug, Error)]
+pub enum ConvertError {
+    #[error(transparent)]
+    IoError(#[from] std::io::Error),
+    #[error(transparent)]
+    PolarsError(#[from] polars::error::PolarsError)
+}
+
 pub fn to_df(rocrate: &RoCrate) -> DataFrame {
     // Get uuid
     let uuid = rocrate.context.get_urn_uuid().unwrap();
@@ -45,13 +54,15 @@ pub fn to_df(rocrate: &RoCrate) -> DataFrame {
     .unwrap()
 }
 
-pub fn write_csv(df: &mut DataFrame, path: PathBuf) {
-    let mut file = std::fs::File::create(path).unwrap();
-    CsvWriter::new(&mut file).finish(df).unwrap();
+pub fn write_csv(df: &mut DataFrame, path: PathBuf) -> Result<(), ConvertError> {
+    let mut file = std::fs::File::create(path)?;
+    CsvWriter::new(&mut file).finish(df)?;
+    Ok(())
 }
-pub fn write_parquet(df: &mut DataFrame, path: PathBuf) {
-    let mut file = std::fs::File::create(path).unwrap();
-    ParquetWriter::new(&mut file).finish(df).unwrap();
+pub fn write_parquet(df: &mut DataFrame, path: PathBuf) -> Result<(), ConvertError> {
+    let mut file = std::fs::File::create(path)?;
+    ParquetWriter::new(&mut file).finish(df)?;
+    Ok(())
 }
 
 pub fn to_parquet(df: &mut DataFrame) -> Result<u64, PolarsError> {
@@ -270,8 +281,8 @@ mod write_crate_tests {
         let mut df = to_df(&rocrate);
         let path_csv = fixture_path("test-ro-crate.csv");
         let path_parquet = fixture_path("test-ro-crate.parquet");
-        write_csv(&mut df, path_csv);
-        write_parquet(&mut df, path_parquet);
+        write_csv(&mut df, path_csv).unwrap();
+        write_parquet(&mut df, path_parquet).unwrap();
     }
 
     #[test]
@@ -287,9 +298,9 @@ mod write_crate_tests {
         println!("Created dataframe");
         let path_csv = fixture_path("test-ro-crate.csv");
         let path_parquet = fixture_path("test-ro-crate.parquet");
-        write_csv(&mut df, path_csv);
+        write_csv(&mut df, path_csv).unwrap();
         println!("CSV created");
-        write_parquet(&mut df, path_parquet);
+        write_parquet(&mut df, path_parquet).unwrap();
         println!("Parquet created");
     }
 }

src/ro_crate/convert.rs

@@ -6,8 +6,8 @@
 use crate::ro_crate::constraints::*;
 use crate::ro_crate::modify::*;
 use serde::{
-    de::{self, MapAccess, Visitor},
     Deserialize, Deserializer, Serialize, Serializer,
+    de::{self, MapAccess, Visitor},
 };
 use std::collections::HashMap;
 use std::fmt;

src/ro_crate/data_entity.rs

@@ -10,8 +10,6 @@ use serde::ser::Serializer;
 use serde::{Deserialize, Deserializer, Serialize};
 use serde_json::{Error as SerdeJsonError, Value};
 use std::collections::HashMap;
-use std::path::Path;
-use url::Url;
 
 /// Represents the various types of entities contained within the graph of an RO-Crate.
 ///
@@ -41,25 +39,25 @@ impl GraphVector {
     /// entity is correctly referenced in the root data entity's `hasPart` property if it is not already listed.
     pub fn overwrite(&mut self, new_data: GraphVector) -> bool {
         match self {
-            GraphVector::DataEntity(ref mut existing_entity) => {
+            GraphVector::DataEntity(existing_entity) => {
                 if let GraphVector::DataEntity(new_entity) = new_data {
                     *existing_entity = new_entity;
                     return true;
                 }
             }
-            GraphVector::ContextualEntity(ref mut existing_entity) => {
+            GraphVector::ContextualEntity(existing_entity) => {
                 if let GraphVector::ContextualEntity(new_entity) = new_data {
                     *existing_entity = new_entity;
                     return true;
                 }
             }
-            GraphVector::RootDataEntity(ref mut existing_entity) => {
+            GraphVector::RootDataEntity(existing_entity) => {
                 if let GraphVector::RootDataEntity(new_entity) = new_data {
                     *existing_entity = new_entity;
                     return true;
                 }
             }
-            GraphVector::MetadataDescriptor(ref mut existing_entity) => {
+            GraphVector::MetadataDescriptor(existing_entity) => {
                 if let GraphVector::MetadataDescriptor(new_entity) = new_data {
                     *existing_entity = new_entity;
                     return true;
@@ -351,10 +349,14 @@ fn try_deserialize_into_graph_vector(value: &Value) -> Result<GraphVector, Serde
             }
             "./" => RootDataEntity::deserialize(value).map(GraphVector::RootDataEntity),
             _ => {
-                if is_valid_url_or_path(id) {
-                    DataEntity::deserialize(value).map(GraphVector::DataEntity)
+                if let Some(entity_type) = value.get("@type") {
+                    if is_data_entity(entity_type, id) {
+                        DataEntity::deserialize(value).map(GraphVector::DataEntity)
+                    } else {
+                        ContextualEntity::deserialize(value).map(GraphVector::ContextualEntity)
+                    }
                 } else {
-                    ContextualEntity::deserialize(value).map(GraphVector::ContextualEntity)
+                    Err(serde::de::Error::custom("Missing or invalid '@type' field"))
                 }
             }
         }
@@ -368,6 +370,32 @@ fn try_deserialize_into_graph_vector(value: &Value) -> Result<GraphVector, Serde
 /// This function is used as part of the custom deserialization process in `GraphVector`
 /// to distinguish between `DataEntity` and `ContextualEntity`. It validates whether the
 /// provided string (`s`) is a well-formed URL or a path that corresponds to an existing file.
-fn is_valid_url_or_path(s: &str) -> bool {
-    Url::parse(s).is_ok() || Path::new(s).exists()
+///
+/// NOTE: The documentation states that:
+/// ```
+/// An entity which has File or Dataset as one of its @type values:
+///
+///     Is considered to be a Data Entity if its @id is an absolute URI or a relative URI.
+///     MAY have an @id which is a local identifier beginning with a #, in which case it is not considered to be a Data Entity.
+/// ```
+/// So the file path does not need to exist, and even if it exists, this library does not
+/// necessarily know the root of the crate. Also ContextualEntities can contain valid URIs as
+/// the documentation says [here](https://www.researchobject.org/ro-crate/specification/1.2/contextual-entities.html#identifiers-for-contextual-entities)
+// fn _is_valid_url_or_path(s: &str) -> bool {
+//     Url::parse(s).is_ok() || Path::new(s).exists()
+// }
+fn is_data_entity(entity_type: &Value, id: &str) -> bool {
+    if id.starts_with('#') {
+        return false;
+    }
+
+    entity_type.as_str().map_or_else(
+        || {
+            entity_type.as_array().map_or(false, |arr| {
+                arr.iter()
+                    .any(|v| matches!(v.as_str(), Some("File" | "Dataset")))
+            })
+        },
+        |s| matches!(s, "File" | "Dataset"),
+    )
 }