Remove hardcoded registry file

delft/__init__.py

@@ -1,3 +1,10 @@
 import os
 
 DELFT_PROJECT_DIR = os.path.dirname(__file__)
+
+
+def get_registry_path():
+    env_path = os.environ.get("DELFT_REGISTRY_PATH")
+    if env_path:
+        return env_path
+    return os.path.join(DELFT_PROJECT_DIR, "resources-registry.json")

delft/sequenceLabelling/models.py

@@ -31,7 +31,9 @@
 """
 
 
-def get_model(config: ModelConfig, preprocessor, ntags=None, load_pretrained_weights=True, local_path=None):
+def get_model(
+    config: ModelConfig, preprocessor, ntags=None, load_pretrained_weights=True, local_path=None, registry=None
+):
     """
     Return a model instance by its name. This is a facilitator function.
     """
@@ -114,6 +116,7 @@ def get_model(config: ModelConfig, preprocessor, ntags=None, load_pretrained_wei
             load_pretrained_weights=load_pretrained_weights,
             local_path=local_path,
             preprocessor=preprocessor,
+            registry=registry,
         )
 
     elif config.architecture == BERT_FEATURES.name:
@@ -126,6 +129,7 @@ def get_model(config: ModelConfig, preprocessor, ntags=None, load_pretrained_wei
             load_pretrained_weights=load_pretrained_weights,
             local_path=local_path,
             preprocessor=preprocessor,
+            registry=registry,
         )
 
     elif config.architecture == BERT_CRF.name:
@@ -138,6 +142,7 @@ def get_model(config: ModelConfig, preprocessor, ntags=None, load_pretrained_wei
             load_pretrained_weights=load_pretrained_weights,
             local_path=local_path,
             preprocessor=preprocessor,
+            registry=registry,
         )
 
     elif config.architecture == BERT_ChainCRF.name:
@@ -151,6 +156,7 @@ def get_model(config: ModelConfig, preprocessor, ntags=None, load_pretrained_wei
             load_pretrained_weights=load_pretrained_weights,
             local_path=local_path,
             preprocessor=preprocessor,
+            registry=registry,
         )
 
     elif config.architecture == BERT_CRF_FEATURES.name:
@@ -164,6 +170,7 @@ def get_model(config: ModelConfig, preprocessor, ntags=None, load_pretrained_wei
             load_pretrained_weights=load_pretrained_weights,
             local_path=local_path,
             preprocessor=preprocessor,
+            registry=registry,
         )
 
     elif config.architecture == BERT_ChainCRF_FEATURES.name:
@@ -178,6 +185,7 @@ def get_model(config: ModelConfig, preprocessor, ntags=None, load_pretrained_wei
             load_pretrained_weights=load_pretrained_weights,
             local_path=local_path,
             preprocessor=preprocessor,
+            registry=registry,
         )
 
     elif config.architecture == BERT_CRF_CHAR.name:
@@ -191,6 +199,7 @@ def get_model(config: ModelConfig, preprocessor, ntags=None, load_pretrained_wei
             load_pretrained_weights=load_pretrained_weights,
             local_path=local_path,
             preprocessor=preprocessor,
+            registry=registry,
         )
 
     elif config.architecture == BERT_CRF_CHAR_FEATURES.name:
@@ -205,6 +214,7 @@ def get_model(config: ModelConfig, preprocessor, ntags=None, load_pretrained_wei
             load_pretrained_weights=load_pretrained_weights,
             local_path=local_path,
             preprocessor=preprocessor,
+            registry=registry,
         )
     else:
         raise (OSError("Model name does exist: " + config.architecture))
@@ -230,16 +240,19 @@ class BaseModel(object):
     transformer_preprocessor = None
 
     def __init__(
-        self, config, ntags=None, load_pretrained_weights: bool = True, local_path: str = None, preprocessor=None
+        self,
+        config,
+        ntags=None,
+        load_pretrained_weights: bool = True,
+        local_path: str = None,
+        preprocessor=None,
     ):
         self.config = config
         self.ntags = ntags
         self.model = None
         self.local_path = local_path
         self.load_pretrained_weights = load_pretrained_weights
 
-        self.registry = load_resource_registry("delft/resources-registry.json")
-
     def predict(self, X, *args, **kwargs):
         y_pred = self.model.predict(X, batch_size=1)
         return y_pred
@@ -273,9 +286,15 @@ def print_summary(self):
         self.model.summary()
 
     def init_transformer(
-        self, config: ModelConfig, load_pretrained_weights: bool, local_path: str, preprocessor: Preprocessor
+        self,
+        config: ModelConfig,
+        load_pretrained_weights: bool,
+        local_path: str,
+        preprocessor: Preprocessor,
+        registry=None,
     ):
-        transformer = Transformer(config.transformer_name, resource_registry=self.registry, delft_local_path=local_path)
+        registry = registry if registry is not None else load_resource_registry()
+        transformer = Transformer(config.transformer_name, resource_registry=registry, delft_local_path=local_path)
         print(config.transformer_name, "will be used, loaded via", transformer.loading_method)
         transformer_model = transformer.instantiate_layer(load_pretrained_weights=load_pretrained_weights)
         self.transformer_config = transformer.transformer_config
@@ -907,11 +926,19 @@ class BERT(BaseModel):
     name = "BERT"
 
     def __init__(
-        self, config, ntags=None, load_pretrained_weights: bool = True, local_path: str = None, preprocessor=None
+        self,
+        config,
+        ntags=None,
+        load_pretrained_weights: bool = True,
+        local_path: str = None,
+        preprocessor=None,
+        registry=None,
     ):
         super().__init__(config, ntags, load_pretrained_weights, local_path)
 
-        transformer_layers = self.init_transformer(config, load_pretrained_weights, local_path, preprocessor)
+        transformer_layers = self.init_transformer(
+            config, load_pretrained_weights, local_path, preprocessor, registry=registry
+        )
 
         input_ids_in = Input(shape=(None,), name="input_token", dtype="int32")
         token_type_ids = Input(shape=(None,), name="input_token_type", dtype="int32")
@@ -950,11 +977,19 @@ class BERT_FEATURES(BaseModel):
     name = "BERT_FEATURES"
 
     def __init__(
-        self, config, ntags=None, load_pretrained_weights: bool = True, local_path: str = None, preprocessor=None
+        self,
+        config,
+        ntags=None,
+        load_pretrained_weights: bool = True,
+        local_path: str = None,
+        preprocessor=None,
+        registry=None,
     ):
         super().__init__(config, ntags, load_pretrained_weights, local_path)
 
-        transformer_layers = self.init_transformer(config, load_pretrained_weights, local_path, preprocessor)
+        transformer_layers = self.init_transformer(
+            config, load_pretrained_weights, local_path, preprocessor, registry=registry
+        )
 
         input_ids_in = Input(shape=(None,), name="input_token", dtype="int32")
         token_type_ids = Input(shape=(None,), name="input_token_type", dtype="int32")
@@ -1021,10 +1056,13 @@ def __init__(
         load_pretrained_weights: bool = True,
         local_path: str = None,
         preprocessor=None,
+        registry=None,
     ):
         super().__init__(config, ntags, load_pretrained_weights, local_path=local_path)
 
-        transformer_layers = self.init_transformer(config, load_pretrained_weights, local_path, preprocessor)
+        transformer_layers = self.init_transformer(
+            config, load_pretrained_weights, local_path, preprocessor, registry=registry
+        )
 
         input_ids_in = Input(shape=(None,), name="input_token", dtype="int32")
         token_type_ids = Input(shape=(None,), name="input_token_type", dtype="int32")
@@ -1078,10 +1116,13 @@ def __init__(
         load_pretrained_weights: bool = True,
         local_path: str = None,
         preprocessor=None,
+        registry=None,
     ):
         super().__init__(config, ntags, load_pretrained_weights, local_path=local_path)
 
-        transformer_layers = self.init_transformer(config, load_pretrained_weights, local_path, preprocessor)
+        transformer_layers = self.init_transformer(
+            config, load_pretrained_weights, local_path, preprocessor, registry=registry
+        )
 
         input_ids_in = Input(shape=(None,), name="input_token", dtype="int32")
         token_type_ids = Input(shape=(None,), name="input_token_type", dtype="int32")
@@ -1112,10 +1153,14 @@ class BERT_CRF_FEATURES(BaseModel):
 
     name = "BERT_CRF_FEATURES"
 
-    def __init__(self, config, ntags=None, load_pretrained_weights=True, local_path: str = None, preprocessor=None):
+    def __init__(
+        self, config, ntags=None, load_pretrained_weights=True, local_path: str = None, preprocessor=None, registry=None
+    ):
         super().__init__(config, ntags, load_pretrained_weights, local_path=local_path)
 
-        transformer_layers = self.init_transformer(config, load_pretrained_weights, local_path, preprocessor)
+        transformer_layers = self.init_transformer(
+            config, load_pretrained_weights, local_path, preprocessor, registry=registry
+        )
 
         input_ids_in = Input(shape=(None,), name="input_token", dtype="int32")
         token_type_ids = Input(shape=(None,), name="input_token_type", dtype="int32")
@@ -1196,10 +1241,14 @@ class BERT_ChainCRF_FEATURES(BaseModel):
 
     name = "BERT_ChainCRF_FEATURES"
 
-    def __init__(self, config, ntags=None, load_pretrained_weights=True, local_path: str = None, preprocessor=None):
+    def __init__(
+        self, config, ntags=None, load_pretrained_weights=True, local_path: str = None, preprocessor=None, registry=None
+    ):
         super().__init__(config, ntags, load_pretrained_weights, local_path=local_path)
 
-        transformer_layers = self.init_transformer(config, load_pretrained_weights, local_path, preprocessor)
+        transformer_layers = self.init_transformer(
+            config, load_pretrained_weights, local_path, preprocessor, registry=registry
+        )
 
         input_ids_in = Input(shape=(None,), name="input_token", dtype="int32")
         token_type_ids = Input(shape=(None,), name="input_token_type", dtype="int32")
@@ -1263,10 +1312,14 @@ class BERT_CRF_CHAR(BaseModel):
 
     name = "BERT_CRF_CHAR"
 
-    def __init__(self, config, ntags=None, load_pretrained_weights=True, local_path: str = None, preprocessor=None):
+    def __init__(
+        self, config, ntags=None, load_pretrained_weights=True, local_path: str = None, preprocessor=None, registry=None
+    ):
         super().__init__(config, ntags, load_pretrained_weights, local_path=local_path)
 
-        transformer_layers = self.init_transformer(config, load_pretrained_weights, local_path, preprocessor)
+        transformer_layers = self.init_transformer(
+            config, load_pretrained_weights, local_path, preprocessor, registry=registry
+        )
 
         input_ids_in = Input(shape=(None,), name="input_token", dtype="int32")
         token_type_ids = Input(shape=(None,), name="input_token_type", dtype="int32")
@@ -1340,10 +1393,14 @@ class BERT_CRF_CHAR_FEATURES(BaseModel):
 
     name = "BERT_CRF_CHAR_FEATURES"
 
-    def __init__(self, config, ntags=None, load_pretrained_weights=True, local_path: str = None, preprocessor=None):
+    def __init__(
+        self, config, ntags=None, load_pretrained_weights=True, local_path: str = None, preprocessor=None, registry=None
+    ):
         super().__init__(config, ntags, load_pretrained_weights, local_path=local_path)
 
-        transformer_layers = self.init_transformer(config, load_pretrained_weights, local_path, preprocessor)
+        transformer_layers = self.init_transformer(
+            config, load_pretrained_weights, local_path, preprocessor, registry=registry
+        )
 
         input_ids_in = Input(shape=(None,), name="input_token", dtype="int32")
         token_type_ids = Input(shape=(None,), name="input_token_type", dtype="int32")

delft/sequenceLabelling/wrapper.py

@@ -9,7 +9,7 @@
 #    get initialised before CUDA 11 from pytorch.
 import tf_keras as keras  # noqa: F401
 
-from delft import DELFT_PROJECT_DIR
+from delft import get_registry_path
 from delft.utilities.misc import print_parameters
 
 # ask tensorflow to be quiet and not print hundred lines of logs
@@ -100,6 +100,7 @@ def __init__(
         transformer_name: str = None,
         report_to_wandb=False,
         nb_workers=6,
+        resource_registry_path=None,
     ):
 
         if model_name is None:
@@ -123,10 +124,10 @@ def __init__(
 
         self.report_to_wandb = report_to_wandb
 
-        self.registry = load_resource_registry(os.path.join(DELFT_PROJECT_DIR, "resources-registry.json"))
+        self.registry = load_resource_registry(resource_registry_path or get_registry_path())
 
         if self.embeddings_name is not None:
-            self.embeddings = Embeddings(self.embeddings_name, resource_registry=self.registry)
+            self.embeddings = self.get_embedding(self.embeddings_name)
             word_emb_size = self.embeddings.embed_size
         else:
             self.embeddings = None
@@ -204,6 +205,10 @@ def __init__(
 
             wandb.define_metric("f1", summary="max")
 
+    def get_embedding(self, embedding_name, use_cache=True):
+        """Return an Embeddings instance for the given name. Override to customize embedding loading."""
+        return Embeddings(embedding_name, resource_registry=self.registry, use_cache=use_cache)
+
     def train(
         self,
         x_train,
@@ -288,7 +293,9 @@ def train_(
             self.model_config.char_vocab_size = len(self.p.vocab_char)
             self.model_config.case_vocab_size = len(self.p.vocab_case)
 
-            self.model = get_model(self.model_config, self.p, len(self.p.vocab_tag), load_pretrained_weights=True)
+            self.model = get_model(
+                self.model_config, self.p, len(self.p.vocab_tag), load_pretrained_weights=True, registry=self.registry
+            )
 
         print_parameters(self.model_config, self.training_config)
         self.model.print_summary()
@@ -494,6 +501,7 @@ def eval_nfold(self, x_test, y_test, features=None):
                         ntags=len(self.p.vocab_tag),
                         load_pretrained_weights=False,
                         local_path=os.path.join(dir_path, self.model_config.model_name),
+                        registry=self.registry,
                     )
                     self.model.load(filepath=os.path.join(dir_path, self.model_config.model_name, weight_file))
                     the_model = self.model
@@ -800,9 +808,7 @@ def load(self, dir_path="data/models/sequenceLabelling/", weight_file=DEFAULT_WE
         if self.model_config.embeddings_name is not None:
             # load embeddings
             # Do not use cache in 'prediction/production' mode
-            self.embeddings = Embeddings(
-                self.model_config.embeddings_name, resource_registry=self.registry, use_cache=False
-            )
+            self.embeddings = self.get_embedding(self.model_config.embeddings_name, use_cache=False)
             self.model_config.word_embedding_size = self.embeddings.embed_size
         else:
             self.embeddings = None
@@ -815,6 +821,7 @@ def load(self, dir_path="data/models/sequenceLabelling/", weight_file=DEFAULT_WE
             ntags=len(self.p.vocab_tag),
             load_pretrained_weights=False,
             local_path=os.path.join(dir_path, self.model_config.model_name),
+            registry=self.registry,
         )
         print("load weights from", os.path.join(dir_path, self.model_config.model_name, weight_file))
         self.model.load(filepath=os.path.join(dir_path, self.model_config.model_name, weight_file))