Skip to content

Commit 187dfae

Browse files
jan-niestadtjan-niestadt
committed
Allow overriding blacklab[-server].yaml settings in .override.yaml files.
You can e.g. have both a blacklab-server.yaml and a blacklab-server.override.yaml. Settings in the second file will overwrite settings from the first. No merging is done.
1 parent aaedfe7 commit 187dfae

7 files changed

Lines changed: 95 additions & 75 deletions

File tree

‎engine/src/main/java/nl/inl/blacklab/config/BlackLabConfig.java‎

Lines changed: 21 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import java.nio.charset.StandardCharsets;
99
import java.util.Arrays;
1010
import java.util.HashMap;
11+
import java.util.List;
1112
import java.util.Map;
1213

1314
import org.apache.commons.io.FileUtils;
@@ -24,11 +25,16 @@
2425
public class BlackLabConfig {
2526

2627
private static final Logger logger = LogManager.getLogger(BlackLabConfig.class);
28+
public static final List<String> CONFIG_EXTENSIONS = Arrays.asList("json", "yaml", "yml");
2729

28-
private static BlackLabConfig read(Reader reader, boolean isJson) throws InvalidConfiguration {
30+
private static BlackLabConfig read(Reader reader, Reader overrideReader, boolean isJson) throws InvalidConfiguration {
2931
try {
3032
ObjectMapper mapper = isJson ? Json.getJsonObjectMapper() : Json.getYamlObjectMapper();
31-
return mapper.readValue(reader, BlackLabConfig.class);
33+
BlackLabConfig config = mapper.readValue(reader, BlackLabConfig.class);
34+
if (overrideReader != null) {
35+
mapper.readerForUpdating(config).readValue(overrideReader, BlackLabConfig.class);
36+
}
37+
return config;
3238
} catch (IOException e) {
3339
throw new InvalidConfiguration("Invalid configuration (" + e.getMessage() + ")", e);
3440
}
@@ -47,29 +53,27 @@ private static BlackLabConfig read(Reader reader, boolean isJson) throws Invalid
4753
* an UnsupportedOperationException.
4854
*
4955
* @param file file to read
56+
* @param overrideFile override file to read (if any); will override settings from the main file
5057
* @return configuration configuration from file
5158
*/
52-
public static synchronized BlackLabConfig readConfigFile(File file) throws IOException {
59+
public static synchronized BlackLabConfig readConfigFile(File file, File overrideFile) throws IOException {
5360
if (file == null || !file.canRead())
5461
throw new FileNotFoundException("Configuration file " + file + " is unreadable.");
5562

56-
if (!FilenameUtils.isExtension(file.getName(), Arrays.asList("yaml", "yml", "json")))
63+
if (!FilenameUtils.isExtension(file.getName(), BlackLabConfig.CONFIG_EXTENSIONS))
5764
throw new InvalidConfiguration("Configuration file " + file + " is of an unsupported type.");
5865

5966
boolean isJson = file.getName().endsWith(".json");
60-
return readConfigFile(file.getCanonicalPath(), FileUtils.readFileToString(file, StandardCharsets.UTF_8), isJson);
61-
}
62-
63-
/**
64-
* Read config file.
65-
*
66-
* @param fileName config file name
67-
* @param fileContents contents of the config file
68-
* @param isJson if true, reads JSON. Otherwise, reads YAML.
69-
*/
70-
private static synchronized BlackLabConfig readConfigFile(String fileName, String fileContents, boolean isJson) throws InvalidConfiguration {
71-
logger.debug("Reading global BlackLab config");
72-
return BlackLabConfig.read(new StringReader(fileContents), isJson);
67+
String fileContents = FileUtils.readFileToString(file, StandardCharsets.UTF_8);
68+
String overrideContents = overrideFile == null ? null : FileUtils.readFileToString(overrideFile, StandardCharsets.UTF_8);
69+
70+
if (overrideContents != null)
71+
logger.debug("Reading global BlackLab config from {} with overrides from {}", fileContents, overrideContents);
72+
else
73+
logger.debug("Reading global BlackLab config from {}", fileContents);
74+
return BlackLabConfig.read(new StringReader(fileContents),
75+
overrideContents == null ? null : new StringReader(overrideContents),
76+
isJson);
7377
}
7478

7579
private int configVersion = 2;

‎engine/src/main/java/nl/inl/blacklab/plugins/PluginData.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import com.fasterxml.jackson.databind.ObjectMapper;
1515

1616
import nl.inl.blacklab.config.BLConfigPlugins;
17+
import nl.inl.blacklab.config.BlackLabConfig;
1718
import nl.inl.blacklab.exceptions.PluginException;
1819
import nl.inl.util.FileUtil;
1920
import nl.inl.util.Json;
@@ -65,7 +66,7 @@ private File findConfigFile() {
6566
List<File> dir = List.of(PluginManager.getPluginsDir());
6667
List<String> names = Arrays.asList(plugin.getId(), altId, plugin.getClass().getName(),
6768
plugin.getClass().getSimpleName());
68-
return FileUtil.findFile(dir, names, List.of("yaml", "yml", "json"));
69+
return FileUtil.findFile(dir, names, BlackLabConfig.CONFIG_EXTENSIONS);
6970
}
7071

7172
/**

‎engine/src/main/java/nl/inl/blacklab/search/BlackLab.java‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
import java.net.URL;
77
import java.net.URLConnection;
88
import java.util.ArrayList;
9-
import java.util.Arrays;
109
import java.util.Collections;
1110
import java.util.List;
1211
import java.util.jar.Attributes;
@@ -54,6 +53,12 @@ public final class BlackLab {
5453
public static final String MSG_DEFAULT_CONFIG_ALREADY_APPLIED = "Cannot set default configuration - " +
5554
" configuration has already been applied.";
5655

56+
/** Name for generic BlackLab config file for e.g. QueryTool, IndexTool, etc. */
57+
public static final String TOOL_CONFIG_FILE_NAME = "blacklab";
58+
59+
/** Suffix for an config override file name (so e.g. blacklab.override.yaml or blacklab-server.override.yaml) */
60+
public static final String OVERRIDE_FILE_SUFFIX = ".override";
61+
5762
/**
5863
* If client doesn't explicitly create a BlackLab instance, one will be instantiated
5964
* automatically.
@@ -350,8 +355,8 @@ public static void setCheckCurrentDirForConfig(boolean checkCurrentDirForConfig)
350355
public static synchronized File configDir() {
351356
if (configDir == null) {
352357
List<File> dirsToSearch = defaultConfigDirs();
353-
File file = FileUtil.findFile(dirsToSearch, List.of("blacklab", "blacklab-server"),
354-
List.of("yaml", "yml", "json"));
358+
File file = FileUtil.findFile(dirsToSearch, List.of(TOOL_CONFIG_FILE_NAME, "blacklab-server"),
359+
BlackLabConfig.CONFIG_EXTENSIONS);
355360
if (file == null) {
356361
logger.warn("None of the directories scanned (" + dirsToSearch + ") contained blacklab.yaml or " +
357362
"blacklab-server.yaml. Using default /etc/blacklab as config directory. " +
@@ -373,10 +378,11 @@ public static synchronized File configDir() {
373378
*/
374379
public static synchronized void setConfigFromFile() {
375380
List<File> dirsToSearch = Collections.singletonList(configDir());
376-
File file = FileUtil.findFile(dirsToSearch, "blacklab", Arrays.asList("yaml", "yml", "json"));
381+
File file = FileUtil.findFile(dirsToSearch, TOOL_CONFIG_FILE_NAME, BlackLabConfig.CONFIG_EXTENSIONS);
382+
File overrideFile = FileUtil.findFile(dirsToSearch, TOOL_CONFIG_FILE_NAME + OVERRIDE_FILE_SUFFIX, BlackLabConfig.CONFIG_EXTENSIONS);
377383
if (file != null) {
378384
try {
379-
setConfig(BlackLabConfig.readConfigFile(file), true);
385+
setConfig(BlackLabConfig.readConfigFile(file, overrideFile), true);
380386
configDir = file.getParentFile();
381387
} catch (IOException e) {
382388
logger.warn("Could not load default blacklab configuration file " + file + ": " + e.getMessage());

‎site/docs/server/050_configuration.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -570,3 +570,11 @@ The `log`, `search`, `indexing` and `plugins` sections can occur in `blacklab.ya
570570
If you're not sure, you probably don't need `blacklab.yaml`.
571571

572572
**NOTE:** if it exists, BlackLab Server will read `blacklab.yaml` first, and only then read `blacklab-server.yaml`. So settings in the latter can override settings in the former.
573+
574+
## Configuration override files
575+
576+
You may choose to have both `blacklab-server.yaml` and `blacklab-server.override.yaml` files. As you might expect, settings in the second file override the ones in the first file. No "merging" is done; objects in the second file just overwrite whatever object was there previously.
577+
578+
This may be useful if you run BlackLab using Docker, e.g. to keep a mostly generic `blacklab-server.yaml` inside the container, and bind mount `blacklab-server.override.yaml` with your customizations.
579+
580+
This override mechanism also works with `blacklab.yaml`.

‎solr/src/main/java/org/ivdnt/blacklab/solr/BlackLabSearchComponent.java‎

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@
2020
import org.apache.solr.handler.component.SearchComponent;
2121
import org.apache.solr.util.plugin.SolrCoreAware;
2222

23+
import com.fasterxml.jackson.databind.ObjectMapper;
24+
2325
import nl.inl.blacklab.Constants;
2426
import nl.inl.blacklab.instrumentation.RequestInstrumentationProvider;
2527
import nl.inl.blacklab.search.BlackLabIndex;
@@ -51,6 +53,7 @@
5153
import nl.inl.blacklab.server.util.WebserviceUtil;
5254
import nl.inl.blacklab.webservice.WebserviceOperation;
5355
import nl.inl.blacklab.webservice.WsParam;
56+
import nl.inl.util.Json;
5457

5558
public class BlackLabSearchComponent extends SearchComponent implements SolrCoreAware {
5659

@@ -100,16 +103,29 @@ private BLSConfig getConfig(SolrCore core) {
100103
boolean isJson = configFilePath.endsWith(".json");
101104
SolrResourceLoader resourceLoader = core.getResourceLoader();
102105
BLSConfig config = null;
103-
104-
106+
107+
// Read blacklab-webservice.yaml
108+
ObjectMapper mapper = isJson ? Json.getJsonObjectMapper() : Json.getYamlObjectMapper();
105109
if (resourceLoader.resourceLocation(configFilePath) != null) {
106110
try (InputStream is = resourceLoader.openResource(configFilePath)) {
107111
InputStreamReader reader = new InputStreamReader(is, StandardCharsets.UTF_8);
108-
config = BLSConfig.read(reader, isJson);
112+
config = mapper.readValue(reader, BLSConfig.class);
113+
} catch (IOException e) {
114+
// ignore, file doesn't exist, fallback to default.
115+
}
116+
}
117+
118+
// Read override file if there is one
119+
String overrideFilePath = configFilePath.replaceAll("\\.(json|yaml|yml)$", ".override.$1");
120+
if (!overrideFilePath.equals(configFilePath) && resourceLoader.resourceLocation(overrideFilePath) != null) {
121+
try (InputStream is = resourceLoader.openResource(overrideFilePath)) {
122+
InputStreamReader reader = new InputStreamReader(is, StandardCharsets.UTF_8);
123+
mapper.readerForUpdating(config).readValue(reader);
109124
} catch (IOException e) {
110125
// ignore, file doesn't exist, fallback to default.
111126
}
112-
}
127+
}
128+
113129
if (config == null) {
114130
config = new BLSConfig(); // Default config if no config file found
115131
logger.error("##### no BLS config file found at " + configFilePath);

‎wslib/src/main/java/nl/inl/blacklab/server/config/BLSConfig.java‎

Lines changed: 6 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,11 @@
11
package nl.inl.blacklab.server.config;
22

3-
import java.io.File;
43
import java.io.IOException;
54
import java.io.Reader;
65
import java.util.Collections;
76
import java.util.List;
87

98
import com.fasterxml.jackson.databind.ObjectMapper;
10-
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
119

1210
import nl.inl.blacklab.config.BLConfigIndexing;
1311
import nl.inl.blacklab.config.BLConfigLog;
@@ -19,19 +17,14 @@
1917

2018
public class BLSConfig {
2119

22-
public static BLSConfig read(File configFile) throws InvalidConfiguration {
23-
try {
24-
ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
25-
return mapper.readValue(configFile, BLSConfig.class);
26-
} catch (IOException e) {
27-
throw new InvalidConfiguration("Invalid configuration file: " + configFile + " (" + e.getMessage() + ")", e);
28-
}
29-
}
30-
31-
public static BLSConfig read(Reader reader, boolean isJson) throws InvalidConfiguration {
20+
public static BLSConfig read(Reader reader, Reader overrides, boolean isJson) throws InvalidConfiguration {
3221
try {
3322
ObjectMapper mapper = isJson ? Json.getJsonObjectMapper() : Json.getYamlObjectMapper();
34-
return mapper.readValue(reader, BLSConfig.class);
23+
BLSConfig blsConfig = mapper.readValue(reader, BLSConfig.class);
24+
if (overrides != null) {
25+
mapper.readerForUpdating(blsConfig).readValue(overrides);
26+
}
27+
return blsConfig;
3528
} catch (IOException e) {
3629
throw new InvalidConfiguration("Invalid configuration (" + e.getMessage() + ")", e);
3730
}

‎wslib/src/main/java/nl/inl/blacklab/server/config/ConfigFileReader.java‎

Lines changed: 27 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,18 @@
11
package nl.inl.blacklab.server.config;
22

33
import java.io.File;
4-
import java.io.FileNotFoundException;
54
import java.io.IOException;
5+
import java.io.Reader;
66
import java.io.StringReader;
77
import java.nio.charset.Charset;
88
import java.nio.charset.StandardCharsets;
9-
import java.util.Arrays;
109
import java.util.List;
1110

1211
import org.apache.commons.io.FileUtils;
1312
import org.apache.logging.log4j.LogManager;
1413
import org.apache.logging.log4j.Logger;
1514

16-
import nl.inl.blacklab.exceptions.InvalidConfiguration;
15+
import nl.inl.blacklab.config.BlackLabConfig;
1716
import nl.inl.blacklab.search.BlackLab;
1817
import nl.inl.blacklab.server.exceptions.ConfigurationException;
1918
import nl.inl.util.FileUtil;
@@ -24,48 +23,41 @@
2423
public class ConfigFileReader {
2524
private static final Logger logger = LogManager.getLogger(ConfigFileReader.class);
2625

27-
public static final Charset CONFIG_ENCODING = StandardCharsets.UTF_8;
28-
29-
private static final List<String> CONFIG_EXTENSIONS = Arrays.asList("json", "yaml", "yml");
26+
private static final Charset CONFIG_ENCODING = StandardCharsets.UTF_8;
3027

3128
public static BLSConfig getBlsConfig(String configFileName) throws ConfigurationException {
32-
File configFile = FileUtil.findFile(List.of(BlackLab.configDir()), configFileName, CONFIG_EXTENSIONS);
33-
if (configFile == null) {
29+
// Find config file
30+
File configFile = FileUtil.findFile(List.of(BlackLab.configDir()), configFileName, BlackLabConfig.CONFIG_EXTENSIONS);
31+
if (configFile == null)
3432
throw new ConfigurationException("Couldn't find blacklab-server.(json|yaml) in BlackLab config dir " +
3533
BlackLab.configDir() + " . See https://blacklab.ivdnt.org/server/configuration.html .");
36-
}
37-
if (!configFile.canRead()) {
34+
if (!configFile.canRead())
3835
throw new ConfigurationException("Config file found but not readable: " + configFileName);
39-
}
40-
ConfigFileReader cfr = new ConfigFileReader(configFile);
41-
return cfr.getConfig();
42-
}
43-
44-
private String configFileContents;
45-
46-
private boolean configFileIsJson;
47-
48-
public ConfigFileReader(File configFile) throws ConfigurationException {
49-
if (configFile == null || !configFile.canRead())
50-
throw new ConfigurationException("Config file not found or not readable: " + configFile);
51-
configFileIsJson = false;
52-
logger.debug("Reading configuration file " + configFile);
36+
boolean isJson = configFile.getName().endsWith(".json");
5337
try {
54-
configFileContents = FileUtils.readFileToString(configFile, CONFIG_ENCODING);
55-
} catch (FileNotFoundException e) {
56-
throw new ConfigurationException("Config file not found", e);
38+
// Find override file
39+
File overrideFile = FileUtil.findFile(List.of(BlackLab.configDir()),
40+
configFileName + BlackLab.OVERRIDE_FILE_SUFFIX, BlackLabConfig.CONFIG_EXTENSIONS);
41+
if (overrideFile != null && !overrideFile.canRead()) {
42+
throw new ConfigurationException("Override config file found but not readable: " + overrideFile);
43+
}
44+
if (overrideFile != null)
45+
logger.debug("Reading configuration file {} and override file {}", configFile, overrideFile);
46+
else
47+
logger.debug("Reading configuration file {}", configFile);
48+
Reader configReader = new StringReader(FileUtils.readFileToString(configFile, CONFIG_ENCODING));
49+
50+
Reader overrideReader = null;
51+
if (overrideFile != null) {
52+
String overrideFileContents = FileUtils.readFileToString(overrideFile, CONFIG_ENCODING);
53+
overrideReader = new StringReader(overrideFileContents);
54+
}
55+
return BLSConfig.read(configReader, overrideReader, isJson);
56+
5757
} catch (IOException e) {
5858
throw new ConfigurationException("Error reading config file: " + configFile, e);
5959
}
60-
configFileIsJson = configFile.getName().endsWith(".json");
61-
}
62-
63-
public boolean isJson() {
64-
return configFileIsJson;
65-
}
6660

67-
public BLSConfig getConfig() throws InvalidConfiguration {
68-
return BLSConfig.read(new StringReader(configFileContents), isJson());
6961
}
7062

7163
}

0 commit comments

Comments
 (0)