Merge pull request #773 from metafacture/755-addMetafixExceptionLimit

Optionally specify limit for number of exceptions in Metafix.

Author: blackwinter

.gitignore

@@ -55,4 +55,4 @@ secring.gpg
 /metafix/src/test/resources/org/metafacture/metafix/integration/**/*.err
 /metafix/src/test/resources/org/metafacture/metafix/integration/**/*.out
 /metafix/src/test/resources/org/metafacture/metafix/integration/**/output-*
-!/metafix/src/test/resources/org/metafacture/metafix/integration/**/expected.err
+!/metafix/src/test/resources/org/metafacture/metafix/integration/**/expected.*

README.md

@@ -1459,6 +1459,46 @@ Executes the functions if/unless the string matches the regular expression patte
 
 [Java Code](https://github.com/metafacture/metafacture-core/blob/master/metafix/src/main/java/org/metafacture/metafix/conditional/StrMatch.java)
 
+## Expert settings
+
+Some settings are only applicable in very specific circumstances and may even be performance-sensitive. These settings cannot be configured via the usual Fix command or function options, but instead have to be set as [Java system properties](https://docs.oracle.com/javase/tutorial/essential/environment/sysprop.html).
+
+When using the _Metafacture runner_ via `flux.sh` or `flux.bat`, you can set these system properties on the command line:
+
+```bash
+FLUX_JAVA_OPTIONS=-D<property>=<value> ./flux.sh ...
+```
+
+Or enable them permanently in the `config/java-options.conf` configuration file:
+
+```bash
+-D<property>=<value>
+```
+
+When using _Metafacture as a Java library_, you can set these system properties on the command line:
+
+```bash
+JAVA_OPTIONS=-D<property>=<value> java ...
+```
+
+Or configure them directly in your application code (before instantiating Metafix):
+
+```java
+System.setProperty("<property>", "<value>");
+```
+
+### Maximum entity count
+
+The maximum number of Metafix entities being processed _per record_. Exceeding this limit leads to the remaining entities (as well as literals) being skipped for the current record; a log message is emitted at _DEBUG_ level for each excessive entity.
+
+Accepts a non-negative integer: `org.metafacture.metafix.maxEntityCount=<int>` (Default: `-1`, i.e. no limit)
+
+### Maximum exception count
+
+The maximum number of Metafix exceptions being handled by the strictness level _per transformation_ (i.e. Metafix instance). Exceeding this limit leads to the configured strictness handling being skipped for the rest of the transformation; a log message is emitted at _INFO_ level for each unhandled exception (including the type of the current exception).
+
+Accepts a non-negative integer: `org.metafacture.metafix.maxExceptionCount=<int>` (Default: `-1`, i.e. no limit)
+
 ## Xtext
 
 The Metafix projects have been originally set up with [Xtext](https://www.eclipse.org/Xtext/) 2.17.0 and Eclipse for Java 2019-03, following [https://www.eclipse.org/Xtext/documentation/104_jvmdomainmodel.html](https://www.eclipse.org/Xtext/documentation/104_jvmdomainmodel.html).

metafacture-runner/src/main/dist/config/log4j2.xml

@@ -10,7 +10,7 @@
   </Appenders>
   <Loggers>
     <!-- For developer logging change level to "DEBUG" -->
-    <Root level="ERROR">
+    <Root level="${sys:log4j2.level:-ERROR}">
       <AppenderRef ref="CONSOLE"/>
     </Root>
     <!-- For user logging with e.g. `log-object` change level to "DEBUG" -->

metafix/integrationTest.sh

@@ -12,6 +12,7 @@ todo_file=todo.txt
 
 input_glob=input.*
 expected_glob=expected.*
+expected_output_extension=out
 expected_errors_extension=err
 
 metafix_output_glob=output-metafix.*
@@ -207,6 +208,21 @@ function test_failed() {
   fi
 }
 
+function test_expected_patterns() {
+  local pattern
+
+  get_file "$1" "$4" "$5" || { log; return; }
+
+  while read -r pattern; do
+    if ! grep -qE "$pattern" "$5"; then
+      test_failed "$1" "$3" " (Pattern not found: $pattern)" FAILED "$6" "$7" "$8" "$9"
+      return
+    fi
+  done <"$2"
+
+  test_passed "$1" "$3" "${10}"
+}
+
 function run_tests() {
   local test matched=1\
     test_directory test_fix test_input test_expected test_todo\
@@ -248,7 +264,10 @@ function run_tests() {
       metafix_elapsed_time=$(elapsed_time "$metafix_start_time")
 
       if [ "$metafix_exit_status" -eq 0 ]; then
-        if get_file "$test" output "$test_directory"/$metafix_output_glob; then
+        if [ "${test_expected##*.}" == "$expected_output_extension" ]; then
+          test_expected_patterns "$test" "$test_expected" "$test_todo" output "$metafix_command_output"\
+            metafix "$metafix_exit_status" "$metafix_command_output" "$metafix_command_error" "$metafix_elapsed_time"
+        elif get_file "$test" output "$test_directory"/$metafix_output_glob; then
           metafix_output=$current_file
           metafix_diff="$test_directory/metafix.diff"
 
@@ -265,18 +284,8 @@ function run_tests() {
           command_info metafix "$metafix_exit_status" "$metafix_command_output" "$metafix_command_error"
         fi
       elif [ "${test_expected##*.}" == "$expected_errors_extension" ]; then
-        get_file "$test" error "$metafix_command_error" || { log; continue; }
-
-        while read -r pattern; do
-          if ! grep -qE "$pattern" "$metafix_command_error"; then
-            test_failed "$test" "$test_todo" " (Pattern not found: $pattern)" FAILED\
-              metafix "$metafix_exit_status" "$metafix_command_output" "$metafix_command_error"
-
-            continue 2
-          fi
-        done <"$test_expected"
-
-        test_passed "$test" "$test_todo" "$metafix_elapsed_time"
+        test_expected_patterns "$test" "$test_expected" "$test_todo" error "$metafix_command_error"\
+          metafix "$metafix_exit_status" "$metafix_command_output" "$metafix_command_error" "$metafix_elapsed_time"
       else
         test_failed "$test" "$test_todo" "$metafix_elapsed_time" ERROR\
           metafix "$metafix_exit_status" "$metafix_command_output" "$metafix_command_error"

metafix/src/main/java/org/metafacture/metafix/Metafix.java

@@ -66,6 +66,7 @@
 @Out(StreamReceiver.class)
 @FluxCommand("fix")
 public class Metafix implements StreamPipe<StreamReceiver>, Maps {
+
     public static final String ARRAY_MARKER = "[]";
     public static final String FIX_EXTENSION = ".fix";
     public static final String VAR_END = "]";
@@ -77,11 +78,14 @@ public class Metafix implements StreamPipe<StreamReceiver>, Maps {
     public static final Map<String, String> NO_VARS = Collections.emptyMap();
 
     public static final int MAX_ENTITY_COUNT = Integer.getInteger("org.metafacture.metafix.maxEntityCount", -1);
+    public static final int MAX_EXCEPTION_COUNT = Integer.getInteger("org.metafacture.metafix.maxExceptionCount", -1);
 
     private static final MetafactureLogger LOG = new MetafactureLogger(Metafix.class);
 
     private static final String ENTITIES_NOT_BALANCED = "Entity starts and ends are not balanced";
 
+    private static final String SKIP_EXCEPTION_PREFIX = "org.metafacture.";
+
     private final Deque<Integer> entityCountStack = new LinkedList<>();
     private final FixRegistry registry = new FixRegistry();
     private final List<Closeable> resources = new ArrayList<>();
@@ -98,12 +102,13 @@ public class Metafix implements StreamPipe<StreamReceiver>, Maps {
     private Record currentRecord = new Record();
     private StreamReceiver outputStreamReceiver;
     private Strictness strictness = DEFAULT_STRICTNESS;
+    private String entityMemberName = DEFAULT_ENTITY_MEMBER_NAME;
     private String fixFile;
     private String recordIdentifier;
-    private String entityMemberName = DEFAULT_ENTITY_MEMBER_NAME;
     private boolean repeatedFieldsToEntities;
     private boolean strictnessHandlesProcessExceptions;
     private int entityCount;
+    private int exceptionCount;
 
     /**
      * Creates an instance of {@link Metafix}.
@@ -641,6 +646,43 @@ private boolean maxEntityCountExceeded() {
         return MAX_ENTITY_COUNT >= 0 && entityCount > MAX_ENTITY_COUNT;
     }
 
+    /**
+     * Handles the exception based on the {@link #setStrictness selected strictness level},
+     * provided that the {@link MAX_EXCEPTION_COUNT exception limit} has not been exceeded.
+     *
+     * @param exception the exception to be handled
+     * @param record    the current record
+     */
+    public void handleException(final MetafactureException exception, final Record record) {
+        ++exceptionCount;
+
+        if (maxExceptionCountExceeded()) {
+            LOG.info("Maximum number of exceptions exceeded: {}/{} (Current exception: {})",
+                    exceptionCount, MAX_EXCEPTION_COUNT, getExceptionCause(exception).getSimpleName());
+        }
+        else {
+            strictness.handle(exception, record);
+        }
+    }
+
+    private boolean maxExceptionCountExceeded() {
+        return MAX_EXCEPTION_COUNT >= 0 && exceptionCount > MAX_EXCEPTION_COUNT;
+    }
+
+    private Class<?> getExceptionCause(final Throwable exception) {
+        final Class<?> clazz = exception.getClass();
+
+        if (clazz.getPackageName().startsWith(SKIP_EXCEPTION_PREFIX)) {
+            final Throwable cause = exception.getCause();
+
+            if (cause != null) {
+                return getExceptionCause(cause);
+            }
+        }
+
+        return clazz;
+    }
+
     public enum Strictness {
 
         /**
@@ -692,4 +734,5 @@ protected void log(final MetafactureException exception, final BiConsumer<String
         }
 
     }
+
 }

metafix/src/main/java/org/metafacture/metafix/RecordTransformer.java

@@ -135,7 +135,7 @@ public void transform(final Record record) {
             final MetafactureException exception = tryRun(() -> consumer.accept(record));
 
             if (exception != null) {
-                metafix.getStrictness().handle(exception, record);
+                metafix.handleException(exception, record);
             }
         });
     }

metafix/src/test/resources/org/metafacture/metafix/integration/script/fromJson/toJson/maxExceptionCount/expected.out

@@ -0,0 +1,3 @@
+^[0-9T:.-]+ \[main\] WARN  org\.metafacture\.metafix\.Metafix - Error while executing Fix expression \(at file:.*/metafix/src/test/resources/org/metafacture/metafix/integration/script/fromJson/toJson/maxExceptionCount/test\.fix, line 1\): upcase\("data"\)$
+^java\.lang\.IllegalStateException: Expected String, got Array$
+^[0-9T:.-]+ \[main\] INFO  org\.metafacture\.metafix\.Metafix - Maximum number of exceptions exceeded: 2/1 \(Current exception: IllegalStateException\)$

metafix/src/test/resources/org/metafacture/metafix/integration/script/fromJson/toJson/maxExceptionCount/input.json

@@ -0,0 +1,4 @@
+{"data":"foo"}
+{"data":"foo","data":"bar"}
+{"data":"foo","data":"baz"}
+{"data":"bar"}

metafix/src/test/resources/org/metafacture/metafix/integration/script/fromJson/toJson/maxExceptionCount/metafix.args

@@ -0,0 +1,2 @@
+-Dlog4j2.level=DEBUG
+-Dorg.metafacture.metafix.maxExceptionCount=1

metafix/src/test/resources/org/metafacture/metafix/integration/script/fromJson/toJson/maxExceptionCount/test.fix

@@ -0,0 +1 @@
+upcase("data")