Merge pull request #77 from cropsinsilico/model_copies

Model copies

Author: langmm

.github/workflows/test-install.yml

@@ -28,6 +28,7 @@ env:
   USEVIRTUALENV: 0
   BUILDDOCS: 0
   YGG_GHA_JOB_ID_SUFFIX: null
+  TMPDIR: /tmp/
 jobs:
   test_pip:
     name: Test (${{ matrix.python-version }}, ${{ matrix.os }}, ${{ matrix.install-method
@@ -202,8 +203,13 @@ jobs:
       if: matrix.os == 'macos-latest' && matrix.install-method == 'conda'
       run: ./utils/setup_old_mac_sdk.sh
     - name: (CONDA) Install dependencies & package from existing build
+      id: conda1
+      continue-on-error: true
       if: matrix.install-method == 'conda'
       run: python utils/setup_test_env.py install ${{ matrix.install-method }} --without-build
+    - name: (CONDA) Install dependencies & package from existing build (re-attempt)
+      if: ${{ steps.test2.outcome == 'failure' && matrix.install-method == 'conda'}}
+      run: python utils/setup_test_env.py install ${{ matrix.install-method }} --without-build
     - name: (MACOS,CONDA) Set allow_multiple_omp config
       if: matrix.install-method == 'conda' && matrix.os == 'macos-latest'
       run: yggconfig --allow-multiple-omp
@@ -226,6 +232,7 @@ jobs:
     - name: Re-run test failures (1st)
       if: ${{ steps.test1.outcome == 'failure' }}
       run: yggtest --ci --cov-append ${{ matrix.test-flags1 }} --last-failed --last-failed-no-failures=none
+        --debug
     - name: Run tests (2nd)
       if: matrix.test-flags2 != ''
       id: test2
@@ -234,16 +241,17 @@ jobs:
     - name: Re-run test failures (2nd)
       if: ${{ steps.test2.outcome == 'failure' }}
       run: yggtest --ci --cov-append ${{ matrix.test-flags2 }} --last-failed --last-failed-no-failures=none
+        --debug
     - name: Combine coverage reports
       continue-on-error: true
-      run: 'ls .*
+      run: '# ls .*
 
         coverage combine --append
 
         '
     - name: (CONDA) Generate XML version of coverage report
       if: matrix.install-method == 'conda'
-      run: 'ls .*
+      run: '# ls .*
 
         coverage xml
 
@@ -461,8 +469,13 @@ jobs:
       if: matrix.os == 'macos-latest' && matrix.install-method == 'conda'
       run: ./utils/setup_old_mac_sdk.sh
     - name: (CONDA) Install dependencies & package from existing build
+      id: conda1
+      continue-on-error: true
       if: matrix.install-method == 'conda'
       run: python utils/setup_test_env.py install ${{ matrix.install-method }} --without-build
+    - name: (CONDA) Install dependencies & package from existing build (re-attempt)
+      if: ${{ steps.test2.outcome == 'failure' && matrix.install-method == 'conda'}}
+      run: python utils/setup_test_env.py install ${{ matrix.install-method }} --without-build
     - name: (MACOS,CONDA) Set allow_multiple_omp config
       if: matrix.install-method == 'conda' && matrix.os == 'macos-latest'
       run: yggconfig --allow-multiple-omp
@@ -485,6 +498,7 @@ jobs:
     - name: Re-run test failures (1st)
       if: ${{ steps.test1.outcome == 'failure' }}
       run: yggtest --ci --cov-append ${{ matrix.test-flags1 }} --last-failed --last-failed-no-failures=none
+        --debug
     - name: Run tests (2nd)
       if: matrix.test-flags2 != ''
       id: test2
@@ -493,16 +507,17 @@ jobs:
     - name: Re-run test failures (2nd)
       if: ${{ steps.test2.outcome == 'failure' }}
       run: yggtest --ci --cov-append ${{ matrix.test-flags2 }} --last-failed --last-failed-no-failures=none
+        --debug
     - name: Combine coverage reports
       continue-on-error: true
-      run: 'ls .*
+      run: '# ls .*
 
         coverage combine --append
 
         '
     - name: (CONDA) Generate XML version of coverage report
       if: matrix.install-method == 'conda'
-      run: 'ls .*
+      run: '# ls .*
 
         coverage xml
 
@@ -700,8 +715,13 @@ jobs:
       if: matrix.os == 'macos-latest' && matrix.install-method == 'conda'
       run: ./utils/setup_old_mac_sdk.sh
     - name: (CONDA) Install dependencies & package from existing build
+      id: conda1
+      continue-on-error: true
       if: matrix.install-method == 'conda'
       run: python utils/setup_test_env.py install ${{ matrix.install-method }} --without-build
+    - name: (CONDA) Install dependencies & package from existing build (re-attempt)
+      if: ${{ steps.test2.outcome == 'failure' && matrix.install-method == 'conda'}}
+      run: python utils/setup_test_env.py install ${{ matrix.install-method }} --without-build
     - name: (MACOS,CONDA) Set allow_multiple_omp config
       if: matrix.install-method == 'conda' && matrix.os == 'macos-latest'
       run: yggconfig --allow-multiple-omp
@@ -724,6 +744,7 @@ jobs:
     - name: Re-run test failures (1st)
       if: ${{ steps.test1.outcome == 'failure' }}
       run: yggtest --ci --cov-append ${{ matrix.test-flags1 }} --last-failed --last-failed-no-failures=none
+        --debug
     - name: Run tests (2nd)
       if: matrix.test-flags2 != ''
       id: test2
@@ -732,16 +753,17 @@ jobs:
     - name: Re-run test failures (2nd)
       if: ${{ steps.test2.outcome == 'failure' }}
       run: yggtest --ci --cov-append ${{ matrix.test-flags2 }} --last-failed --last-failed-no-failures=none
+        --debug
     - name: Combine coverage reports
       continue-on-error: true
-      run: 'ls .*
+      run: '# ls .*
 
         coverage combine --append
 
         '
     - name: (CONDA) Generate XML version of coverage report
       if: matrix.install-method == 'conda'
-      run: 'ls .*
+      run: '# ls .*
 
         coverage xml
 
@@ -941,8 +963,13 @@ jobs:
       if: matrix.os == 'macos-latest' && matrix.install-method == 'conda'
       run: ./utils/setup_old_mac_sdk.sh
     - name: (CONDA) Install dependencies & package from existing build
+      id: conda1
+      continue-on-error: true
       if: matrix.install-method == 'conda'
       run: python utils/setup_test_env.py install ${{ matrix.install-method }} --without-build
+    - name: (CONDA) Install dependencies & package from existing build (re-attempt)
+      if: ${{ steps.test2.outcome == 'failure' && matrix.install-method == 'conda'}}
+      run: python utils/setup_test_env.py install ${{ matrix.install-method }} --without-build
     - name: (MACOS,CONDA) Set allow_multiple_omp config
       if: matrix.install-method == 'conda' && matrix.os == 'macos-latest'
       run: yggconfig --allow-multiple-omp
@@ -965,6 +992,7 @@ jobs:
     - name: Re-run test failures (1st)
       if: ${{ steps.test1.outcome == 'failure' }}
       run: yggtest --ci --cov-append ${{ matrix.test-flags1 }} --last-failed --last-failed-no-failures=none
+        --debug
     - name: Run tests (2nd)
       if: matrix.test-flags2 != ''
       id: test2
@@ -973,16 +1001,17 @@ jobs:
     - name: Re-run test failures (2nd)
       if: ${{ steps.test2.outcome == 'failure' }}
       run: yggtest --ci --cov-append ${{ matrix.test-flags2 }} --last-failed --last-failed-no-failures=none
+        --debug
     - name: Combine coverage reports
       continue-on-error: true
-      run: 'ls .*
+      run: '# ls .*
 
         coverage combine --append
 
         '
     - name: (CONDA) Generate XML version of coverage report
       if: matrix.install-method == 'conda'
-      run: 'ls .*
+      run: '# ls .*
 
         coverage xml
 
@@ -1180,8 +1209,13 @@ jobs:
       if: matrix.os == 'macos-latest' && matrix.install-method == 'conda'
       run: ./utils/setup_old_mac_sdk.sh
     - name: (CONDA) Install dependencies & package from existing build
+      id: conda1
+      continue-on-error: true
       if: matrix.install-method == 'conda'
       run: python utils/setup_test_env.py install ${{ matrix.install-method }} --without-build
+    - name: (CONDA) Install dependencies & package from existing build (re-attempt)
+      if: ${{ steps.test2.outcome == 'failure' && matrix.install-method == 'conda'}}
+      run: python utils/setup_test_env.py install ${{ matrix.install-method }} --without-build
     - name: (MACOS,CONDA) Set allow_multiple_omp config
       if: matrix.install-method == 'conda' && matrix.os == 'macos-latest'
       run: yggconfig --allow-multiple-omp
@@ -1204,6 +1238,7 @@ jobs:
     - name: Re-run test failures (1st)
       if: ${{ steps.test1.outcome == 'failure' }}
       run: yggtest --ci --cov-append ${{ matrix.test-flags1 }} --last-failed --last-failed-no-failures=none
+        --debug
     - name: Run tests (2nd)
       if: matrix.test-flags2 != ''
       id: test2
@@ -1212,16 +1247,17 @@ jobs:
     - name: Re-run test failures (2nd)
       if: ${{ steps.test2.outcome == 'failure' }}
       run: yggtest --ci --cov-append ${{ matrix.test-flags2 }} --last-failed --last-failed-no-failures=none
+        --debug
     - name: Combine coverage reports
       continue-on-error: true
-      run: 'ls .*
+      run: '# ls .*
 
         coverage combine --append
 
         '
     - name: (CONDA) Generate XML version of coverage report
       if: matrix.install-method == 'conda'
-      run: 'ls .*
+      run: '# ls .*
 
         coverage xml
 

HISTORY.rst

@@ -2,7 +2,7 @@
 History
 =======
 
-1.5.0 (XXXX-XX-XX)
+1.6.0 (2021-04-14)
 ------------------
 
 * Made the asynchronous comm class more generic so it can be used to wrap any comm type and is more robust
@@ -24,7 +24,18 @@ History
 * Added new tests for transformations as part of comms and fixed bugs that those tests showed in how empty messages are transformed
 * Modify comm class such that the type is updated based on the transformed datatype when receiving *and* sending
 * Added a dedicated CommMessage class for wrapping messages with information about the message (e.g. header, work comms, status) and update comm & connection methods to expect this type
-* Fixed a bug that caused segfault when calling yggdrasil interface from inside a threaded model by introducing an 'allow_threading' parameter for models which sets a new parameter 'allow_multiple_comms' for comms associated with the model and causes the comm to be initialized such that multiple connections to the same address can be made (this is really just important for ZMQ comms and should only be invoked when using a server/client communication pattern).
+* Fixed a bug that caused segfault when calling yggdrasil interface from inside a threaded model by introducing an 'allow_threading' parameter for models which sets a new parameter 'allow_multiple_comms' for comms associated with the model and causes the comm to be initialized such that multiple connections to the same address can be made (this is really just important for ZMQ comms and should only be invoked when using a server/client communication pattern)
+* Allow multiple models to be run from a single YAML entry via the 'copies' model parameter.
+* Added DuplicatedModelDriver to handle model duplication via 'copies'
+* Added comm parameter 'dont_copy' to prevent duplication of comms (sharing) when a model is duplicated.
+* Updated ZMQProxy class so that server comms 'sign on' to the proxy by responding to a sign-on message that is sent continuously until a server signs on. Requests from clients received before the sign-on exchange are backlogged and sent after sign-on.
+* Updated ZMQComm to allow multiple connections during threading or when a model is duplicated.
+* Added rpc_lesson2b to demonstrate use of 'copies' parameter.
+* Updated the classes in the C interface to use bit flags
+* Updated documentation with information on using threads with yggdrasil and more advanced RPC features.
+* Refactored CommBase so that there are two components to send and receive calls and use the refactoring to cut down on repeat serialization in async comms and connection drivers.
+* Change fmt input parameter to YggAsciiArrayOutput Python interface to optional
+* Allow delimiter in YAML to override format_str provided via the interface for output serialization
 * Refactor CommBase so that there are two components to send and receive calls and use the refactoring to cut down on repeat serialization in async comms and connection drivers.
 
   When sending...
@@ -37,10 +48,6 @@ History
   1) recv_message, which receives the message and deserializes it, and
   2) finalize_message, which filters and transforms messages and performs actions associated with specific message types.
 
-TODO
-~~~~
-
-* Updated documentation with information on using threads with yggdrasil.
 
 1.5.0 (2021-02-10) Migrate to GHA, refactor CLI, & fix bugs
 ------------------

docs/source/examples/make_example_rst.py

@@ -15,7 +15,7 @@
               'all': 'Mixed',
               'all_nomatlab': 'Mixed w/o Matlab',
               'fortran': 'Fortran'}
-_default_lang = 'python'
+_default_lang = ['python', 'cpp', 'c', 'R', 'fortran', 'matlab']
 
 
 def get_file(fname, local=False):
@@ -75,13 +75,25 @@ def get_rel_path(fname, upone=False):
 
 
 def write_src_ref(fd, k):
-    write_src(fd, k, _default_lang, upone=True)
+    default = None
+    for x in _default_lang:
+        if x in source[k]:
+            default = x
+            break
+    assert(default)
+    write_src(fd, k, default, upone=True)
     fd.write('\n')
     write_ref_link(fd, k)
 
 
 def write_yml_ref(fd, k):
-    write_yml(fd, k, _default_lang, upone=True)
+    default = None
+    for x in _default_lang:
+        if x in source[k]:
+            default = x
+            break
+    assert(default)
+    write_yml(fd, k, default, upone=True)
     fd.write('\n')
     write_ref_link(fd, k)
 
@@ -178,6 +190,7 @@ def write_yml(fd, k, lang, upone=False):
 rst_examples.append('gs_lesson4b')  # Special case
 rst_examples += ['formatted_io%d' % x for x in range(1, 10)]
 rst_examples += ['rpc_lesson%d' % x for x in range(1, 4)]
+rst_examples += ['rpc_lesson%db' % x for x in range(2, 4)]
 rst_examples += ['model_function', 'conditional_io', 'transformed_io']
 rst_examples += ['timesync%d' % x for x in range(1, 3)]
 make_toc_file(rst_examples)

docs/source/index.rst

@@ -24,6 +24,7 @@ Welcome to yggdrasil's documentation!
    units
    c_format_strings
    debugging
+   threading
    examples/examples_toc
    advanced/index
    development/index

docs/source/server_client_io.rst

@@ -41,6 +41,26 @@ that the model will act as a client of the ``<server_model_name>`` model.
 
 .. include:: examples/rpc_lesson1_yml.rst
 
+In addition to the RPC API call, the example server also has an input ``params``.
+Models acting as servers can have as many inputs/outputs as desired in addition to
+the RPC connections. While the example input is not used to modify the output
+in this example, such a comm could be used to initialize a model with
+parameters and/or initial conditions.
+
+
+Using Existing Inputs/Outputs
+-----------------------------
+
+Models that have already been integrated via |yggdrasil| can also be turned
+into servers without modifying the code. Instead of passing a boolean to
+the ``is_server`` parameter, such models can provide a mapping with ``input``
+and ``output`` parameters that explicitly outline which of a existing model's 
+inputs/outputs should be used for the RPC call. Receive/send calls to named
+input/output channels will then behave as receive/send calls on a server 
+interface comm.
+
+.. todo:: Example source code and YAML of server replacing an existing input/output
+
 
 One Server, Two Clients
 -----------------------
@@ -62,4 +82,52 @@ source code.
 During runtime, request messages from both clients will be routed to the 
 server model which will process the requests in the order they are received.
 
-.. todo:: Section on having multiple servers.
+
+Two Servers, Two Clients
+------------------------
+
+There is also no limit on the number of copies of a server model that can be 
+used to responsd to RPC requests from the clients. In the example below, the 
+server and clients are the same as above, but 2 copies of the server model 
+are run as specified by the model ``copies`` parameter in the server YAML.
+
+.. include:: examples/rpc_lesson2b_yml.rst
+
+This allow client requests to be returned twice as fast, but precludes any 
+use of an internal state by the server model as there is no way for a client 
+to be sure that the same server model is responding to its requests and only
+its requests.
+
+
+Wrapped Function Server
+-----------------------
+
+Models that are created by letting |yggdrasil| automatically
+:ref:`wrap a function <autowrap_rst>` can also act as servers and/or clients.
+In the example below, the model acting as a server
+is a very simple function that takes a string as an input and returns the
+same string and the client is a function that takes a string as an input, 
+calls the server models with the input string and returns the response.
+
+When a client model is autowrapped from a function, additional care must be
+taken so that the client RPC comm can be reused during each call to the
+model. In interpreted models (Python, R, MATLAB), this is done by passing the
+keyword ``global_scope`` to the RPC client interface initialization function 
+(``YggRpcClient`` in Python). In compiled models (C, C++, Fortran), this is
+done by framing RPC client interface initialization calls with the
+``WITH_GLOBAL_SCOPE`` macro (see the language specific versions of this
+example for specifics).
+
+.. include:: examples/rpc_lesson3_src.rst
+
+The RPC connection between the server and the client is controlled by the
+same ``is_server`` and ``client_of`` YAML parameters as before.
+
+.. include:: examples/rpc_lesson3_yml.rst
+
+By default, all inputs to a wrapped server function will be used in
+the RPC call. However if only some of the inputs should be passed in by the
+RPC calls, they can be specified explicitly by providing the ``is_server``
+parameter with a map that contains ``input`` and ``output`` parameters that 
+map to the names of function input/output variables (as in the case of
+using existing input/output channels above).