python bindings

.github/workflows/macos.yml

@@ -49,5 +49,16 @@ jobs:
         ./marian --version
         ./marian-decoder --version
         ./marian-scorer --version
-        ./spm_encode --version
         ls -hlv $(find . -maxdepth 1 -type f -perm +ugo+x \( -name "marian*" -o -name "spm*" \))
+        cd ..
+        rm -rf build/
+
+    - name: Install PyMarian
+      working-directory: src/python
+      run: |
+        python3 -m venv .venv && source .venv/bin/activate
+        python3 -m pip install -U setuptools wheel
+        python3 -m pip install pybind11 sentencepiece scikit-build
+        python3 setup.py install -j2 -- -G"Unix Makefiles"
+        cd ../../
+        python3 -c "from pymarian import Translator"

.github/workflows/ubuntu.yml

@@ -115,6 +115,7 @@ jobs:
           -DCOMPILE_CPU=${{ matrix.cpu }} \
           -DCOMPILE_CUDA=${{ matrix.gpu }} \
           -DCOMPILE_EXAMPLES=${{ matrix.examples }} \
+          -DUSE_TCMALLOC=OFF \
           -DCOMPILE_SERVER=on \
           -DCOMPILE_TESTS=${{ matrix.unit_tests }} \
           -DCUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda-${{ matrix.cuda }} \
@@ -139,7 +140,18 @@ jobs:
       run: |
         ./marian --version
         ./marian-decoder --version
-        ./marian-scorer --version
-        ./marian-server --version
-        ./spm_encode --version
         ls -hlv $(find . -maxdepth 1 -type f -executable \( -name "marian*" -o -name "spm*" \))
+
+
+    - name: Install PyMarian
+      working-directory: build
+      env:
+        CUDA_VERSION: ${{ matrix.cuda }}
+      run: |
+        python3 -m venv .venv && source .venv/bin/activate
+        python3 -m pip install -U setuptools wheel
+        python3 -m pip install pybind11 sentencepiece scikit-build
+        cd ../src/python
+        python3 setup.py build -j2 install
+        cd ../../
+        python3 -c "from pymarian import Translator"

.github/workflows/windows.yml

@@ -134,4 +134,20 @@ jobs:
         .\marian-decoder.exe --version
         .\marian-scorer.exe --version
         dir *.exe
+        cd ..
+        rd /s /q build
+      shell: cmd
+
+    - name: Install PyMarian
+      working-directory: src/python
+      run: |
+        python3 -m venv .venv
+        .venv\Scripts\activate.bat
+        python3 -m pip install -U setuptools wheel
+        python3 -m pip install pybind11 sentencepiece scikit-build
+        python3 setup.py install -j2
+        cd ..\..\
+        python3 -c "from pymarian import Translator"
+      env:
+        CUDA_VERSION: ${{ matrix.cuda }}
       shell: cmd

.gitignore

@@ -1,4 +1,3 @@
-# Config files from CMake
 src/common/project_version.h
 src/common/git_revision.h
 src/common/build_info.cpp
@@ -61,4 +60,10 @@ examples/mnist/*ubyte
 /vs/MarianDll.VC.VC.opendb
 
 .vs
-.vscode
+.vscode
+
+src/python/build/
+src/python/dist/
+src/python/_skbuild/
+src/python/pymarian.egg-info/
+src/python/bench/*.txt

CMakeLists.txt

@@ -29,6 +29,7 @@ option(USE_MKL "Compile with MKL support" ON)
 option(USE_MPI "Use MPI library" OFF)
 option(USE_NCCL "Use NCCL library" ON)
 option(USE_SENTENCEPIECE "Download and compile SentencePiece" ON)
+option(USE_TCMALLOC "Use TCMALLOC if available" ON)
 option(USE_STATIC_LIBS "Link statically against non-system libs" OFF)
 option(GENERATE_MARIAN_INSTALL_TARGETS "Generate Marian install targets (requires CMake 3.12+)" OFF)
 option(DETERMINISTIC "Try to make training results as deterministic as possible (e.g. for testing)" OFF)
@@ -105,7 +106,7 @@ if(MSVC)
   set(INTRINSICS "/arch:AVX2")
   # set(INTRINSICS "/arch:AVX512")
   # /bigobj is necessary for expression_operators.cpp. See https://stackoverflow.com/questions/15110580/penalty-of-the-msvs-compiler-flag-bigobj
-  set(CMAKE_CXX_FLAGS           "/EHsc /DWIN32 /D_WINDOWS /DUNICODE /D_UNICODE /D_CRT_NONSTDC_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /bigobj ${DISABLE_GLOBALLY}")
+  set(CMAKE_CXX_FLAGS           "/permissive- /EHsc /DWIN32 /D_WINDOWS /DUNICODE /D_UNICODE /D_CRT_NONSTDC_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /bigobj ${DISABLE_GLOBALLY}")
   set(CMAKE_CXX_FLAGS_RELEASE   "${CMAKE_CXX_FLAGS} /MT /O2 ${INTRINSICS} /Zi /MP /GL /DNDEBUG")
   set(CMAKE_CXX_FLAGS_DEBUG     "${CMAKE_CXX_FLAGS} /MTd /Od /Ob0 ${INTRINSICS} /RTC1 /Zi /D_DEBUG")
 

src/3rd_party/CMakeLists.txt

@@ -1,13 +1,17 @@
 
 include_directories(.)
 
-add_subdirectory(./yaml-cpp)
-add_subdirectory(./SQLiteCpp)
-add_subdirectory(./pathie-cpp)
-add_subdirectory(./zlib)
-add_subdirectory(./faiss)
+add_subdirectory(./yaml-cpp EXCLUDE_FROM_ALL)
+add_subdirectory(./SQLiteCpp EXCLUDE_FROM_ALL)
+add_subdirectory(./pathie-cpp EXCLUDE_FROM_ALL)
+add_subdirectory(./zlib EXCLUDE_FROM_ALL)
+add_subdirectory(./faiss EXCLUDE_FROM_ALL)
 include_directories(./faiss)
 
+# if(SKBUILD)
+#   add_subdirectory(./pybind11 EXCLUDE_FROM_ALL)
+# endif(SKBUILD)
+
 if(COMPILE_CPU)
   if(NOT GENERATE_MARIAN_INSTALL_TARGETS)
     set(INTGEMM_DONT_BUILD_TESTS ON CACHE BOOL "Disable intgemm tests")
@@ -42,7 +46,7 @@ if(USE_FBGEMM)
 
   set(FBGEMM_BUILD_TESTS OFF CACHE BOOL "Disable fbgemm tests")
   set(FBGEMM_BUILD_BENCHMARKS OFF CACHE BOOL "Disable fbgemm benchmark")
-  add_subdirectory(./fbgemm)
+  add_subdirectory(./fbgemm EXCLUDE_FROM_ALL)
 
   # asmjit (3rd-party submodule of fbgemm) sets -Wall -Wextra near the end of
   # the compile options, invalidating any -Wno-... flags that we may have set
@@ -72,15 +76,19 @@ if(USE_SENTENCEPIECE)
 
   # regardless of -DUSE_STATIC_LIBS setting always build sentencepiece statically
   set(SPM_ENABLE_SHARED OFF CACHE BOOL  "Builds shared libaries in addition to static libraries." FORCE)
-  set(SPM_ENABLE_TCMALLOC ON CACHE BOOL "Enable TCMalloc if available.")
-
-  if(USE_STATIC_LIBS)
-    set(SPM_TCMALLOC_STATIC ON CACHE BOOL "Link static library of TCMALLOC." FORCE)
-  else(USE_STATIC_LIBS)
-    set(SPM_TCMALLOC_STATIC OFF CACHE BOOL "Link static library of TCMALLOC.")
-  endif(USE_STATIC_LIBS)
 
-  add_subdirectory(./sentencepiece)
+  if(USE_TCMALLOC)
+    set(SPM_ENABLE_TCMALLOC ON CACHE BOOL "Enable TCMalloc if available.")
+    if(USE_STATIC_LIBS)
+      set(SPM_TCMALLOC_STATIC ON CACHE BOOL "Link static library of TCMALLOC." FORCE)
+    else(USE_STATIC_LIBS)
+      set(SPM_TCMALLOC_STATIC OFF CACHE BOOL "Link static library of TCMALLOC.")
+    endif(USE_STATIC_LIBS)
+  else(USE_TCMALLOC)
+    set(SPM_ENABLE_TCMALLOC OFF CACHE BOOL "Enable TCMalloc if available.")
+  endif(USE_TCMALLOC)
+
+  add_subdirectory(./sentencepiece EXCLUDE_FROM_ALL)
   include_directories(./sentencepiece)
 
   set_target_properties(spm_encode spm_decode spm_train spm_normalize spm_export_vocab

src/CMakeLists.txt

@@ -4,6 +4,9 @@ include_directories(.)
 include_directories(3rd_party)
 include_directories(3rd_party/SQLiteCpp/include)
 include_directories(3rd_party/sentencepiece)
+
+include_directories(3rd_party/pybind11/include)
+
 if(USE_SENTENCEPIECE)
   include_directories(3rd_party/sentencepiece/third_party/protobuf-lite)
 endif(USE_SENTENCEPIECE)
@@ -255,11 +258,11 @@ if (NOT COMPILE_LIBRARY_ONLY)
   endif(COMPILE_SERVER)
 
   foreach(exec ${EXECUTABLES})
-    target_link_libraries(${exec} marian)
-    if(CUDA_FOUND)
-      target_link_libraries(${exec} marian_cuda)
-    endif(CUDA_FOUND)
-    set_target_properties(${exec} PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}")
+  target_link_libraries(${exec} marian)
+  if(CUDA_FOUND)
+    target_link_libraries(${exec} marian_cuda)
+  endif(CUDA_FOUND)
+  set_target_properties(${exec} PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}")
   endforeach(exec)
 endif(NOT COMPILE_LIBRARY_ONLY)
 
@@ -277,9 +280,31 @@ endif(COMPILE_EXAMPLES)
 
 if(GENERATE_MARIAN_INSTALL_TARGETS)
   # Install the marian library if given a "make install" target
-  include(GNUInstallDirs)                 # This defines default values for installation directories (all platforms even if named GNU)
+  include(GNUInstallDirs) # This defines default values for installation directories (all platforms even if named GNU)
   install(TARGETS marian
     EXPORT marian-targets
     ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
   )
 endif(GENERATE_MARIAN_INSTALL_TARGETS)
+
+if(SKBUILD)
+  # Scikit-Build does not add your site-packages to the search path
+  # automatically, so we need to add it _or_ the pybind11 specific directory
+  # here.
+  execute_process(
+    COMMAND "${PYTHON_EXECUTABLE}" -c
+            "import pybind11; print(pybind11.get_cmake_dir())"
+    OUTPUT_VARIABLE _tmp_dir
+    OUTPUT_STRIP_TRAILING_WHITESPACE COMMAND_ECHO STDOUT)
+  list(APPEND CMAKE_PREFIX_PATH "${_tmp_dir}")
+
+  # Now we can find pybind11
+  find_package(pybind11 CONFIG REQUIRED)
+
+  pybind11_add_module(pymarian MODULE python/pymarian/bind.cpp)
+  target_link_libraries(pymarian PUBLIC marian)
+  if(CUDA_FOUND)
+    target_link_libraries(pymarian PUBLIC marian_cuda)
+  endif(CUDA_FOUND)
+  install(TARGETS pymarian DESTINATION .)
+endif(SKBUILD)

src/common/config.cpp

@@ -276,12 +276,22 @@ std::vector<DeviceId> Config::getDevices(Ptr<Options> options,
   return devices;
 }
 
-Ptr<Options>
-parseOptions(int argc, char** argv, cli::mode mode, bool validate){
+Ptr<Options> parseOptions(int argc, char** argv, cli::mode mode, bool validate) {
   ConfigParser cp(mode);
   return cp.parseOptions(argc, argv, validate);
 }
 
+Ptr<Options> parseOptions(const std::string& args, cli::mode mode, bool validate) {
+  std::vector<std::string> vArgs = utils::split(args, " ");
+
+  std::string dummy("dummy");
+  std::vector<char*> cArgs = { &dummy[0] };
+  for(auto& arg : vArgs)
+    cArgs.push_back(&arg[0]);
+
+  return parseOptions((int)cArgs.size(), cArgs.data(), mode, validate);
+}
+
 std::ostream& operator<<(std::ostream& out, const Config& config) {
   YAML::Emitter outYaml;
   cli::OutputYaml(config.get(), outYaml);

src/common/config.h

@@ -119,4 +119,19 @@ Ptr<Options> parseOptions(int argc,
                           cli::mode mode,
                           bool validate = true);
 
+/**
+ * Parse the command line options. 
+ * Same as above, but args provided as C++ string object, space-delimited. This is used for instance
+ * in the python bindings as a simple string-based interface.
+ *
+ * @param args space delimited command line options
+ * @param mode change the set of available command-line options, e.g. training, translation, etc.
+ * @param validate validate parsed options and abort on failure
+ *
+ * @return parsed options
+ */
+Ptr<Options> parseOptions(const std::string& args, 
+                          cli::mode mode,
+                          bool validate = true);
+
 }  // namespace marian

src/common/logging.cpp

@@ -26,11 +26,13 @@ std::shared_ptr<spdlog::logger> createStderrLogger(const std::string& name,
                                                    const std::string& pattern,
                                                    const std::vector<std::string>& files,
                                                    bool quiet) {
-  std::vector<spdlog::sink_ptr> sinks;
+  auto logger = spdlog::get(name);
+  if(!logger) {
+    std::vector<spdlog::sink_ptr> sinks;
 
-  auto stderr_sink = spdlog::sinks::stderr_sink_mt::instance();
-  if(!quiet)
-    sinks.push_back(stderr_sink);
+    auto stderr_sink = spdlog::sinks::stderr_sink_mt::instance();
+    if(!quiet)
+      sinks.push_back(stderr_sink);
 
   // @TODO: think how to solve this better than using OMPI_COMM_WORLD_RANK env variable
   // only create output files if we are the main process or if MPI rank is not defined
@@ -42,10 +44,11 @@ std::shared_ptr<spdlog::logger> createStderrLogger(const std::string& name,
     }
   }
 
-  auto logger = std::make_shared<spdlog::logger>(name, begin(sinks), end(sinks));
+    logger = std::make_shared<spdlog::logger>(name, begin(sinks), end(sinks));
 
-  spdlog::register_logger(logger);
-  logger->set_pattern(pattern);
+    spdlog::register_logger(logger);
+    logger->set_pattern(pattern);
+  }
   return logger;
 }
 
@@ -72,6 +75,7 @@ bool setLoggingLevel(spdlog::logger& logger, std::string const level) {
 }
 
 static void setErrorHandlers();
+
 void createLoggers(const marian::Config* config) {
   std::vector<std::string> generalLogs;
   std::vector<std::string> validLogs;

src/models/model_task.h

@@ -11,6 +11,7 @@ struct ModelTask {
 
 struct ModelServiceTask {
   virtual ~ModelServiceTask() {}
-  virtual std::string run(const std::string&) = 0;
+  virtual std::string run(const std::string&, const std::string&) = 0;
+  virtual std::vector<std::string> run(const std::vector<std::string>&, const std::string&) = 0;
 };
 }  // namespace marian

src/python/README.md

@@ -0,0 +1,20 @@
+Stuff required to build module (on Linux)
+```
+python3 -m venv ./venv
+source ./venv/bin/activate
+python -mpip install sentencepiece scikit-build pybind11
+```
+
+Stuff required to run windowed demo
+```
+python -mpip install pyqt5 sacremoses git+https://github.com/mediacloud/sentence-splitter
+```
+
+Build the module (CPU version)
+```
+python setup.py build -j16 install
+```
+
+```
+echo "Hello World." | python test_translate.py '--config /home/marcinjd/MTMA/decoder.yml' /home/marcinjd/MTMA/source.spm /home/marcinjd/MTMA/target.spm
+```

src/python/bench/README.md

@@ -0,0 +1,14 @@
+# 
+
+Install pymarian then
+
+```sh
+pip install torch --extra-index-url https://download.pytorch.org/whl/cu113
+pip install transformers parallelformers sentencepiece sacrebleu
+```
+
+To run the benchmark,
+
+```sh
+bash benchmark.sh
+```

src/python/bench/benchmark.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env sh
+
+function translate_native {
+ sacrebleu -t wmt20 -l en-de --echo src | \
+   spm_encode --model /home/marcinjd/MTMA/source.spm |  ../_skbuild/linux-x86_64-3.8/cmake-build/marian-decoder --quiet \
+     -c  /home/marcinjd/MTMA/decoder.yml -b4 --mini-batch 16 --maxi-batch 100 -d 0 1 2 3 \
+   | spm_decode --model  /home/marcinjd/MTMA/target.spm > translations_native.txt
+}
+
+function translate_pybind {
+   sacrebleu -t wmt20 -l en-de --echo src | \
+   spm_encode --model /home/marcinjd/MTMA/source.spm | \
+   python test_translate.py '--config /home/marcinjd/MTMA/decoder.yml -b4 --mini-batch 16 --quiet --maxi-batch 100 -d 0 1 2 3' \
+   | spm_decode --model  /home/marcinjd/MTMA/target.spm > translations_pybind.txt
+}
+
+function translate_hf_native {
+   sacrebleu -t wmt20 -l en-de --echo src | python test_hf_raw.py > translations_hf_raw.txt
+}
+
+echo -n "Native: "
+time translate_native
+echo
+echo -n "Pybind: "
+time translate_pybind
+echo
+echo -n "Transformers (native): "
+time translate_hf_native