template-targets.R

# Targets pipeline template for converting SAS registers to Parquet.
# Generated by running `fastreg::use_template()`.
#
# Setup:
#
# 1. Set `sas_paths` and `output_dir` under "Configuration" below.
# 2. Run `targets::tar_make()` (in the same directory) to convert
#    registers to Parquet.
#
# Note: The main benefit of targets is parallel conversion across workers.
#
# For more information on targets, see https://books.ropensci.org/targets/

library(targets)
library(tarchetypes)

# Configuration ----------------------------------------------------------------

config <- list(
  # Paths to SAS files
  sas_paths = fastreg::list_sas_files("/path/to/sas/directory"),
  # Path to output Parquet files in. Parquet files will be located in
  # subdirectories of this directory.
  output_dir = "/path/to/output/directory"
)

# Target options ---------------------------------------------------------------

tar_option_set(
  packages = c("fs", "fastreg"),
  format = "qs",
  # Set controller with max 10 workers run as local R processes, launching
  # when there's work to do and exiting after 60 seconds if there's no task to
  # run.
  # NOTE: 10 workers might be too many for some systems.
  controller = crew::crew_controller_local(
    workers = 10,
    seconds_idle = 60
  ),
  # Delegate data management to the parallel crew workers.
  storage = "worker",
  retrieval = "worker",

  # Remove data from the R environment as soon as it's no longer needed. But
  # computer memory is not freed until garbage collection is run.
  memory = "transient",
  # Run gc() every 10th active target, both locally and on each parallel worker.
  garbage_collection = 10
)

# Pipeline ---------------------------------------------------------------------

list(
  tar_target(
    name = sas_paths,
    command = config$sas_paths
  ),

  # Stop the pipeline if the output directory isn't empty to avoid duplicating
  # data from a previous conversion. Create it, if it doesn't exist.
  tar_target(
    name = output_dir,
    command = {
      if (
        fs::dir_exists(config$output_dir) &&
          !fs::is_dir_empty(config$output_dir)
      ) {
        stop(
          "Output directory is not empty: ",
          config$output_dir,
          ". Remove it or choose a different `output_dir` before running the pipeline.",
          call. = FALSE
        )
      }
      fs::dir_create(config$output_dir)
      config$output_dir
    },
    deployment = "main",
    # Always check the output directory before proceeding.
    cue = tar_cue(mode = "always")
  ),

  # Convert each SAS file in parallel.
  tar_target(
    name = conversion_log,
    command = convert(path = sas_paths, output_dir = output_dir),
    pattern = map(sas_paths),
    # mode = "always" is required because the target `output_dir` returns the
    # same path string on every run. Otherwise, targets would consider this
    # target up-to-date and skip it despite the output directory having been
    # cleaned.
    cue = tar_cue(mode = "always")
  ),
  tar_quarto(
    name = log,
    path = "conversion-log.qmd",
    output_file = glue::glue(
      "conversion-log-",
      format(Sys.time(), "%d%m%y-%H%M%S"),
      ".pdf"
    )
  )
)