config-example.yml

TASK_SCHEDULER:  # configures: dane_workflows.task_scheduler
  BATCH_SIZE: 5  # number of items returned by DataProvider.get_next_batch
  BATCH_LIMIT: -1 # limit of batches to process (-1 for no limit)
  MONITOR_FREQ: -1  # after each n batches call the STATUS_MONITOR
STATUS_HANDLER:  # recommended implementation; stores to local file
  TYPE: dane_workflows.status.SQLiteStatusHandler
  CONFIG:
    DB_FILE : ./proc_stats/all_stats.db  # Local file db
DATA_PROVIDER:  # configures: ExampleDataProvider
  TYPE: dane_workflows.data_provider.ExampleDataProvider
  CONFIG:
    SOURCE_BATCH_SIZE: 10
    DATA:  # usually DANE environments simply require an ID and content URL to work
      -
        id: video_1  # should refer to your source catalog ID
        url: https://your_video_file_1.mp4  # publicly accessible video
      -
        id: video_1
        url: https://your_video_files_2.mp4
PROC_ENV:  # to connect to a DANE environment, set TYPE to DANEEnvironment and provide a valid CONFIG
  TYPE: dane_workflows.data_processing.DANEEnvironment
  CONFIG:
    DANE_HOST: your-dane-host
    DANE_SERVER_PROTOCOL: https
    DANE_TASK_ID: DOWNLOAD  # tested BG_DOWNLOAD, which works; now testing ASR
    DANE_STATUS_DIR : ../dane_stats
    DANE_MONITOR_INTERVAL: 3  # seconds
    DANE_ES_HOST: your-dane-es-host
    DANE_ES_PORT: 1234
    DANE_ES_SCHEME: https  # default
    DANE_ES_USER: elastic  # optional
    DANE_ES_PW: your-password  # optional
    DANE_ES_INDEX: dane-index-your-env
    DANE_ES_QUERY_TIMEOUT: 20 #seconds?
    DANE_BATCH_PREFIX: your_test  # used to track different batches
EXPORTER:  # implement your own Exporter by subclassing from Exporter
  TYPE: dane_workflows.exporter.ExampleExporter
STATUS_MONITOR:  # optional, for monitoring
  TYPE: dane_workflows.status_monitor.SlackStatusMonitor
  CONFIG:
    TOKEN: your-token-here
    CHANNEL: your-slack-channel-here
    WORKFLOW_NAME: workflow-name-here
    INCLUDE_EXTRA_INFO: true