feat: train incrementally after the first round

Author: lfoppiano

scripts/train_all_models.sh

@@ -62,6 +62,23 @@ train_model() {
     echo ""
 }
 
+train_model_incremental() {
+    local model=$1
+    local architecture=$2
+    
+    echo "=========================================="
+    echo "Incremental training: $model"
+    echo "Architecture: $architecture"
+    echo "=========================================="
+    
+    srun $SRUN_OPTS $PYTHON_CMD $model train \
+        --architecture $architecture \
+        --incremental
+    
+    echo "Completed (incremental): $model with $architecture"
+    echo ""
+}
+
 # Main training loop
 echo "Starting training of all GROBID models..."
 echo "Total models: ${#MODELS[@]}"
@@ -78,3 +95,15 @@ done
 echo "=========================================="
 echo "All training runs completed!"
 echo "=========================================="
+echo ""
+echo "Starting incremental training phase..."
+
+for model in "${MODELS[@]}"; do
+    for arch in "${ARCHITECTURES[@]}"; do
+        train_model_incremental "$model" "$arch"
+    done
+done
+
+echo "==========================================="
+echo "All incremental training runs completed!"
+echo "==========================================="

scripts/train_distributed_all_models.sh

@@ -107,6 +107,30 @@ submit_job() {
     fi
 }
 
+submit_job_incremental() {
+    local model=$1
+    local architecture=$2
+    local experiment_id=$3
+
+    local job_name="train_inc_${model}_${architecture}"
+    local log_file="${LOG_DIR}/${job_name}_%j.log"
+
+    echo ">>> Submitting incremental experiment $experiment_id: $job_name"
+
+    job_id=$(sbatch $SBATCH_OPTS \
+        --job-name="$job_name" \
+        --output="$log_file" \
+        --error="$log_file" \
+        --wrap="$PYTHON_CMD $model train --architecture $architecture --incremental" 2>&1 | grep -oP '\d+')
+
+    if [[ -n "$job_id" ]]; then
+        JOB_IDS+=("$job_id")
+        echo "    Submitted incremental job ID: $job_id"
+    else
+        echo "    Warning: Failed to submit incremental job for $job_name"
+    fi
+}
+
 # Calculate total number of experiments
 total_experiments=$((${#MODELS[@]} * ${#ARCHITECTURES[@]}))
 
@@ -147,4 +171,28 @@ if [[ "${WAIT_FOR_COMPLETION:-false}" == "true" ]]; then
         done
     done
     echo "All jobs completed!"
+
+    echo ""
+    echo "==========================================="
+    echo "Starting incremental training phase"
+    echo "==========================================="
+    echo ""
+
+    experiment_count=0
+
+    for model in "${MODELS[@]}"; do
+        for arch in "${ARCHITECTURES[@]}"; do
+            experiment_count=$((experiment_count + 1))
+            wait_for_capacity
+            submit_job_incremental "$model" "$arch" "$experiment_count"
+        done
+    done
+
+    echo ""
+    echo "==========================================="
+    echo "All $total_experiments incremental experiments submitted!"
+    echo "Job IDs: ${JOB_IDS[*]}"
+    echo "Monitor with: squeue -u \$USER"
+    echo "Logs in: $LOG_DIR"
+    echo "==========================================="
 fi