draft of sliding validation

[krsnapaudel]

NOTES: The test set is different per dataset when comparing with the ML Baseline from Paudel et al, 2021.
The sliding validation is not implemented for hyperparameter tuning. It can be implemented using the code from cypml:

# Returns an array containings tuples (train_idxs, test_idxs) for each fold
  # NOTE Y_train should include IDREGION, FYEAR as first two columns.
  def customKFoldValidationSplit(self, Y_train_full, num_folds, log_fh=None):
    """
    Custom K-fold Validation Splits:
    When using yield trend, we cannot do k-fold cross-validation. The custom
    K-Fold validation splits data in time-ordered fashion. The test data
    always comes after the training data.
    """
    all_years = sorted(np.unique(Y_train_full[:, 1]))
    num_years = len(all_years)
    num_test_years = 1
    num_train_years = num_years - num_test_years * num_folds

    custom_cv = []
    custom_split_info = '\nCustom sliding validation train, test splits'
    custom_split_info += '\n----------------------------------------------'

    for k in range(num_folds):
      train_years_start = k * num_test_years
      test_years_start = train_years_start + num_train_years
      train_years = all_years[train_years_start:test_years_start]
      test_years = all_years[test_years_start:test_years_start + num_test_years]
      test_indexes = np.ravel(np.nonzero(np.isin(Y_train_full[:, 1], test_years)))
      train_indexes = np.ravel(np.nonzero(np.isin(Y_train_full[:, 1], train_years)))
      custom_cv.append(tuple((train_indexes, test_indexes)))

      train_years = [str(y) for y in train_years]
      test_years = [str(y) for y in test_years]
      custom_split_info += '\nValidation set ' + str(k + 1) + ' training years: ' + ', '.join(train_years)
      custom_split_info += '\nValidation set ' + str(k + 1) + ' test years: ' + ', '.join(test_years)

    custom_split_info += '\n'
    if (log_fh is not None):
      log_fh.write(custom_split_info)

    if (self.verbose > 1):
      print(custom_split_info)

    return custom_cv