Merge pull request #285 from tfenne/tf_fix_oom_from_signed_array_size_arithmetic

Fix signed integer overflow in phasing HMM array indexing

Author: srubinacci

phase/src/models/phasing_hmm.cpp

@@ -220,7 +220,7 @@ void phasing_hmm::backward() {
 		if (curr_segment_locus == 0) {
 			SUMK();
 			//phasingProb[curr_segment_index] = prob;
-			std::copy(prob.begin(), prob.end(), phasingProb.begin()+curr_segment_index*C->n_states*HAP_NUMBER);
+			std::copy(prob.begin(), prob.end(), phasingProb.begin()+(size_t)curr_segment_index*C->n_states*HAP_NUMBER);
 			//phasingProbSum[curr_segment_index] = probSumH;
 			std::copy(probSumH.begin(), probSumH.end(), phasingProbSum.begin()+curr_segment_index*HAP_NUMBER);
 			phasingProbSumSum[curr_segment_index] = probSumT;
@@ -229,7 +229,7 @@ void phasing_hmm::backward() {
 		//STORE PROBS FOR PHASING RARE HETS
 		if (VAR_TYP[curr_idx_locus] == VAR_FLAT_HET) {
 			//imputeProb[curr_missing_locus] = prob;
-			std::copy(prob.begin(), prob.end(), imputeProb.begin()+curr_missing_locus*C->n_states*HAP_NUMBER);
+			std::copy(prob.begin(), prob.end(), imputeProb.begin()+(size_t)curr_missing_locus*C->n_states*HAP_NUMBER);
 			//imputeProbSum[curr_missing_locus] = probSumH;
 			std::copy(probSumH.begin(), probSumH.end(), imputeProbSum.begin()+curr_missing_locus*HAP_NUMBER);
 			imputeProbSumSum[curr_missing_locus] = probSumT;

phase/src/models/phasing_hmm.h

@@ -289,7 +289,7 @@ void phasing_hmm::SUMK() {
 inline
 bool phasing_hmm::TRANS_HAP()
 {
-	const int states_haps = C->n_states*HAP_NUMBER;
+	const size_t states_haps = (size_t)C->n_states*HAP_NUMBER;
 	sumHProbs = 0.0f;
 	yt = C->getTransition(VAR_ABS[curr_idx_locus], VAR_ABS[curr_idx_locus+1]);
 	nt = 1.0f - yt;
@@ -327,7 +327,7 @@ bool phasing_hmm::SAMPLE_DIP() {
 inline
 void phasing_hmm::IMPUTE_FLAT_HET()
 {
-	const int states_haps = C->n_states*HAP_NUMBER;
+	const size_t states_haps = (size_t)C->n_states*HAP_NUMBER;
 	const __m256 _one = _mm256_set1_ps(1.0f);
 	const __m256 _zero = _mm256_set1_ps(0.0f);
 	__m256 _scaleR = _mm256_load_ps(&imputeProbSum[curr_missing_locus*HAP_NUMBER]);