Upgrade to Hive 3.1.3x and several fixes

runSuite.pl

@@ -25,7 +25,7 @@
 } # end if
 my @queries = glob '*.sql';
 
-my $db = { 
+my $db = {
 	'tpcds' => "tpcds_bin_partitioned_orc_$scale",
 	'tpch' => "tpch_flat_orc_$scale"
 };
@@ -36,7 +36,7 @@
 	my $cmd="echo 'use $db->{${suite}}; source $query;' | hive -i testbench.settings 2>&1  | tee $query.log";
 #	my $cmd="cat $query.log";
 	#print $cmd ; exit;
-	
+
 	my $hiveStart = time();
 
 	my @hiveoutput=`$cmd`;
@@ -45,13 +45,16 @@
 	my $hiveEnd = time();
 	my $hiveTime = $hiveEnd - $hiveStart;
 	foreach my $line ( @hiveoutput ) {
-		if( $line =~ /Time taken:\s+([\d\.]+)\s+seconds,\s+Fetched:\s+(\d+)\s+row/ ) {
-			print "$query,success,$hiveTime,$2\n"; 
-		} elsif( 
-			$line =~ /^FAILED: /
-			# || /Task failed!/ 
+		if( $line =~ /(([\d,])+|No) row(s)? selected \((([\d\.]+)\s+seconds)\)/ ) {
+			my $queryTime = $5;
+			my $numRows = $1 eq "No" ? 0 : $1;
+			$numRows =~ s/,//;
+			print "$query,success,$queryTime,$numRows\n";
+		} elsif(
+			$line =~ /^Error: /
+			# || /Task failed!/
 			) {
-			print "$query,failed,$hiveTime\n"; 
+			print "$query,failed,$hiveTime\n";
 		} # end if
 	} # end while
 } # end for
@@ -73,4 +76,3 @@ (;$)
 USAGE
 	exit 1;
 }
-

sample-queries-tpcds/testbench.settings

@@ -0,0 +1,2 @@
+-- avoid caching results, useful if the benchmark is run multiple times
+set hive.query.results.cache.enabled=false;

tpcds-gen/src/main/java/org/notmysock/tpcds/GenTable.java

@@ -75,7 +75,6 @@ public int run(String[] args) throws Exception {
                     dsuri.getPort(),dsuri.getPath(), 
                     dsuri.getQuery(),"dsdgen");
         Configuration conf = getConf();
-        conf.setInt("mapred.task.timeout",0);
         conf.setInt("mapreduce.task.timeout",0);
         conf.setBoolean("mapreduce.map.output.compress", true);
         conf.set("mapreduce.map.output.compress.codec", "org.apache.hadoop.io.compress.GzipCodec");

tpcds-setup.sh

@@ -124,4 +124,7 @@ make -j 1 -f $LOAD_FILE
 echo "Loading constraints"
 runcommand "$HIVE -f ddl-tpcds/bin_partitioned/add_constraints.sql --hivevar DB=${DATABASE}"
 
+echo "Computing statistics"
+runcommand "$HIVE -f ddl-tpch/bin_partitioned/analyze.sql --hivevar DB=${DATABASE}"
+
 echo "Data loaded into database ${DATABASE}."

tpch-gen/src/main/java/org/notmysock/tpch/GenTable.java

@@ -104,7 +104,6 @@ public int run(String[] args) throws Exception {
                     dsuri.getPort(),dsuri.getPath(), 
                     dsuri.getQuery(),"dbgen");
         Configuration conf = getConf();
-        conf.setInt("mapred.task.timeout",0);
         conf.setInt("mapreduce.task.timeout",0);
         DistributedCache.addCacheArchive(link, conf);
         Job job = new Job(conf, "GenTable+"+table+"_"+scale);