Update DSE version to 6.9.19

auditlog_test.py

@@ -18,6 +18,13 @@
 
 @since('4.0')
 class TestAuditlog(Tester):
+
+    @pytest.fixture(autouse=True)
+    def fixture_add_additional_log_patterns(self, fixture_dtest_setup):
+        fixture_dtest_setup.ignore_log_patterns = (
+                r'deprecated RollCycle',
+        )
+
     def test_archiving(self):
         cluster = self.cluster
         log_dir = tempfile.mkdtemp('logs')

auth_test.py

@@ -578,10 +578,16 @@ def test_materialized_views_auth(self):
         * Create a new user, 'cathy', with no permissions
         * Create a ks, table
         * Connect as cathy
+        *
         * Try CREATE MV without ALTER permission on base table, assert throws Unauthorized
         * Grant cathy ALTER permissions, then CREATE MV successfully
+        *
+        * Try to MODIFY base without WRITE permission on base, assert throws Unauthorized
+        * Grant cathy WRITE permissions on base, and modify base successfully
+        *
         * Try to SELECT from the mv, assert throws Unauthorized
-        * Grant cathy SELECT permissions, and read from the MV successfully
+        * Grant cathy SELECT permissions on base, and read from the MV successfully
+        *
         * Revoke cathy's ALTER permissions, assert DROP MV throws Unauthorized
         * Restore cathy's ALTER permissions, DROP MV successfully
         """
@@ -602,12 +608,36 @@ def test_materialized_views_auth(self):
         cassandra.execute("GRANT ALTER ON ks.cf TO cathy")
         cathy.execute(create_mv)
 
-        # TRY SELECT MV without SELECT permission on base table
-        assert_unauthorized(cathy, "SELECT * FROM ks.mv1", "User cathy has no SELECT permission on <table ks.cf> or any of its parents")
+        # Try MODIFY base without WRITE permission on base
+        assert_unauthorized(cathy, "INSERT INTO ks.cf(id, value) VALUES(1, '1')", "User cathy has no MODIFY permission on <table ks.cf> or any of its parents")
 
-        # Grant SELECT permission and CREATE MV
-        cassandra.execute("GRANT SELECT ON ks.cf TO cathy")
-        cathy.execute("SELECT * FROM ks.mv1")
+        if self.cluster.version() >= LooseVersion('4.0'):
+            # From 4.0 onward, only base MODIFY permission is required to update base with MV
+            # Grant WRITE permission on Base
+            cassandra.execute("GRANT MODIFY ON ks.cf TO cathy")
+            cathy.execute("INSERT INTO ks.cf(id, value) VALUES(1, '1')")
+
+            # TRY SELECT MV without SELECT permission on base table
+            assert_unauthorized(cathy, "SELECT * FROM ks.cf", "User cathy has no SELECT permission on <table ks.cf> or any of its parents")
+            assert_unauthorized(cathy, "SELECT * FROM ks.mv1", "User cathy has no SELECT permission on <table ks.cf> or any of its parents")
+
+            # Grant SELECT permission
+            cassandra.execute("GRANT SELECT ON ks.cf TO cathy")
+            assert_one(cathy, "SELECT * FROM ks.cf", [1, '1'])
+            assert_one(cathy, "SELECT * FROM ks.mv1", ['1', 1])
+        else:
+            # Before 4.0, MODIFY on MV is required to insert to base
+            # Grant WRITE permission on Base
+            cassandra.execute("GRANT MODIFY ON ks.cf TO cathy")
+            assert_unauthorized(cathy, "INSERT INTO ks.cf(id, value) VALUES(1, '1')", "User cathy has no SELECT permission on <table ks.cf> or any of its parents")
+            cassandra.execute("GRANT SELECT ON ks.cf TO cathy")
+            assert_unauthorized(cathy, "INSERT INTO ks.cf(id, value) VALUES(1, '1')", "User cathy has no MODIFY permission on <table ks.mv1> or any of its parents")
+
+            # Grant WRITE permission on MV
+            cassandra.execute("GRANT MODIFY ON ks.mv1 TO cathy")
+            cathy.execute("INSERT INTO ks.cf(id, value) VALUES(1, '1')")
+            assert_one(cathy, "SELECT * FROM ks.cf", [1, '1'])
+            assert_one(cathy, "SELECT * FROM ks.mv1", ['1', 1])
 
         # Revoke ALTER permission and try DROP MV
         cassandra.execute("REVOKE ALTER ON ks.cf FROM cathy")
@@ -1199,7 +1229,7 @@ def fixture_dtest_setup_overrides(self, dtest_config):
         @jira_ticket CASSANDRA-7653
         """
         dtest_setup_overrides = DTestSetupOverrides()
-        if '3.0' <= dtest_config.cassandra_version_from_build < '4.2':
+        if '3.0' <= dtest_config.cassandra_version_from_build < '4.0':
             dtest_setup_overrides.cluster_options = ImmutableMapping({'enable_user_defined_functions': 'true',
                                                                       'enable_scripted_user_defined_functions': 'true'})
         else:
@@ -1371,7 +1401,7 @@ def test_creator_of_db_resource_granted_all_permissions(self):
         as_mike.execute("CREATE KEYSPACE ks WITH replication = {'class':'SimpleStrategy', 'replication_factor':1}")
         as_mike.execute("CREATE TABLE ks.cf (id int primary key, val int)")
         as_mike.execute("CREATE ROLE role1 WITH PASSWORD = '11111' AND SUPERUSER = false AND LOGIN = true")
-        if self.cluster.version() < LooseVersion('4.2'):
+        if self.cluster.version() < LooseVersion('4.0'):
             as_mike.execute("""CREATE FUNCTION ks.state_function_1(a int, b int)
                             CALLED ON NULL INPUT
                             RETURNS int
@@ -1668,7 +1698,7 @@ def test_filter_granted_permissions_by_resource_type(self):
         self.superuser.execute("CREATE TABLE ks.cf (id int primary key, val int)")
         self.superuser.execute("CREATE ROLE mike WITH PASSWORD = '12345' AND SUPERUSER = false AND LOGIN = true")
         self.superuser.execute("CREATE ROLE role1 WITH SUPERUSER = false AND LOGIN = false")
-        if self.cluster.version() < LooseVersion('4.2'):
+        if self.cluster.version() < LooseVersion('4.0'):
             self.superuser.execute("CREATE FUNCTION ks.state_func(a int, b int) CALLED ON NULL INPUT RETURNS int LANGUAGE javascript AS 'a+b'")
         else:
             self.superuser.execute("CREATE FUNCTION ks.state_func(a int, b int) CALLED ON NULL INPUT RETURNS int LANGUAGE java AS ' return a+b;'")
@@ -2150,7 +2180,7 @@ def test_grant_revoke_udf_permissions(self):
         """
         self.setup_table()
         self.superuser.execute("CREATE ROLE mike WITH PASSWORD = '12345' AND LOGIN = true")
-        if self.cluster.version() < LooseVersion('4.2'):
+        if self.cluster.version() < LooseVersion('4.0'):
             self.superuser.execute("CREATE FUNCTION ks.plus_one ( input int ) CALLED ON NULL INPUT RETURNS int LANGUAGE javascript AS 'input + 1'")
             self.superuser.execute("CREATE FUNCTION ks.\"plusOne\" ( input int ) CALLED ON NULL INPUT RETURNS int LANGUAGE javascript AS 'input + 1'")
         else:
@@ -2199,7 +2229,7 @@ def test_grant_revoke_are_idempotent(self):
         """
         self.setup_table()
         self.superuser.execute("CREATE ROLE mike")
-        if self.cluster.version() < LooseVersion('4.2'):
+        if self.cluster.version() < LooseVersion('4.0'):
             self.superuser.execute("CREATE FUNCTION ks.plus_one ( input int ) CALLED ON NULL INPUT RETURNS int LANGUAGE javascript AS 'input + 1'")
         else:
             self.superuser.execute("CREATE FUNCTION ks.plus_one ( input int ) CALLED ON NULL INPUT RETURNS int LANGUAGE java AS 'return input + 1;'")
@@ -2230,7 +2260,7 @@ def test_function_resource_hierarchy_permissions(self):
         self.superuser.execute("INSERT INTO ks.t1 (k,v) values (1,1)")
         self.superuser.execute("CREATE ROLE mike WITH PASSWORD = '12345' AND LOGIN = true")
         self.superuser.execute("GRANT SELECT ON ks.t1 TO mike")
-        if self.cluster.version() < LooseVersion('4.2'):
+        if self.cluster.version() < LooseVersion('4.0'):
             self.superuser.execute("CREATE FUNCTION ks.func_one ( input int ) CALLED ON NULL INPUT RETURNS int LANGUAGE javascript AS 'input + 1'")
             self.superuser.execute("CREATE FUNCTION ks.func_two ( input int ) CALLED ON NULL INPUT RETURNS int LANGUAGE javascript AS 'input + 1'")
         else:
@@ -2289,15 +2319,15 @@ def test_udf_permissions_validation(self):
         * Verify mike can create a new UDF iff he has the CREATE permission
         """
         self.setup_table()
-        if self.cluster.version() < LooseVersion('4.2'):
+        if self.cluster.version() < LooseVersion('4.0'):
             self.superuser.execute("CREATE FUNCTION ks.plus_one ( input int ) CALLED ON NULL INPUT RETURNS int LANGUAGE javascript AS 'input + 1'")
         else:
             self.superuser.execute("CREATE FUNCTION ks.plus_one ( input int ) CALLED ON NULL INPUT RETURNS int LANGUAGE java AS 'return input + 1;'")
         self.superuser.execute("CREATE ROLE mike WITH PASSWORD = '12345' AND LOGIN = true")
         as_mike = self.get_session(user='mike', password='12345')
 
         # can't replace an existing function without ALTER permission on the parent ks
-        if self.cluster.version() < LooseVersion('4.2'):
+        if self.cluster.version() < LooseVersion('4.0'):
             cql = "CREATE OR REPLACE FUNCTION ks.plus_one( input int ) CALLED ON NULL INPUT RETURNS int LANGUAGE javascript as '1 + input'"
         else:
             cql = "CREATE OR REPLACE FUNCTION ks.plus_one( input int ) CALLED ON NULL INPUT RETURNS int LANGUAGE java as 'return 1 + input;'"
@@ -2340,7 +2370,7 @@ def test_udf_permissions_validation(self):
                        InvalidRequest)
 
         # can't create a new function without CREATE on the parent keyspace's collection of functions
-        if self.cluster.version() < LooseVersion('4.2'):
+        if self.cluster.version() < LooseVersion('4.0'):
             cql = "CREATE FUNCTION ks.plus_one ( input int ) CALLED ON NULL INPUT RETURNS int LANGUAGE javascript AS 'input + 1'"
         else:
             cql = "CREATE FUNCTION ks.plus_one ( input int ) CALLED ON NULL INPUT RETURNS int LANGUAGE java AS 'return input + 1;'"
@@ -2360,7 +2390,7 @@ def test_drop_role_cleans_up_udf_permissions(self):
         """
         self.setup_table()
         self.superuser.execute("CREATE ROLE mike WITH PASSWORD = '12345' AND LOGIN = true")
-        if self.cluster.version() < LooseVersion('4.2'):
+        if self.cluster.version() < LooseVersion('4.0'):
             self.superuser.execute("CREATE FUNCTION ks.plus_one ( input int ) CALLED ON NULL INPUT RETURNS int LANGUAGE javascript AS 'input + 1'")
         else:
             self.superuser.execute("CREATE FUNCTION ks.plus_one ( input int ) CALLED ON NULL INPUT RETURNS int LANGUAGE java AS 'return input + 1;'")
@@ -2391,7 +2421,7 @@ def test_drop_function_and_keyspace_cleans_up_udf_permissions(self):
         """
         self.setup_table()
         self.superuser.execute("CREATE ROLE mike WITH PASSWORD = '12345' AND LOGIN = true")
-        if self.cluster.version() < LooseVersion('4.2'):
+        if self.cluster.version() < LooseVersion('4.0'):
             self.superuser.execute("CREATE FUNCTION ks.plus_one ( input int ) CALLED ON NULL INPUT RETURNS int LANGUAGE javascript AS 'input + 1'")
         else: 
             self.superuser.execute("CREATE FUNCTION ks.plus_one ( input int ) CALLED ON NULL INPUT RETURNS int LANGUAGE java AS 'return input + 1;'")
@@ -2422,7 +2452,7 @@ def test_udf_with_overloads_permissions(self):
         """
         self.setup_table()
         self.superuser.execute("CREATE ROLE mike WITH PASSWORD = '12345' AND LOGIN = true")
-        if self.cluster.version() < LooseVersion('4.2'):
+        if self.cluster.version() < LooseVersion('4.0'):
             self.superuser.execute("CREATE FUNCTION ks.plus_one ( input int ) CALLED ON NULL INPUT RETURNS int LANGUAGE javascript AS 'input + 1'")
             self.superuser.execute("CREATE FUNCTION ks.plus_one ( input double ) CALLED ON NULL INPUT RETURNS double LANGUAGE javascript AS 'input + 1'")
         else:
@@ -2469,7 +2499,7 @@ def test_drop_keyspace_cleans_up_function_level_permissions(self):
         """
         self.setup_table()
         self.superuser.execute("CREATE ROLE mike WITH PASSWORD = '12345' AND LOGIN = true")
-        if self.cluster.version() < LooseVersion('4.2'):
+        if self.cluster.version() < LooseVersion('4.0'):
             self.superuser.execute("CREATE FUNCTION ks.state_func (a int, b int) CALLED ON NULL INPUT RETURNS int LANGUAGE javascript AS 'a + b'")
         else:
             self.superuser.execute("CREATE FUNCTION ks.state_func (a int, b int) CALLED ON NULL INPUT RETURNS int LANGUAGE java AS 'return a + b;'")
@@ -2525,7 +2555,7 @@ def verify_udf_permissions(self, cql):
         @param cql The statement to verify. Should contain the UDF ks.plus_one
         """
         self.setup_table()
-        if self.cluster.version() < LooseVersion('4.2'):
+        if self.cluster.version() < LooseVersion('4.0'):
             self.superuser.execute("CREATE FUNCTION ks.plus_one ( input int ) CALLED ON NULL INPUT RETURNS int LANGUAGE javascript AS 'input + 1'")
         else:
             self.superuser.execute("CREATE FUNCTION ks.plus_one ( input int ) CALLED ON NULL INPUT RETURNS int LANGUAGE java AS 'return input + 1;'")
@@ -2549,7 +2579,7 @@ def test_inheritence_of_udf_permissions(self):
         self.setup_table()
         self.superuser.execute("CREATE ROLE function_user")
         self.superuser.execute("GRANT EXECUTE ON ALL FUNCTIONS IN KEYSPACE ks TO function_user")
-        if self.cluster.version() < LooseVersion('4.2'):
+        if self.cluster.version() < LooseVersion('4.0'):
             self.superuser.execute("CREATE FUNCTION ks.plus_one ( input int ) CALLED ON NULL INPUT RETURNS int LANGUAGE javascript AS 'input + 1'")
         else:
             self.superuser.execute("CREATE FUNCTION ks.plus_one ( input int ) CALLED ON NULL INPUT RETURNS int LANGUAGE java AS 'return input + 1;'")

bootstrap_test.py

@@ -735,13 +735,7 @@ def test_decommissioned_wiped_node_can_gossip_to_single_seed(self):
         node2.stop(wait_other_notice=False)
 
         # Wipe its data
-        for data_dir in node2.data_directories():
-            logger.debug("Deleting {}".format(data_dir))
-            shutil.rmtree(data_dir)
-
-        commitlog_dir = os.path.join(node2.get_path(), 'commitlogs')
-        logger.debug("Deleting {}".format(commitlog_dir))
-        shutil.rmtree(commitlog_dir)
+        self._cleanup(node2)
 
         # Now start it, it should be allowed to join
         mark = node2.mark_log()
@@ -895,6 +889,12 @@ def test_simultaneous_bootstrap(self):
                           " cannot bootstrap while cassandra.consistent.rangemovement is true"
 
         cluster = self.cluster
+        configuration_options = {
+            'request_timeout_in_ms': 120000,
+            'read_request_timeout_in_ms': 120000,
+            'range_request_timeout_in_ms': 120000
+        }
+        cluster.set_configuration_options(configuration_options)
         cluster.set_environment_variable('CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True')
         cluster.populate(1)
         cluster.start()
@@ -930,7 +930,10 @@ def test_simultaneous_bootstrap(self):
         # bugs like 9484, where count(*) fails at higher
         # data loads.
         for _ in range(5):
-            assert_one(session, "SELECT count(*) from keyspace1.standard1", [500000], cl=ConsistencyLevel.ONE)
+            logger.info("Querying: SELECT count(*) from keyspace1.standard1")
+            # Improve reliability for slower/loaded test systems by using larger client timeout
+            assert_one(session, "SELECT count(*) from keyspace1.standard1", [500000], cl=ConsistencyLevel.ONE, timeout=180)
+            logger.info("Querying completed")
 
     def test_cleanup(self):
         """
@@ -979,6 +982,11 @@ def _cleanup(self, node):
             logger.debug("Deleting {}".format(data_dir))
             shutil.rmtree(data_dir)
         shutil.rmtree(commitlog_dir)
+        metadata_dir = os.path.join(node.get_path(), 'metadata')
+        if os.path.exists(metadata_dir):
+            logger.debug("Deleting {}".format(metadata_dir))
+            shutil.rmtree(metadata_dir)
+
 
     @since('2.2')
     @pytest.mark.ported_to_in_jvm # see org.apache.cassandra.distributed.test.BootstrapBinaryDisabledTest
@@ -1027,7 +1035,7 @@ def test_bootstrap_binary_disabled(self):
 
         try:
             node2.nodetool('join')
-            pytest.fail('nodetool should have errored and failed to join ring')
+            pytest.fail(reason='nodetool should have errored and failed to join ring')
         except ToolError as t:
             assert "Cannot join the ring until bootstrap completes" in t.stdout
 
@@ -1052,7 +1060,7 @@ def test_bootstrap_binary_disabled(self):
 
         try:
             node3.nodetool('join')
-            pytest.fail('nodetool should have errored and failed to join ring')
+            pytest.fail(reason='nodetool should have errored and failed to join ring')
         except ToolError as t:
             assert "Cannot join the ring until bootstrap completes" in t.stdout
 
@@ -1076,6 +1084,56 @@ def nodetool_resume_command(cluster):
             nt_resume_cmd += ' -f'
         return nt_resume_cmd
 
+    @since('4.0')
+    @pytest.mark.no_vnodes
+    def test_simple_bootstrap_with_everywhere_strategy(self):
+        cluster = self.cluster
+        tokens = cluster.balanced_tokens(2)
+        cluster.set_configuration_options(values={'num_tokens': 1})
+
+        logger.debug("[node1, node2] tokens: %r" % (tokens,))
+
+        keys = 10000
+
+        # Create a single node cluster
+        cluster.populate(1)
+        node1 = cluster.nodelist()[0]
+        node1.set_configuration_options(values={'initial_token': tokens[0]})
+        cluster.start()
+
+        session = self.patient_cql_connection(node1)
+        create_ks(session, 'ks', 'EverywhereStrategy')
+        create_cf(session, 'cf', columns={'c1': 'text', 'c2': 'text'})
+
+        insert_statement = session.prepare("INSERT INTO ks.cf (key, c1, c2) VALUES (?, 'value1', 'value2')")
+        execute_concurrent_with_args(session, insert_statement, [['k%d' % k] for k in range(keys)])
+
+        node1.flush()
+        node1.compact()
+
+        # Reads inserted data all during the bootstrap process. We shouldn't
+        # get any error
+        query_c1c2(session, random.randint(0, keys - 1), ConsistencyLevel.ONE)
+        session.shutdown()
+
+        # Bootstrapping a new node in the current version
+        node2 = new_node(cluster)
+        node2.set_configuration_options(values={'initial_token': tokens[1]})
+        node2.start(wait_for_binary_proto=True)
+        node2.compact()
+
+        node1.cleanup()
+        logger.debug("node1 size for ks.cf after cleanup: %s" % float(data_size(node1,'ks','cf')))
+        node1.compact()
+        logger.debug("node1 size for ks.cf after compacting: %s" % float(data_size(node1,'ks','cf')))
+
+        logger.debug("node2 size for ks.cf after compacting: %s" % float(data_size(node2,'ks','cf')))
+
+        size1 = float(data_size(node1,'ks','cf'))
+        size2 = float(data_size(node2,'ks','cf'))
+        assert_almost_equal(size1, size2, error=0.3)
+
+        assert_bootstrap_state(self, node2, 'COMPLETED')
 
 class TestBootstrap(BootstrapTester):
     """

byteman/guardrails/disk_usage_full.btm

@@ -0,0 +1,8 @@
+RULE return FULL disk usage
+CLASS org.apache.cassandra.service.disk.usage.DiskUsageMonitor
+METHOD getState
+AT EXIT
+IF TRUE
+DO
+    return org.apache.cassandra.service.disk.usage.DiskUsageState.FULL;
+ENDRULE

byteman/guardrails/disk_usage_stuffed.btm

@@ -0,0 +1,8 @@
+RULE return STUFFED disk usage
+CLASS org.apache.cassandra.service.disk.usage.DiskUsageMonitor
+METHOD getState
+AT EXIT
+IF TRUE
+DO
+    return org.apache.cassandra.service.disk.usage.DiskUsageState.STUFFED;
+ENDRULE

byteman/merge_schema_failure_4x.btm

@@ -3,7 +3,7 @@
 #
 RULE inject node failure on merge schema exit
 CLASS org.apache.cassandra.schema.Schema
-METHOD merge
+METHOD mergeAndUpdateVersion
 AT EXIT
 # set flag to only run this rule once.
 IF TRUE