chore: Migrate gsutil usage to gcloud storage

Author: gurusai-voleti

docs/advanced-topics/code_coverage.md

@@ -51,7 +51,7 @@ corpora by doing the following:
 * Check whether you have access to the corpus for your project:
 
 ```bash
-$ gsutil ls gs://${PROJECT_NAME}-corpus.clusterfuzz-external.appspot.com/
+$ gcloud storage ls gs://${PROJECT_NAME}-corpus.clusterfuzz-external.appspot.com/
 ```
 
 If you see an authorization error from the command above, run this:

docs/advanced-topics/corpora.md

@@ -53,12 +53,12 @@ Copy the corpus to a directory on your
 machine by running the following command:
 
 ```bash
-$ gsutil -m cp -r gs://<bucket_path> <local_directory>
+$ gcloud storage cp --recursive gs://<bucket_path> <local_directory>
 ```
 Using the expat example above, this would be:
 
 ```bash
-$ gsutil -m cp -r \
+$ gcloud storage cp --recursive \
     gs://expat-corpus.clusterfuzz-external.appspot.com/libFuzzer/expat_parse_fuzzer \
     <local_directory>
 ```
@@ -67,4 +67,4 @@ $ gsutil -m cp -r \
 
 We keep daily zipped backups of your corpora. These can be accessed from the
 **corpus_backup** column of the fuzzer statistics page. Downloading these can
-be significantly faster than running `gsutil -m cp -r` on the corpus bucket.
+be significantly faster than running `gcloud storage cp --recursive` on the corpus bucket.

infra/base-images/base-builder/jcc/build_jcc.bash

@@ -18,8 +18,8 @@
 
 go build jcc.go
 go build jcc2.go
-gsutil cp jcc gs://clusterfuzz-builds/jcc/clang++-jcc
-gsutil cp jcc gs://clusterfuzz-builds/jcc/clang-jcc
+gcloud storage cp jcc gs://clusterfuzz-builds/jcc/clang++-jcc
+gcloud storage cp jcc gs://clusterfuzz-builds/jcc/clang-jcc
 
-gsutil cp jcc2 gs://clusterfuzz-builds/jcc/clang++-jcc2
-gsutil cp jcc2 gs://clusterfuzz-builds/jcc/clang-jcc2
+gcloud storage cp jcc2 gs://clusterfuzz-builds/jcc/clang++-jcc2
+gcloud storage cp jcc2 gs://clusterfuzz-builds/jcc/clang-jcc2

infra/build/functions/project_experiment.py

@@ -82,11 +82,11 @@ def run_experiment(project_name, command, output_path, experiment_name):
           ]
       },
       {
-          'name': 'gcr.io/cloud-builders/gsutil',
+          'name': 'gcr.io/cloud-builders/gcloud',
           'args': [
-              '-m',
+              'storage',
               'cp',
-              '-r',
+              '--recursive',
               '/workspace/out/*',
               output_path,
           ]

infra/build/functions/target_experiment.py

@@ -110,21 +110,21 @@ def run_experiment(project_name,
   }
   steps.append(build_lib.dockerify_run_step(run_step, build))
   steps.append({
-      'name': 'gcr.io/cloud-builders/gsutil',
-      'args': ['-m', 'cp', local_output_path, output_path]
+      'name': 'gcr.io/cloud-builders/gcloud',
+      'args': ['storage', 'cp', local_output_path, output_path]
   })
 
   # Upload corpus.
   steps.append({
       'name':
-          'gcr.io/cloud-builders/gsutil',
+          'gcr.io/cloud-builders/gcloud',
       'entrypoint':
           '/bin/bash',
       'args': [
           '-c',
           (f'cd {local_corpus_path} && '
            f'zip -r {local_corpus_zip_path} * && '
-           f'gsutil -m cp {local_corpus_zip_path} {upload_corpus_path} || '
+           f'gcloud storage cp {local_corpus_zip_path} {upload_corpus_path} || '
            f'rm -f {local_corpus_zip_path}'),
       ],
   })
@@ -136,36 +136,36 @@ def run_experiment(project_name,
     # If multiple files are found, suffix them and upload them all.
     steps.append({
         'name':
-            'gcr.io/cloud-builders/gsutil',
+            'gcr.io/cloud-builders/gcloud',
         'entrypoint':
             '/bin/bash',
         'args': [
             '-c',
             (f'cp {default_target_path} {local_target_dir} 2>/dev/null || '
              f'find {build.out} -type f -name {target_name} -exec bash -c '
              f'\'cp "$0" "{local_target_dir}/$(echo "$0" | sed "s@/@_@g")"\' '
-             f'{{}} \\; && gsutil cp -r {local_target_dir} '
+             f'{{}} \\; && gcloud storage cp --recursive {local_target_dir} '
              f'{upload_reproducer_path}/target_binary || true'),
         ],
     })
 
     # Upload reproducer.
     steps.append({
         'name':
-            'gcr.io/cloud-builders/gsutil',
+            'gcr.io/cloud-builders/gcloud',
         'entrypoint':
             '/bin/bash',
         'args': [
             '-c',
-            (f'gsutil -m cp -r {local_artifact_path} {upload_reproducer_path} '
+            (f'gcloud storage cp --recursive {local_artifact_path} {upload_reproducer_path} '
              '|| true'),
         ],
     })
 
     # Upload stacktrace.
     steps.append({
         'name':
-            'gcr.io/cloud-builders/gsutil',
+            'gcr.io/cloud-builders/gcloud',
         'entrypoint':
             '/bin/bash',
         'args': [
@@ -178,7 +178,7 @@ def run_experiment(project_name,
              f'"$target" {local_artifact_path}/* > '
              f'"{local_stacktrace_path}/$target.st" 2>&1; '
              'done; fi; '
-             f'gsutil -m cp -r {local_stacktrace_path} {upload_reproducer_path}'
+             f'gcloud storage cp --recursive {local_stacktrace_path} {upload_reproducer_path}'
              ' || true'),
         ],
     })
@@ -230,11 +230,11 @@ def run_experiment(project_name,
   # Upload raw coverage data.
   steps.append({
       'name':
-          'gcr.io/cloud-builders/gsutil',
+          'gcr.io/cloud-builders/gcloud',
       'args': [
-          '-m',
+          'storage',
           'cp',
-          '-r',
+          '--recursive',
           os.path.join(build.out, 'dumps'),
           os.path.join(upload_coverage_path, 'dumps'),
       ],
@@ -243,11 +243,11 @@ def run_experiment(project_name,
   # Upload coverage report.
   steps.append({
       'name':
-          'gcr.io/cloud-builders/gsutil',
+          'gcr.io/cloud-builders/gcloud',
       'args': [
-          '-m',
+          'storage',
           'cp',
-          '-r',
+          '--recursive',
           os.path.join(build.out, 'report'),
           os.path.join(upload_coverage_path, 'report'),
       ],
@@ -256,11 +256,11 @@ def run_experiment(project_name,
   # Upload textcovs.
   steps.append({
       'name':
-          'gcr.io/cloud-builders/gsutil',
+          'gcr.io/cloud-builders/gcloud',
       'args': [
-          '-m',
+          'storage',
           'cp',
-          '-r',
+          '--recursive',
           os.path.join(build.out, 'textcov_reports'),
           os.path.join(upload_coverage_path, 'textcov_reports'),
       ],

infra/build/status/deploy.sh

@@ -15,4 +15,5 @@
 #
 ################################################################################
 
-gsutil -h "Cache-Control:no-cache,max-age=0" -m cp -r bower_components index.html src manifest.json gs://oss-fuzz-build-logs
+# gcloud storage cp does not support setting metadata during upload. The -h flag is not translated.
+gcloud storage cp --recursive bower_components index.html src manifest.json gs://oss-fuzz-build-logs

infra/cifuzz/filestore/gsutil/__init__.py

@@ -27,23 +27,23 @@
 
 
 def _gsutil_execute(*args, parallel=True):
-  """Executes a gsutil command, passing |*args| to gsutil and returns the
+  """Executes a gcloud storage command, passing |*args| to it and returns the
   stdout, stderr and returncode. Exceptions on failure."""
-  command = ['gsutil']
+  command = ['gcloud', 'storage']
   if parallel:
-    command.append('-m')
+    pass
   command += list(args)
-  logging.info('Executing gsutil command: %s', command)
+  logging.info('Executing gcloud storage command: %s', command)
   return utils.execute(command, check_result=True)
 
 
 def _rsync(src, dst, recursive=True, delete=False):
-  """Executes gsutil rsync on |src| and |dst|"""
+  """Executes gcloud storage rsync on |src| and |dst|"""
   args = ['rsync']
   if recursive:
-    args.append('-r')
+    args.append('--recursive')
   if delete:
-    args.append('-d')
+    args.append('--delete-unmatched-destination-objects')
   args += [src, dst]
   return _gsutil_execute(*args)
 

infra/cifuzz/platform_config/gcb.py

@@ -37,4 +37,4 @@ def workspace(self):
   @property
   def filestore(self):
     """Returns the filestore used to store persistent data."""
-    return os.environ.get('FILESTORE', 'gsutil')
+    return os.environ.get('FILESTORE', 'gcloud storage')

infra/experimental/contrib/arvo/hacks/wolfssl.py

@@ -10,7 +10,7 @@ def apply_dockerfile_fixes(self, dft) -> bool:
     """Fix WolfSSL Dockerfile issues."""
     # Replace gsutil cp command with a simple touch and zip
     dft.str_replace(
-        'RUN gsutil cp '
+        'RUN gcloud storage cp '
         'gs://wolfssl-backup.clusterfuzz-external.appspot.com/'
         'corpus/libFuzzer/wolfssl_cryptofuzz-disable-fastmath/public.zip '
         '$SRC/corpus_wolfssl_disable-fastmath.zip', "RUN touch 0xdeadbeef && "

projects/wolfssl/Dockerfile

@@ -31,17 +31,17 @@ RUN git clone https://github.com/wolfssl/oss-fuzz-targets --depth 1 $SRC/fuzz-ta
 RUN wget https://storage.googleapis.com/pub/gsutil.tar.gz -O $SRC/gsutil.tar.gz
 RUN tar zxf $SRC/gsutil.tar.gz
 ENV PATH="${PATH}:$SRC/gsutil"
-RUN gsutil cp gs://bearssl-backup.clusterfuzz-external.appspot.com/corpus/libFuzzer/bearssl_cryptofuzz-bearssl/public.zip $SRC/corpus_bearssl.zip
-RUN gsutil cp gs://nettle-backup.clusterfuzz-external.appspot.com/corpus/libFuzzer/nettle_cryptofuzz-nettle-with-mini-gmp/public.zip $SRC/corpus_nettle.zip
-RUN gsutil cp gs://libecc-backup.clusterfuzz-external.appspot.com/corpus/libFuzzer/libecc_cryptofuzz-libecc/public.zip $SRC/corpus_libecc.zip
-RUN gsutil cp gs://relic-backup.clusterfuzz-external.appspot.com/corpus/libFuzzer/relic_cryptofuzz-relic/public.zip $SRC/corpus_relic.zip
-RUN gsutil cp gs://cryptofuzz-backup.clusterfuzz-external.appspot.com/corpus/libFuzzer/cryptofuzz_cryptofuzz-openssl/public.zip $SRC/corpus_cryptofuzz-openssl.zip
-RUN gsutil cp gs://cryptofuzz-backup.clusterfuzz-external.appspot.com/corpus/libFuzzer/cryptofuzz_cryptofuzz-boringssl/public.zip $SRC/corpus_cryptofuzz-boringssl.zip
-RUN gsutil cp gs://cryptofuzz-backup.clusterfuzz-external.appspot.com/corpus/libFuzzer/cryptofuzz_cryptofuzz-nss/public.zip $SRC/corpus_cryptofuzz-nss.zip
+RUN gcloud storage cp gs://bearssl-backup.clusterfuzz-external.appspot.com/corpus/libFuzzer/bearssl_cryptofuzz-bearssl/public.zip $SRC/corpus_bearssl.zip
+RUN gcloud storage cp gs://nettle-backup.clusterfuzz-external.appspot.com/corpus/libFuzzer/nettle_cryptofuzz-nettle-with-mini-gmp/public.zip $SRC/corpus_nettle.zip
+RUN gcloud storage cp gs://libecc-backup.clusterfuzz-external.appspot.com/corpus/libFuzzer/libecc_cryptofuzz-libecc/public.zip $SRC/corpus_libecc.zip
+RUN gcloud storage cp gs://relic-backup.clusterfuzz-external.appspot.com/corpus/libFuzzer/relic_cryptofuzz-relic/public.zip $SRC/corpus_relic.zip
+RUN gcloud storage cp gs://cryptofuzz-backup.clusterfuzz-external.appspot.com/corpus/libFuzzer/cryptofuzz_cryptofuzz-openssl/public.zip $SRC/corpus_cryptofuzz-openssl.zip
+RUN gcloud storage cp gs://cryptofuzz-backup.clusterfuzz-external.appspot.com/corpus/libFuzzer/cryptofuzz_cryptofuzz-boringssl/public.zip $SRC/corpus_cryptofuzz-boringssl.zip
+RUN gcloud storage cp gs://cryptofuzz-backup.clusterfuzz-external.appspot.com/corpus/libFuzzer/cryptofuzz_cryptofuzz-nss/public.zip $SRC/corpus_cryptofuzz-nss.zip
 
 # OpenSSL/LibreSSL corpora, which require a special import procedure
-RUN gsutil cp gs://openssl-backup.clusterfuzz-external.appspot.com/corpus/libFuzzer/openssl_bignum/public.zip $SRC/corpus_openssl_expmod.zip
-RUN gsutil cp gs://libressl-backup.clusterfuzz-external.appspot.com/corpus/libFuzzer/libressl_bignum/public.zip $SRC/corpus_libressl_expmod.zip
+RUN gcloud storage cp gs://openssl-backup.clusterfuzz-external.appspot.com/corpus/libFuzzer/openssl_bignum/public.zip $SRC/corpus_openssl_expmod.zip
+RUN gcloud storage cp gs://libressl-backup.clusterfuzz-external.appspot.com/corpus/libFuzzer/libressl_bignum/public.zip $SRC/corpus_libressl_expmod.zip
 
 WORKDIR wolfssl