[PEP389] introduce argparse deprecate optparse

bin/crab.py

@@ -60,7 +60,7 @@ def __init__( self ):
 
     def __call__(self):
 
-        (options, args) = self.parser.parse_args()
+        (options, args) = self.parser.parse_cmd()
 
         ## The default logfile destination is ./crab.log. It will be changed once we
         ## know/create the CRAB project directory.

doc/generate_modules.py

@@ -28,7 +28,7 @@
 
 
 import os
-import optparse
+import argparse
 
 
 # automodule options
@@ -232,17 +232,17 @@ def main():
     """
     Parse and check the command line arguments.
     """
-    parser = optparse.OptionParser(usage="""usage: %prog [options] <package path> [exclude paths, ...]
+    parser = argparse.ArgumentParser(usage="""usage: %(prog)s, [options] <package path> [exclude paths, ...]
 
 Note: By default this script will not overwrite already created files.""")
-    parser.add_option("-n", "--doc-header", action="store", dest="header", help="Documentation Header (default=Project)", default="Project")
-    parser.add_option("-d", "--dest-dir", action="store", dest="destdir", help="Output destination directory", default="")
-    parser.add_option("-s", "--suffix", action="store", dest="suffix", help="module suffix (default=txt)", default="txt")
-    parser.add_option("-m", "--maxdepth", action="store", dest="maxdepth", help="Maximum depth of submodules to show in the TOC (default=4)", type="int", default=4)
-    parser.add_option("-r", "--dry-run", action="store_true", dest="dryrun", help="Run the script without creating the files")
-    parser.add_option("-f", "--force", action="store_true", dest="force", help="Overwrite all the files")
-    parser.add_option("-t", "--no-toc", action="store_true", dest="notoc", help="Don't create the table of content file")
-    (opts, args) = parser.parse_args()
+    parser.add_argument("-n", "--doc-header", action="store", dest="header", help="Documentation Header (default=Project)", default="Project")
+    parser.add_argument("-d", "--dest-dir", action="store", dest="destdir", help="Output destination directory", default="")
+    parser.add_argument("-s", "--suffix", action="store", dest="suffix", help="module suffix (default=txt)", default="txt")
+    parser.add_argument("-m", "--maxdepth", action="store", dest="maxdepth", help="Maximum depth of submodules to show in the TOC (default=4)", type=int, default=4)
+    parser.add_argument("-r", "--dry-run", action="store_true", dest="dryrun", help="Run the script without creating the files")
+    parser.add_argument("-f", "--force", action="store_true", dest="force", help="Overwrite all the files")
+    parser.add_argument("-t", "--no-toc", action="store_true", dest="notoc", help="Don't create the table of content file")
+    opts, args = parser.parse_known_args()
     if not args:
         parser.error("package path is required.")
     else:

src/python/CRABAPI/RawCommand.py

@@ -33,7 +33,7 @@ def execRaw(command, args):
     """
         execRaw - executes a given command with certain arguments and returns
                   the raw result back from the client. args is a python list,
-                  the same python list parsed by the optparse module
+                  the same python list parsed by the argument parser
                   Every command returns a dictionary of the form
                   {'commandStatus': status, key: val, key: val ....}
                   where status can have the values 'SUCCESS' or 'FAILED'
@@ -51,7 +51,7 @@ def execRaw(command, args):
         cmdobj = getattr(mod, command)(logger, args)
         res = cmdobj()
     except SystemExit as se:
-        # most likely an error from the OptionParser in Subcommand.
+        # most likely an error from argument parsing in Subcommand.
         # CRABClient #4283 should make this less ugly
         if se.code == 2:
             raise CRABAPI.BadArgumentException

src/python/CRABAPI/__init__.py

@@ -13,7 +13,7 @@ class APIException(Exception):
 
 class BadArgumentException(APIException):
     """
-        BadArgumentException - Arguments passed didn't pass optparse's muster
+        BadArgumentException - Arguments passed didn't pass argument parsing
     """
     pass
 

src/python/CRABClient/CRABOptParser.py

@@ -1,13 +1,80 @@
 # silence pylint complaints about things we need for Python 2.6 compatibility
 # pylint: disable=unspecified-encoding, raise-missing-from, consider-using-f-string
 
-from optparse import OptionParser  # pylint: disable=deprecated-module
+import sys
+from argparse import ArgumentParser
 
 from ServerUtilities import SERVICE_INSTANCES
 from CRABClient import __version__ as client_version
 
-
-class CRABOptParser(OptionParser):
+def _split_at_first_positional(parser, argv):
+    """
+    Emulate optparse.disable_interspersed_args:
+    parse options only until the first positional token, consuming option values.
+    """
+    idx = 0
+    size = len(argv)
+    while idx < size:
+        tok = argv[idx]
+        if tok == "--":
+            idx += 1
+            break
+        if tok == "-" or not tok.startswith("-"):
+            break
+
+        optname = tok.split("=", 1)[0] if tok.startswith("--") else tok
+        action = parser._option_string_actions.get(optname)  # pylint: disable=protected-access
+        idx += 1
+        if action is None:
+            continue
+        if tok.startswith("--") and "=" in tok:
+            continue
+
+        nargs = action.nargs
+        if nargs in (None, 1):
+            if idx < size:
+                idx += 1
+        elif nargs in (0,):
+            continue
+        elif nargs == "?":
+            if idx < size and not argv[idx].startswith("-"):
+                idx += 1
+        elif isinstance(nargs, int):
+            idx += nargs
+        else:
+            idx = size
+
+    return argv[:idx], argv[idx:]
+
+class CRABArgParser(ArgumentParser):
+    def __init__(self, *args, disable_interspersed_args=False, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._disable_interspersed_args = disable_interspersed_args
+
+    def parse_cmd(self, argv=None):
+        argv = list(sys.argv[1:] if argv is None else argv)
+
+        if not self._disable_interspersed_args:
+            if "--" in argv:
+                marker = argv.index("--")
+                parseable = argv[:marker]
+                tail = argv[marker + 1:]
+            else:
+                parseable = argv
+                tail = []
+
+            ns, rest = self.parse_known_args(parseable)
+            unknown = [arg for arg in rest if arg.startswith("-") and arg != "-"]
+            if unknown:
+                self.error("no such option: %s" % unknown[0])
+            return ns, rest + tail
+
+        opt_argv, rest = _split_at_first_positional(self, argv)
+        ns = self.parse_args(opt_argv)
+        return ns, rest
+
+
+class CRABOptParser(CRABArgParser):
     """
     Allows to make OptionParser behave how we prefer
     """
@@ -19,7 +86,7 @@ def __init__(self, subCommands=None):
 
             subCommands: if present used to prepare a nice help summary for all the commands
         """
-        usage  = "usage: %prog [options] COMMAND [command-options] [args]"
+        usage = "usage: %(prog)s [options] COMMAND [command-options] [args]"
         epilog = ""
         if subCommands:
             epilog = '\nValid commands are: \n'
@@ -32,50 +99,41 @@ def __init__(self, subCommands=None):
             epilog += '\nFor more information on how to run CRAB-3 please follow this link:\n'
             epilog += 'https://twiki.cern.ch/twiki/bin/view/CMSPublic/WorkBookCRAB3Tutorial\n'
 
-        OptionParser.__init__(self, usage   = usage, epilog  = epilog,
-                                version = "CRAB client %s" % client_version
-                             )
+        CRABArgParser.__init__(self, usage=usage, epilog=epilog, disable_interspersed_args=True)
 
-        # This is the important bit
-        self.disable_interspersed_args()
-
-        self.add_option( "--quiet",
+        self.add_argument("--version", action="version", version="CRAB client %s" % client_version)
+        self.add_argument( "--quiet",
                                 action = "store_true",
                                 dest = "quiet",
                                 default = False,
                                 help = "don't print any messages to stdout" )
 
-        self.add_option( "--debug",
+        self.add_argument( "--debug",
                                 action = "store_true",
                                 dest = "debug",
                                 default = False,
                                 help = "print extra messages to stdout" )
 
 
-    def format_epilog(self, formatter):
-        """
-        do not strip the new lines from the epilog
-        """
-        return self.epilog
-
-
-
-class CRABCmdOptParser(OptionParser):
+class CRABCmdOptParser(CRABArgParser):
     """ A class that extract the pieces for parsing the command line arguments
         of the CRAB commands.
 
     """
 
-    def __init__(self, cmdname, doc, disable_interspersed_args):
+    def __init__(self, cmdname, doc, disable_interspersed_args=False):
         """
             doc:        the description of the command. Taken from self.__doc__
             disable_interspersed_args: some commands (e.g.: submit) allow to overwrite configuration parameters
         """
-        usage = "usage: %prog " + cmdname + " [options] [args]"
-        OptionParser.__init__(self, description = doc, usage = usage, add_help_option = True)
-        if disable_interspersed_args:
-            self.disable_interspersed_args()
-
+        usage = "usage: %(prog)s " + cmdname + " [options] [args]"
+        CRABArgParser.__init__(
+            self,
+            description=doc,
+            usage=usage,
+            add_help=True,
+            disable_interspersed_args=disable_interspersed_args
+        )
 
     def addCommonOptions(self, cmdconf):
         """
@@ -86,35 +144,35 @@ def addCommonOptions(self, cmdconf):
             a value is not indicated anywhere by the user is defined in ClientMapping.py
         """
         if cmdconf['requiresDirOption']:
-            self.add_option("-d", "--dir",
-                                   dest = "projdir",
-                                   default = None,
-                                   help = "Path to the CRAB project directory for which the crab command should be executed.")
-            self.add_option("--task",
-                                dest = "cmptask",
-                                default = None,
-                                help = "In alternative to -d, a complete task name. Can be taken from 'crab status' output, or from dashboard.")
+            self.add_argument("-d", "--dir",
+                              dest="projdir",
+                              default=None,
+                              help="Path to the CRAB project directory for which the crab command should be executed.")
+            self.add_argument("--task",
+                              dest="cmptask",
+                              default=None,
+                              help="In alternative to -d, a complete task name. Can be taken from 'crab status' output, or from dashboard.")
 
         if cmdconf['requiresREST']:
-            self.add_option("--instance",
-                                   dest = "instance",
-                                   type = "string",
-                                   default = None,
-                                   help = "Running instance of CRAB service." \
-                                          " Needed whenever --task is used." \
-                                          " Default value is 'prod'. " \
-                                          " Valid values are %s." \
-                                          % str(list(SERVICE_INSTANCES.keys())) )
+            self.add_argument("--instance",
+                              dest="instance",
+                              type=str,
+                              default=None,
+                              help="Running instance of CRAB service."
+                                   " Needed whenever --task is used."
+                                   " Default value is 'prod'. "
+                                   " Valid values are %s."
+                                   % str(list(SERVICE_INSTANCES.keys())))
 
         if cmdconf['requiresProxyVOOptions']:
-            self.add_option("--voRole",
-                                   dest = "voRole",
-                                   default = None)
-            self.add_option("--voGroup",
-                                   dest = "voGroup",
-                                   default = None)
-
-        self.add_option("--proxy",
-                        dest="proxy",
-                        default=False,
-                        help="Use the given proxy. Skip Grid proxy creation and myproxy delegation.")
+            self.add_argument("--voRole",
+                              dest="voRole",
+                              default=None)
+            self.add_argument("--voGroup",
+                              dest="voGroup",
+                              default=None)
+
+        self.add_argument("--proxy",
+                          dest="proxy",
+                          default=False,
+                          help="Use the given proxy. Skip Grid proxy creation and myproxy delegation.")

src/python/CRABClient/ClientUtilities.py

@@ -634,26 +634,26 @@ def setSubmitParserOptions(parser):
     """ Set the option for the parser of the submit command.
         Method put here in the utilities since it is shared between the submit command and the crab3bootstrap script.
     """
-    parser.add_option('-c', '--config',
+    parser.add_argument('-c', '--config',
                            dest='config',
                            default=None,
                            help="CRAB configuration file.",
                            metavar='FILE')
 
-    parser.add_option('--wait',
+    parser.add_argument('--wait',
                            dest='wait',
                            default=False,
                            action='store_true',
                            help="DEPRECATED.")
 
-    parser.add_option('--dryrun',
+    parser.add_argument('--dryrun',
                            dest='dryrun',
                            default=False,
                            action='store_true',
                            help="Do not actually submit the task; instead, return how many jobs this task would create, "\
                                   "along with processing time and memory consumption estimates.")
 
-    parser.add_option('--skip-estimates',
+    parser.add_argument('--skip-estimates',
                            dest='skipEstimates',
                            default=False,
                            action='store_true',

src/python/CRABClient/Commands/SubCommand.py

@@ -272,7 +272,7 @@ def __init__(self, logger, cmdargs=None, disable_interspersed_args=False):
         # Parse the command options/arguments.
         cmdargs = cmdargs or []
         self.cmdargs = cmdargs
-        (self.options, self.args) = self.parser.parse_args(cmdargs)
+        (self.options, self.args) = self.parser.parse_cmd(cmdargs)
 
         self.transferringIds = None
         self.dest = None

src/python/CRABClient/Commands/checkdataset.py

@@ -273,11 +273,11 @@ def setOptions(self):
 
         This allows to set specific command options
         """
-        self.parser.add_option('--dataset',
+        self.parser.add_argument('--dataset',
                                dest='dataset',
                                default=None,
                                help='dataset of block ID or Rucio DID (scope:name)')
-        self.parser.add_option('--dbs-instance', dest='dbsInstance', default='prod/global',
+        self.parser.add_argument('--dbs-instance', dest='dbsInstance', default='prod/global',
                                help="DBS instance. e.g. prod/global (default) or prod/phys03 or full URL."
                                     + "\nUse at your own risk only if you really know what you are doing"
                                )

src/python/CRABClient/Commands/checkfile.py

@@ -302,19 +302,19 @@ def setOptions(self):
 
         This allows to set specific command options
         """
-        self.parser.add_option('--lfn',
+        self.parser.add_argument('--lfn',
                                dest='lfn',
                                default=None,
                                help='LFN of the file to check')
-        self.parser.add_option('--checksum',
+        self.parser.add_argument('--checksum',
                                dest='checkChecksum',
                                action="store_true",
                                help="check checksum of all disk replicas. SLOW and needs GB's of disk !")
-        self.parser.add_option('--dbs-instance', dest='dbsInstance', default='prod/global',
+        self.parser.add_argument('--dbs-instance', dest='dbsInstance', default='prod/global',
                                help="DBS instance. e.g. prod/global (default) or prod/phys03 or full URL."
                                     + "\nUse at your own risk only if you really know what you are doing"
                                )
-        self.parser.add_option('--rucio-scope', dest='scope', default=None,
+        self.parser.add_argument('--rucio-scope', dest='scope', default=None,
                                help="Rucio scope. Default is 'cms' for global DBS and 'user:username' for phys03"
                                )
 

src/python/CRABClient/Commands/checkwrite.py

@@ -326,20 +326,20 @@ def setOptions(self):
 
         This allows to set specific command options
         """
-        self.parser.add_option('--site',
+        self.parser.add_argument('--site',
                                dest='sitename',
                                default=None,
                                help='The PhEDEx node name of the site to be checked.')
-        self.parser.add_option('--lfn',
+        self.parser.add_argument('--lfn',
                                dest='userlfn',
                                default=None,
                                help='A user lfn address.')
-        self.parser.add_option('--checksum',
+        self.parser.add_argument('--checksum',
                                dest='checksum',
                                default='no',
                                help='Set it to yes if needed. It will use ADLER32 checksum' +\
                                        'Allowed values are yes/no. Default is no.')
-        self.parser.add_option('--command',
+        self.parser.add_argument('--command',
                                dest='command',
                                default=None,
                                help='A command which to use. Available commands are LCG or GFAL.')