Skip to content

Out-of-memory error - sits_tuning for LightTAE #31

Description

@biancazadrozny

When I run sits_tuning for LightTAE:

# Load samples
samples_matogrosso_mod13q1 = load_samples(
    name = "samples_matogrosso_mod13q1", 
    package = "sitsdata"
)

# Tuning ``sits_lighttae`` model
tuned_mt = sits_tuning(
     samples = samples_matogrosso_mod13q1,
     ml_method = sits_lighttae,
     params = sits_tuning_hparams(
         optimizer = "torch::optim_adamw",
         opt_hparams = dict(
             lr = hparam("loguniform", 10**-2, 10**-4),
             weight_decay = hparam("loguniform", 10**-2, 10**-8)
             )
         ),
     trials = 40,
     multicores = 6,
     progress = True
)

I get the following memory error after the Progress bar shows 100%:

RRuntimeError                             Traceback (most recent call last)
Cell In[5], line 2
      1 # Tuning ``sits_lighttae`` model
----> 2 tuned_mt = sits_tuning(
      3      samples = samples_matogrosso_mod13q1,
      4      ml_method = sits_lighttae,
      5      params = sits_tuning_hparams(

File [~/kernels/data/lib/python3.12/site-packages/pysits/conversions/decorators.py:108](http://20.67.103.55:8888/lab/tree/~/kernels/data/lib/python3.12/site-packages/pysits/conversions/decorators.py#line=107), in rpy2_fix_type_custom.<locals>.decorator.<locals>.wrapper(*args, **kwargs)
    105     else:
    106         converted_kwargs[k] = v
--> 108 return func(*converted_args, **converted_kwargs)

File [~/kernels/data/lib/python3.12/site-packages/pysits/conversions/decorators.py:53](http://20.67.103.55:8888/lab/tree/~/kernels/data/lib/python3.12/site-packages/pysits/conversions/decorators.py#line=52), in rpy2_fix_type.<locals>.wrapper(*args, **kwargs)
     51 converted_args = [convert_to_r(arg) for arg in args]
     52 converted_kwargs = {k: convert_to_r(v) for k, v in kwargs.items()}
---> 53 return func(*converted_args, **converted_kwargs)

File [~/kernels/data/lib/python3.12/site-packages/pysits/conversions/decorators.py:143](http://20.67.103.55:8888/lab/tree/~/kernels/data/lib/python3.12/site-packages/pysits/conversions/decorators.py#line=142), in function_call.<locals>.decorator.<locals>.wrapped(*args, **kwargs)
    140 @rpy2_fix_type
    141 @functools.wraps(func)
    142 def wrapped(*args: P.args, **kwargs: P.kwargs) -> T:
--> 143     result = r_function(*args, **kwargs)
    144     return output_wrapper(result)

File [~/kernels/data/lib/python3.12/site-packages/rpy2/robjects/functions.py:208](http://20.67.103.55:8888/lab/tree/~/kernels/data/lib/python3.12/site-packages/rpy2/robjects/functions.py#line=207), in SignatureTranslatedFunction.__call__(self, *args, **kwargs)
    206         v = kwargs.pop(k)
    207         kwargs[r_k] = v
--> 208 return (super(SignatureTranslatedFunction, self)
    209         .__call__(*args, **kwargs))

File [~/kernels/data/lib/python3.12/site-packages/rpy2/robjects/functions.py:131](http://20.67.103.55:8888/lab/tree/~/kernels/data/lib/python3.12/site-packages/rpy2/robjects/functions.py#line=130), in Function.__call__(self, *args, **kwargs)
    129     else:
    130         new_kwargs[k] = cv.py2rpy(v)
--> 131 res = super(Function, self).__call__(*new_args, **new_kwargs)
    132 res = cv.rpy2py(res)
    133 return res

File [~/kernels/data/lib/python3.12/site-packages/rpy2/rinterface_lib/conversion.py:46](http://20.67.103.55:8888/lab/tree/~/kernels/data/lib/python3.12/site-packages/rpy2/rinterface_lib/conversion.py#line=45), in _cdata_res_to_rinterface.<locals>._(*args, **kwargs)
     45 def _(*args, **kwargs):
---> 46     cdata = function(*args, **kwargs)
     47     # TODO: test cdata is of the expected CType
     48     return _cdata_to_rinterface(cdata)

File [~/kernels/data/lib/python3.12/site-packages/rpy2/rinterface/__init__.py:930](http://20.67.103.55:8888/lab/tree/~/kernels/data/lib/python3.12/site-packages/rpy2/rinterface/__init__.py#line=929), in SexpClosure.__call__(self, *args, **kwargs)
    923     res = rmemory.protect(
    924         openrlib.rlib.R_tryEval(
    925             call_r,
    926             call_context.__sexp__._cdata,
    927             error_occured)
    928     )
    929     if error_occured[0]:
--> 930         raise embedded.RRuntimeError(_rinterface._geterrmessage())
    931 return res

RRuntimeError: Error in .parallel_check_remote_errors(val) : 
  1 node(s) produced errors; first error: not enough GPU memory - should have at least 2GB memory free to use

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions