[Bug]: Model optimization fails when run in Docker container

[RKuttruff]

Checked for duplicates

Yes - I've already checked

Describe the bug

When trying to run the docker container distribution of DIST-S1, we get the following error from model compilation:

Traceback (most recent call last):
  File "/opt/conda/envs/dist-s1-env/bin/dist-s1", line 8, in <module>
    sys.exit(cli())
             ~~~^^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/click/core.py", line 1442, in __call__
    return self.main(*args, **kwargs)
           ~~~~~~~~~^^^^^^^^^^^^^^^^^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/click/core.py", line 1363, in main
    rv = self.invoke(ctx)
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/click/core.py", line 1830, in invoke
    return _process_result(sub_ctx.command.invoke(sub_ctx))
                           ~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/click/core.py", line 1226, in invoke
    return ctx.invoke(self.callback, **ctx.params)
           ~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/click/core.py", line 794, in invoke
    return callback(*args, **kwargs)
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/dist_s1/__main__.py", line 327, in run_sas
    run_dist_s1_sas_workflow(run_config)
    ~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/dist_s1/workflows.py", line 523, in run_dist_s1_sas_workflow
    _ = run_dist_s1_processing_workflow(run_config)
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/dist_s1/workflows.py", line 420, in run_dist_s1_processing_workflow
    run_normal_param_estimation_workflow(run_config)
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/dist_s1/workflows.py", line 248, in run_normal_param_estimation_workflow
    compute_normal_params_per_burst_and_serialize(
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
        path_data['copol_paths_pre'],
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    ...<12 lines>...
        optimize=run_config.optimize,
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    )
    ^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/dist_s1/processing.py", line 95, in compute_normal_params_per_burst_and_serialize
    logits_mu, logits_sigma = estimate_normal_params_of_logits(
                              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
        model,
        ^^^^^^
    ...<5 lines>...
        batch_size=batch_size,
        ^^^^^^^^^^^^^^^^^^^^^^
    )
    ^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/distmetrics/transformer.py", line 541, in estimate_normal_params_of_logits
    mu, sigma = estimate_logits(
                ~~~~~~~~~~~~~~~^
        model, imgs_copol, imgs_crosspol, stride=stride, batch_size=batch_size, tqdm_enabled=tqdm_enabled, device=device
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    )
    ^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
    return func(*args, **kwargs)
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/distmetrics/transformer.py", line 490, in _estimate_logit_params_via_folding
    chip_mean, chip_logvar = model(patch_batch)
                             ~~~~~^^^^^^^^^^^^^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
           ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl
    return forward_call(*args, **kwargs)
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 663, in _fn
    raise e.remove_dynamo_frames() from None  # see TORCHDYNAMO_VERBOSE=1
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_inductor/compile_fx.py", line 760, in _compile_fx_inner
    raise InductorError(e, currentframe()).with_traceback(
        e.__traceback__
    ) from None
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_inductor/compile_fx.py", line 745, in _compile_fx_inner
    mb_compiled_graph = fx_codegen_and_compile(
        gm, example_inputs, inputs_to_check, **graph_kwargs
    )
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_inductor/compile_fx.py", line 1295, in fx_codegen_and_compile
    return scheme.codegen_and_compile(gm, example_inputs, inputs_to_check, graph_kwargs)
           ~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_inductor/compile_fx.py", line 1119, in codegen_and_compile
    graph.run(*example_inputs)
    ~~~~~~~~~^^^^^^^^^^^^^^^^^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_inductor/graph.py", line 877, in run
    return super().run(*args)
           ~~~~~~~~~~~^^^^^^^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/fx/interpreter.py", line 171, in run
    self.env[node] = self.run_node(node)
                     ~~~~~~~~~~~~~^^^^^^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_inductor/graph.py", line 1527, in run_node
    result = super().run_node(n)
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/fx/interpreter.py", line 240, in run_node
    return getattr(self, n.op)(n.target, args, kwargs)
           ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_inductor/graph.py", line 1198, in call_function
    raise LoweringException(e, target, args, kwargs).with_traceback(
        e.__traceback__
    ) from None
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_inductor/graph.py", line 1188, in call_function
    out = lowerings[target](*args, **kwargs)  # type: ignore[index]
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_inductor/lowering.py", line 466, in wrapped
    out = decomp_fn(*args, **kwargs)
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_inductor/kernel/mm.py", line 678, in tuned_addmm
    if use_cpp_gemm_template(layout, mat1, mat2):
       ~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_inductor/utils.py", line 1549, in use_cpp_gemm_template
    micro_gemm = create_micro_gemm(
        "micro_gemm",
    ...<8 lines>...
        q_group_size=q_group_size,
    )
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_inductor/codegen/cpp_micro_gemm.py", line 1609, in create_micro_gemm
    vec_isa = pick_vec_isa()
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_inductor/cpu_vec_isa.py", line 418, in pick_vec_isa
    _valid_vec_isa_list: list[VecISA] = valid_vec_isa_list()
                                        ~~~~~~~~~~~~~~~~~~^^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_inductor/cpu_vec_isa.py", line 405, in valid_vec_isa_list
    isa_list.extend(
    ~~~~~~~~~~~~~~~^
        isa
        ^^^
        for isa in supported_vec_isa_list
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
        if all(flag in _cpu_supported_x86_isa for flag in str(isa).split()) and isa
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    )
    ^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_inductor/cpu_vec_isa.py", line 408, in <genexpr>
    if all(flag in _cpu_supported_x86_isa for flag in str(isa).split()) and isa
                                                                            ^^^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_inductor/cpu_vec_isa.py", line 142, in __bool__
    return self.__bool__impl(config.cpp.vec_isa_ok)
           ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_inductor/cpu_vec_isa.py", line 152, in __bool__impl
    return self.check_build(VecISA._avx_code)
           ~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_inductor/cpu_vec_isa.py", line 102, in check_build
    extra=_get_isa_dry_compile_fingerprint(self._arch_flags),
          ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_inductor/cpu_vec_isa.py", line 28, in _get_isa_dry_compile_fingerprint
    compiler_info = get_compiler_version_info(get_cpp_compiler())
                                              ~~~~~~~~~~~~~~~~^^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_inductor/cpp_builder.py", line 156, in get_cpp_compiler
    compiler = cpp_compiler_search(search)
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_inductor/cpp_builder.py", line 102, in cpp_compiler_search
    raise exc.InvalidCxxCompiler
torch._inductor.exc.InductorError: LoweringException: InvalidCxxCompiler: No working C++ compiler found in torch._inductor.config.cpp.cxx: (None, 'g++')
  target: aten.addmm.default
  args[0]: TensorBox(StorageBox(
    InputBuffer(name='arg2_1', layout=FixedLayout('cpu', torch.float32, size=[256], stride=[1]))
  ))
  args[1]: TensorBox(
    View(
      View(
        StorageBox(
          ComputedBuffer(name='buf0', layout=FixedLayout('cpu', torch.float32, size=[32, 9, 2, 2, 2, 8, 8], stride=[4608, 512, 256, 128, 64, 8, 1]), data=Pointwise(device=device(type='cpu'), dtype=torch.float32, inner_fn=<function BaseView.make_loader.<locals>.loader at 0x7f4d69317d80>, ranges=[32, 9, 2, 2, 2, 8, 8]))
        ),
        size=[32, 9, 4, 128],
        reindex=lambda i0, i1, i2, i3: [i0, i1, ModularIndexing(i2, 2, 2), ModularIndexing(i2, 1, 2), ModularIndexing(i3, 64, 2), ModularIndexing(i3, 8, 8), ModularIndexing(i3, 1, 8)],
        origins=OrderedSet([view_1, clone])
      ),
      size=[1152, 128],
      reindex=lambda i0, i1: [ModularIndexing(i0, 36, 32), ModularIndexing(i0, 4, 9), ModularIndexing(i0, 1, 4), i1],
      origins=OrderedSet([view_2])
    )
  )
  args[2]: TensorBox(
    ReinterpretView(
      StorageBox(
        InputBuffer(name='arg1_1', layout=FixedLayout('cpu', torch.float32, size=[256, 128], stride=[128, 1]))
      ),
      FixedLayout('cpu', torch.float32, size=[128, 256], stride=[1, 128]),
      origins=OrderedSet([permute_1])
    )
  )

Set TORCHDYNAMO_VERBOSE=1 for the internal stack trace (please do this especially if you're reporting a bug to PyTorch). For even more developer context, set TORCH_LOGS="+dynamo"

What did you expect?

I expected the model to compile.

Reproducible steps

Follow the steps at https://github.com/nasa/opera-sds-pge/wiki/DIST%E2%80%90S1-Gamma-Acceptance-Testing-Instructions/cfb38674690c32cb1f6bbb951887271f95da5821 with an image like ghcr.io/opera-adt/dist-s1:1.0.1.dev14_g460a95d

Environment

ghcr.io/opera-adt/dist-s1:1.0.1.dev14_g460a95d

[RKuttruff]

This issue persists in 2.0.1 delivery, but with a new error:

Traceback (most recent call last):
  File "/opt/conda/envs/dist-s1-env/bin/dist-s1", line 7, in <module>
    sys.exit(cli())
             ~~~^^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/click/core.py", line 1442, in __call__
    return self.main(*args, **kwargs)
           ~~~~~~~~~^^^^^^^^^^^^^^^^^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/click/core.py", line 1363, in main
    rv = self.invoke(ctx)
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/click/core.py", line 1830, in invoke
    return _process_result(sub_ctx.command.invoke(sub_ctx))
                           ~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/click/core.py", line 1226, in invoke
    return ctx.invoke(self.callback, **ctx.params)
           ~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/click/core.py", line 794, in invoke
    return callback(*args, **kwargs)
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/dist_s1/__main__.py", line 464, in run_sas
    run_dist_s1_sas_workflow(run_config)
    ~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/dist_s1/workflows.py", line 522, in run_dist_s1_sas_workflow
    _ = run_dist_s1_processing_workflow(run_config)
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/dist_s1/workflows.py", line 397, in run_dist_s1_processing_workflow
    run_burst_disturbance_workflow(run_config)
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/dist_s1/workflows.py", line 272, in run_burst_disturbance_workflow
    _dist_processing_one_burst_wrapper(args)
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/dist_s1/workflows.py", line 100, in _dist_processing_one_burst_wrapper
    compute_burst_disturbance_and_serialize(
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
        pre_copol_paths=args.pre_copol_paths,
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    ...<19 lines>...
        use_date_encoding=args.use_date_encoding,
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    )
    ^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/dist_s1/dist_processing.py", line 112, in compute_burst_disturbance_and_serialize
    mu_2d, sigma_2d = estimate_normal_params(
                      ~~~~~~~~~~~~~~~~~~~~~~^
        model,
        ^^^^^^
    ...<6 lines>...
        dtype=model_dtype,
        ^^^^^^^^^^^^^^^^^^
    )
    ^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/distmetrics/tf_inference.py", line 312, in estimate_normal_params
    mu, sigma = estimate_norm_params(
                ~~~~~~~~~~~~~~~~~~~~^
        model,
        ^^^^^^
    ...<6 lines>...
        dtype=dtype,
        ^^^^^^^^^^^^
    )
    ^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
    return func(*args, **kwargs)
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/distmetrics/tf_inference.py", line 260, in _estimate_params_via_folding
    chip_mean, chip_logvar = model(patch_batch)
                             ~~~~~^^^^^^^^^^^^^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
           ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl
    return forward_call(*args, **kwargs)
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 663, in _fn
    raise e.remove_dynamo_frames() from None  # see TORCHDYNAMO_VERBOSE=1
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_inductor/compile_fx.py", line 760, in _compile_fx_inner
    raise InductorError(e, currentframe()).with_traceback(
        e.__traceback__
    ) from None
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_inductor/compile_fx.py", line 745, in _compile_fx_inner
    mb_compiled_graph = fx_codegen_and_compile(
        gm, example_inputs, inputs_to_check, **graph_kwargs
    )
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_inductor/compile_fx.py", line 1295, in fx_codegen_and_compile
    return scheme.codegen_and_compile(gm, example_inputs, inputs_to_check, graph_kwargs)
           ~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_inductor/compile_fx.py", line 1119, in codegen_and_compile
    graph.run(*example_inputs)
    ~~~~~~~~~^^^^^^^^^^^^^^^^^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_inductor/graph.py", line 877, in run
    return super().run(*args)
           ~~~~~~~~~~~^^^^^^^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/fx/interpreter.py", line 171, in run
    self.env[node] = self.run_node(node)
                     ~~~~~~~~~~~~~^^^^^^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_inductor/graph.py", line 1527, in run_node
    result = super().run_node(n)
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/fx/interpreter.py", line 240, in run_node
    return getattr(self, n.op)(n.target, args, kwargs)
           ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_inductor/graph.py", line 1198, in call_function
    raise LoweringException(e, target, args, kwargs).with_traceback(
        e.__traceback__
    ) from None
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_inductor/graph.py", line 1188, in call_function
    out = lowerings[target](*args, **kwargs)  # type: ignore[index]
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_inductor/lowering.py", line 466, in wrapped
    out = decomp_fn(*args, **kwargs)
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_inductor/kernel/mm.py", line 678, in tuned_addmm
    if use_cpp_gemm_template(layout, mat1, mat2):
       ~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_inductor/utils.py", line 1549, in use_cpp_gemm_template
    micro_gemm = create_micro_gemm(
        "micro_gemm",
    ...<8 lines>...
        q_group_size=q_group_size,
    )
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_inductor/codegen/cpp_micro_gemm.py", line 1609, in create_micro_gemm
    vec_isa = pick_vec_isa()
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_inductor/cpu_vec_isa.py", line 418, in pick_vec_isa
    _valid_vec_isa_list: list[VecISA] = valid_vec_isa_list()
                                        ~~~~~~~~~~~~~~~~~~^^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_inductor/cpu_vec_isa.py", line 405, in valid_vec_isa_list
    isa_list.extend(
    ~~~~~~~~~~~~~~~^
        isa
        ^^^
        for isa in supported_vec_isa_list
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
        if all(flag in _cpu_supported_x86_isa for flag in str(isa).split()) and isa
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    )
    ^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_inductor/cpu_vec_isa.py", line 408, in <genexpr>
    if all(flag in _cpu_supported_x86_isa for flag in str(isa).split()) and isa
                                                                            ^^^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_inductor/cpu_vec_isa.py", line 142, in __bool__
    return self.__bool__impl(config.cpp.vec_isa_ok)
           ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_inductor/cpu_vec_isa.py", line 152, in __bool__impl
    return self.check_build(VecISA._avx_code)
           ~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_inductor/cpu_vec_isa.py", line 110, in check_build
    buid_options = CppTorchOptions(vec_isa=self, warning_all=False)
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_inductor/cpp_builder.py", line 1146, in __init__
    ) = get_cpp_torch_options(
        ~~~~~~~~~~~~~~~~~~~~~^
        cpp_compiler=self._compiler,
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    ...<5 lines>...
        use_mmap_weights=use_mmap_weights,
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    )
    ^
  File "/opt/conda/envs/dist-s1-env/lib/python3.13/site-packages/torch/_inductor/cpp_builder.py", line 1077, in get_cpp_torch_options
    + [os.getenv('CONDA_PREFIX') + '/include']
       ~~~~~~~~~~~~~~~~~~~~~~~~~~^~~~~~~~~~~~
torch._inductor.exc.InductorError: LoweringException: TypeError: unsupported operand type(s) for +: 'NoneType' and 'str'
  target: aten.addmm.default
  args[0]: TensorBox(StorageBox(
    InputBuffer(name='arg2_1', layout=FixedLayout('cpu', torch.float32, size=[128], stride=[1]))
  ))
  args[1]: TensorBox(
    View(
      View(
        StorageBox(
          ComputedBuffer(name='buf0', layout=FixedLayout('cpu', torch.float32, size=[1, 10, 2, 2, 2, 8, 8], stride=[5120, 512, 256, 128, 64, 8, 1]), data=Pointwise(device=device(type='cpu'), dtype=torch.float32, inner_fn=<function BaseView.make_loader.<locals>.loader at 0x7f010480e0c0>, ranges=[1, 10, 2, 2, 2, 8, 8]))
        ),
        size=[1, 10, 4, 128],
        reindex=lambda i0, i1, i2, i3: [0, i1, ModularIndexing(i2, 2, 2), ModularIndexing(i2, 1, 2), ModularIndexing(i3, 64, 2), ModularIndexing(i3, 8, 8), ModularIndexing(i3, 1, 8)],
        origins=OrderedSet([view_1, clone])
      ),
      size=[40, 128],
      reindex=lambda i0, i1: [0, ModularIndexing(i0, 4, 10), ModularIndexing(i0, 1, 4), i1],
      origins=OrderedSet([view_2])
    )
  )
  args[2]: TensorBox(
    ReinterpretView(
      StorageBox(
        InputBuffer(name='arg1_1', layout=FixedLayout('cpu', torch.float32, size=[128, 128], stride=[128, 1]))
      ),
      FixedLayout('cpu', torch.float32, size=[128, 128], stride=[1, 128]),
      origins=OrderedSet([permute_1])
    )
  )

Set TORCHDYNAMO_VERBOSE=1 for the internal stack trace (please do this especially if you're reporting a bug to PyTorch). For even more developer context, set TORCH_LOGS="+dynamo"

[cmarshak]

But model compilation is turned off right?

https://github.com/opera-adt/dist-s1/blob/dev/src/dist_s1/data_models/defaults.py#L39

[RKuttruff]

Yes, turning it off will avoid the error

The configuration I was testing with had it on (unknown to me)