Skip to content

Commit 95a837a

Browse files
committed
fix(cwl_reana): preserve staged inputs during output copy (reanahub#322)
Unlink InitialWorkDirRequirement input symlinks immediately before copying job outputs back into the CWL outdir. Prevent cp from following a destination symlink and overwriting the original workspace input, which otherwise depends on its write mode. Closes reanahub#321
1 parent 5e8a9dd commit 95a837a

2 files changed

Lines changed: 92 additions & 0 deletions

File tree

reana_workflow_engine_cwl/cwl_reana.py

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,39 @@ def add_volumes(self, pathmapper):
160160
with os.fdopen(fd, "wb") as f:
161161
f.write(vol.resolved.encode("utf-8"))
162162

163+
def _initial_workdir_symlink_cleanup_command(self):
164+
"""Build a command that unlinks staged inputs before copying outputs."""
165+
generatemapper = getattr(self, "generatemapper", None)
166+
if not generatemapper:
167+
return ""
168+
169+
container_outdir = self.builder.outdir.rstrip("/")
170+
target_prefix = container_outdir + "/"
171+
host_outdir = os.path.abspath(self.outdir)
172+
cleanup_command = ""
173+
for _, volume in generatemapper.items():
174+
is_linked_type = volume.type in ("File", "Directory") or (
175+
self.inplace_update
176+
and volume.type in ("WritableFile", "WritableDirectory")
177+
)
178+
if (
179+
not volume.staged
180+
or not is_linked_type
181+
or volume.resolved.startswith("_:")
182+
or not volume.target.startswith(target_prefix)
183+
):
184+
continue
185+
186+
relative_target = volume.target[len(target_prefix) :]
187+
host_target = os.path.abspath(os.path.join(host_outdir, relative_target))
188+
if os.path.commonpath([host_outdir, host_target]) != host_outdir:
189+
continue
190+
quoted_target = shellescape.quote(host_target)
191+
cleanup_command += f"; if [ -L {quoted_target} ]; then "
192+
cleanup_command += f"rm -f {quoted_target}; fi"
193+
194+
return cleanup_command
195+
163196
def create_task_msg(self, working_dir, workflow_uuid): # noqa: C901
164197
"""Create job message spec to be sent to REANA-Job-Controller."""
165198
job_name = self.name
@@ -247,6 +280,7 @@ def shouldquote(x):
247280
docker_req, _ = self.get_requirement("DockerRequirement")
248281
if docker_req:
249282
docker_output_dir = docker_req.get("dockerOutputDirectory", None)
283+
wf_space_cmd += self._initial_workdir_symlink_cleanup_command()
250284
if docker_output_dir:
251285
wf_space_cmd = (
252286
f"mkdir -p {docker_output_dir} && {wf_space_cmd}"

tests/test_cwl_reana.py

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
# -*- coding: utf-8 -*-
2+
#
3+
# This file is part of REANA.
4+
# Copyright (C) 2026 CERN.
5+
#
6+
# REANA is free software; you can redistribute it and/or modify it
7+
# under the terms of the MIT License; see LICENSE file for more details.
8+
9+
"""REANA CWL job tests."""
10+
11+
import subprocess
12+
from types import SimpleNamespace
13+
14+
import shellescape
15+
16+
from reana_workflow_engine_cwl.cwl_reana import ReanaPipelineJob
17+
18+
19+
def test_initial_workdir_input_is_not_overwritten_during_copy(tmp_path):
20+
"""Unlink a staged input before copying an output with the same name."""
21+
uploaded_input = tmp_path / "workspace" / "gendata.C"
22+
uploaded_input.parent.mkdir()
23+
uploaded_input.write_text("uploaded input")
24+
25+
outdir = tmp_path / "cwl" / "outdir"
26+
outdir.mkdir(parents=True)
27+
staged_input = outdir / uploaded_input.name
28+
staged_input.symlink_to(uploaded_input)
29+
30+
container_outdir = "/var/lib/cwl/job"
31+
job = object.__new__(ReanaPipelineJob)
32+
job.builder = SimpleNamespace(outdir=container_outdir)
33+
job.generatemapper = {
34+
"input": SimpleNamespace(
35+
staged=True,
36+
type="File",
37+
resolved=str(uploaded_input),
38+
target=f"{container_outdir}/{uploaded_input.name}",
39+
)
40+
}
41+
job.inplace_update = False
42+
job.outdir = str(outdir)
43+
44+
generated_output = tmp_path / "job-output"
45+
generated_output.mkdir()
46+
(generated_output / uploaded_input.name).write_text("generated output")
47+
48+
cleanup_command = job._initial_workdir_symlink_cleanup_command()
49+
command = (
50+
f"true{cleanup_command}; cp -r "
51+
f"{shellescape.quote(str(generated_output))}/* "
52+
f"{shellescape.quote(str(outdir))}"
53+
)
54+
subprocess.run(["/bin/sh", "-c", command], check=True)
55+
56+
assert uploaded_input.read_text() == "uploaded input"
57+
assert not staged_input.is_symlink()
58+
assert staged_input.read_text() == "generated output"

0 commit comments

Comments
 (0)