datashuttle/tests/tests_integration/test_local_only_mode.py at bc8fd59bec399b2881ed6ce16ad030c05fe44200 · neuroinformatics-unit/datashuttle · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
import shutil

import pytest
import test_utils
from base import BaseTest

from datashuttle.utils.custom_exceptions import (
    ConfigError,
)

TEST_PROJECT_NAME = "test_project"


class TestLocalOnlyProject(BaseTest):

    def test_bad_setup(self, tmp_path):
        """
        Test setup without providing both central_path and connection
        method (distinguishing a full vs local-only project)
        """
        local_path = tmp_path / "test_local"

        project = test_utils.make_project("this_project_is_not_torn_down")

        with pytest.raises(ConfigError) as e:
            project.make_config_file(
                local_path, central_path=tmp_path / "central"
            )
        assert (
            "Either both `central_path` and `connection_method` must be set"
            in str(e.value)
        )

        with pytest.raises(ConfigError) as e:
            project.make_config_file(local_path, connection_method="ssh")
        assert (
            "Either both `central_path` and `connection_method` must be set"
            in str(e.value)
        )

    @pytest.mark.parametrize("project", ["local"], indirect=True)
    def test_full_to_local_project(self, project):
        """
        Make a full project a local-only project, and check the transfer
        functionality is now restricted.
        """
        project.update_config_file(central_path=None, connection_method=None)

        with pytest.raises(ConfigError) as e:
            project.upload_entire_project()

        assert "This function cannot be used for a local-project." in str(
            e.value
        )

        project.create_folders("rawdata", "sub-001", "ses-001", "ephys")

        project.validate_project("rawdata", "error")

    @pytest.mark.parametrize("project", ["local"], indirect=True)
    def test_local_project_to_full(self, tmp_path, project):
        """
        Test updating a local-only project to a full one
        by adding the required configs (both must be set together)
        Perform a quick check that data transfer does not error out
        now that the project is full, and the configs are set as expected.
        """
        central_path = tmp_path / "central"
        connection_method = "local_filesystem"

        # check must set both at once
        with pytest.raises(ConfigError):
            project.update_config_file(central_path=central_path)

        with pytest.raises(ConfigError):
            project.update_config_file(connection_method=connection_method)

        project.update_config_file(
            central_path=central_path, connection_method=connection_method
        )

        # smoke test, should not raise now is a full project.
        project.upload_entire_project()

        assert project.cfg["central_path"] == central_path / TEST_PROJECT_NAME
        assert project.cfg["connection_method"] == connection_method

    @pytest.mark.parametrize("project", ["local"], indirect=True)
    def test_local_to_full_project(self, project):
        """
        Change a project from local-only to a normal project by updating
        the relevant configs. Smoke test that general functionality is maintained
        and that transfers work correctly.
        """
        central_path = project.cfg["local_path"].parent / "central"

        project.update_config_file(
            central_path=central_path, connection_method="local_filesystem"
        )
        (project.cfg["central_path"] / "rawdata").mkdir(
            parents=True
        )  # to pass validation

        paths_ = project.create_folders(
            "rawdata", "sub-001", "ses-001", "ephys"
        )
        test_utils.write_file(
            paths_["ephys"][0] / "test_file", contents="test_entry"
        )

        project.validate_project("rawdata", "error")

        project.upload_entire_project()

        assert (
            central_path
            / TEST_PROJECT_NAME
            / "rawdata"
            / "sub-001"
            / "ses-001"
            / "ephys"
            / "test_file"
        ).is_file()

    @pytest.mark.parametrize("top_level_folder", ["rawdata", "derivatives"])
    @pytest.mark.parametrize("project", ["full"], indirect=True)
    def test_get_next_sub_and_ses(self, project, top_level_folder):
        """
        Make a 'full' project with subject and session > 1 in both local
        and central projects. Then, delete the local and run get next sub / ses,
        and make the project local-only. Call validation requesting to also
        check central path, which should be ignored as we are in local-only mode.
        """
        test_utils.make_local_folders_with_files_in(
            project,
            top_level_folder,
            subs=["001", "002", "003"],
            sessions=["01", "02"],
        )

        project.upload_entire_project()

        shutil.rmtree(project.cfg["local_path"] / top_level_folder)

        project.update_config_file(central_path=None, connection_method=None)

        next_sub = project.get_next_sub(
            top_level_folder, include_central=False
        )

        next_ses = project.get_next_ses(
            top_level_folder, sub="001", include_central=False
        )

        assert next_sub == "sub-001"
        assert next_ses == "ses-001"