Skip to content

ENA upload tool - move credentials to requirements #7606

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bgruening merged 16 commits into from
Mar 11, 2026
Merged

ENA upload tool - move credentials to requirements #7606

Changes from 10 commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
7b25ee1
ENA upload tool - move credentials to requirements
RZ9082 Jan 21, 2026
469f467
Bump suffix
RZ9082 Jan 22, 2026
01ed2e5
Remove unnecessary test
RZ9082 Jan 22, 2026
e6f2ac0
Adjust tests index
RZ9082 Jan 22, 2026
622a5cd
Change username to variable
RZ9082 Jan 22, 2026
715d779
Use configfiles to create credenials.yml
RZ9082 Jan 23, 2026
53553e0
Bump profile
RZ9082 Jan 23, 2026
7a3da89
Add credential variables default values
RZ9082 Jan 23, 2026
e16d361
Username is now a variable
RZ9082 Jan 23, 2026
3004c44
Fix typo
RZ9082 Jan 23, 2026
730acd1
Remove credentials configfile
RZ9082 Mar 9, 2026
3c78860
Remove secrects flag from command section
RZ9082 Mar 10, 2026
742e054
Update tools/ena_upload/ena_upload.xml
SaimMomin12 Mar 10, 2026
858196a
Update tools/ena_upload/ena_upload.xml
SaimMomin12 Mar 10, 2026
e91537e
Remove README
RZ9082 Mar 11, 2026
c6304bd
Add required_files
RZ9082 Mar 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
155 changes: 44 additions & 111 deletions tools/ena_upload/ena_upload.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,19 +1,23 @@
<tool id="ena_upload" name="ENA Upload tool" version="@TOOL_VERSION@+galaxy@VERSION_SUFFIX@" profile="20.05" license="MIT">
<tool id="ena_upload" name="ENA Upload tool" version="@TOOL_VERSION@+galaxy@VERSION_SUFFIX@" profile="25.1" license="MIT">
Comment thread
SaimMomin12 marked this conversation as resolved.
Outdated
<description>Submission of (meta)data to the European Nucleotide Archive (ENA)</description>
<macros>
<token name="@TOOL_VERSION@">0.9.0</token>
<token name="@VERSION_SUFFIX@">0</token>
<token name="@VERSION_SUFFIX@">1</token>
Comment thread
SaimMomin12 marked this conversation as resolved.
<import>samples_macros.xml</import>
</macros>
<requirements>
<requirement type="package" version="@TOOL_VERSION@">ena-upload-cli</requirement>
<credentials name="ena_credentials" version="1.0" label="ENA (European Nucleotide Archive) credentials">
<secret name="ena_usename" inject_as_env="ENA_USERNAME" optional="false" label="ENA Webin username"/>
<secret name="ena_password" inject_as_env="ENA_PASSWORD" optional="false" label="ENA Webin password"/>
</credentials>
</requirements>
<stdio>
<regex match="Oops" source="stderr" level="fatal"/>
<regex match="different file names between command line and RUN table" source="stderr" level="fatal"/>
</stdio>
<command detect_errors="exit_code"><![CDATA[
mkdir ./submission_files;
mkdir ./submission_files;
#set $studies_table_path = './submission_files/studies.tsv'
#set $samples_table_path = './submission_files/samples.tsv'
#set $experiments_table_path = './submission_files/experiments.tsv'
Expand All @@ -29,11 +33,6 @@ mkdir ./submission_files;
python '$__tool_directory__/extract_tables.py' --action $action_options.action --out_dir ./submission_files --studies $studies_json;
#end if

credentials_path='test_fake_path';
echo "username: test_user" > \$credentials_path;
echo "password: test_password" >> \$credentials_path;


#if $action_options.input_format_conditional.input_format == "build_tables":
cp $studies_table_path $studies_table_out &&
cp $samples_table_path $samples_table_out &&
Expand All @@ -53,25 +52,9 @@ echo "password: test_password" >> \$credentials_path;
ln -s '$action_options.input_format_conditional.isa_json_file' ./isa_json_input.json &&
#end if
#if $action_options.test_submit_parameters.dry_run == "False":
webin_id=`grep 'username' $credentials`;
if [ "\$webin_id" = "" ]; then
## No credentials in user defined preferences
## Fallback to global defined credentials (if exist)
#import os
#if os.path.isfile(os.environ.get('GALAXY_ENA_SECRETS', '')):
credentials_path=\${GALAXY_ENA_SECRETS};
webin_id=`grep 'username' \$GALAXY_ENA_SECRETS`;
if [ "\$webin_id" = "" ]; then
echo "No global credentials defined. Check your GALAXY_ENA_SECRETS file or set your credentials via: User -> Preferences -> Manage Information";
exit 1;
fi;
#else:
echo "No ENA credentials defined. Set your credentials via: User -> Preferences -> Manage Information";
exit 1;
#end if
else
credentials_path='$credentials';
fi;
python '$credentials' &&
#else
python '$credentials' --dry_run &&
#end if


Expand Down Expand Up @@ -179,7 +162,7 @@ ena-upload-cli
--tool 'ena-upload-cli v@TOOL_VERSION@ @ Galaxy'
--action '$action_options.action'
--center '$action_options.test_submit_parameters.center'
--secret \${credentials_path}
--secret credentials.yml
--data
#for $dataset in $files_to_upload:
'$dataset'
Expand Down Expand Up @@ -217,7 +200,7 @@ ena-upload-cli
echo -e 'center_name\t$action_options.test_submit_parameters.center' >> '$output';
echo -e 'action_option\t$action_options.action' >> '$output';
#end if

## copy updated files
#if $action_options.input_format_conditional.input_format == "excel_tables" or $action_options.input_format_conditional.input_format == "isa_json":
## for the excel/ISA json input case, copy the upload-cli generated tables to the output files
Expand All @@ -237,13 +220,33 @@ python '$__tool_directory__/dump_yaml.py' $studies_table_out $samples_table_out
]]></command>
<configfiles>
<configfile name="credentials"><![CDATA[
#set $webin_id = $__user__.extra_preferences.get('ena_webin_account|webin_id', "").strip()
#set $password = $__user__.extra_preferences.get('ena_webin_account|password', "").strip()
#if $webin_id != "":
username: "$webin_id"
password: "$password"
#end if
]]></configfile>
import os
import argparse
import yaml

parser = argparse.ArgumentParser(description="Create credentials.yml for ENA access")
parser.add_argument(
"--dry_run",
action="store_true",
)
args = parser.parse_args()

if args.dry_run:
credentials = {
"username": "test_user",
"password": "test_password"
}
else:
username = os.environ.get("ENA_USERNAME", "")

@mvdbeek mvdbeek Jan 26, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you add reading the credentials from env vars to https://github.com/usegalaxy-eu/ena-upload-cli instead ?

@bernt-matthias bernt-matthias Jan 27, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you add reading the credentials from env vars to https://github.com/usegalaxy-eu/ena-upload-cli instead

This would be even better, indeed. Guess we should ask upstream in any case.

Just before I forget it: If this appoach is still needed I would suggest to use bash for creating the config file (since python might add another requirement).

@RZ9082 RZ9082 Jan 29, 2026 •
edited
Loading

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done here: usegalaxy-eu/ena-upload-cli#121

@bedroesb bedroesb Mar 6, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@RZ9082 I made a release with the feature, v0.10.0. I will let you know if it is live on BioConda

@bedroesb bedroesb Mar 6, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

One merge away: bioconda/bioconda-recipes#63307 (review)

password = os.environ.get("ENA_PASSWORD", "")
credentials = {
"username": username,
"password": password
}

with open("credentials.yml", "w") as f:
yaml.dump(credentials, f, default_flow_style=False)
]]></configfile>
<configfile name="studies_json">
#import json
#import re
Expand Down Expand Up @@ -473,77 +476,7 @@ $samples.append({'title':str($sample.sample_title),'description':str($sample.sam
</assert_contents>
</output>
</test>
<!--Test 4: failure on excel input of NON-VIRAL samples with runs PAIRED collection -->
Comment thread
RZ9082 marked this conversation as resolved.
<test expect_failure="True">
<conditional name="action_options">
<param name="action" value="add"/>
<section name="test_submit_parameters">
<param name="center" value="Some research center"/>
<param name="submit_dev" value="False" />
<param name="dry_run" value="False" />
</section>
<conditional name="input_format_conditional">
<param name="add_extension" value="True"/>
<param name="input_format" value="excel_tables"/>
<param name="checklist_id" value="ERC000011"/>
<param name="xlsx_file" value="metadata_test_nonviral.xlsx"/>
<conditional name="run_input_format_conditional">
<param name="run_input_format" value="paired_list"/>
<param name="paired_end_collection">
<collection type="list:paired">
<element name="paired_run_name">
<collection type="paired">
<element name="forward" value="1.fastqsanger.gz" ftype="fastqsanger.gz" />
<element name="reverse" value="2.fastqsanger.gz" ftype="fastqsanger.gz" />
</collection>
</element>
</collection>
</param>
</conditional>
</conditional>
</conditional>
<assert_command>
<has_text_matching expression="ena-upload-cli"/>
<has_text_matching expression="--data 'paired_run_name_1.fastq.gz' 'paired_run_name_2.fastq.gz'"/>
<has_text_matching expression="--action 'add' --center 'Some research center'"/>
</assert_command>
</test>
<!--Test 4b: failure on excel input of NON-VIRAL samples with runs PAIRED collection -->
<test expect_failure="True">
<conditional name="action_options">
<param name="action" value="add"/>
<section name="test_submit_parameters">
<param name="center" value="Some research center"/>
<param name="submit_dev" value="False" />
<param name="dry_run" value="False" />
</section>
<conditional name="input_format_conditional">
<param name="add_extension" value="True"/>
<param name="input_format" value="excel_tables"/>
<param name="checklist_id" value="ERC000011"/>
<param name="xlsx_file" value="metadata_test_nonviral.xlsx"/>
<conditional name="run_input_format_conditional">
<param name="run_input_format" value="paired_list"/>
<param name="paired_end_collection">
<collection type="list:paired">
<element name="paired_run_name">
<collection type="paired">
<element name="forward" value="1.fastqsanger.gz" ftype="fastqsanger.gz" />
<element name="reverse" value="2.fastqsanger.gz" ftype="fastqsanger.gz" />
</collection>
</element>
</collection>
</param>
</conditional>
</conditional>
</conditional>
<assert_command>
<has_text_matching expression="ena-upload-cli"/>
<has_text_matching expression="--data 'paired_run_name_1.fastq.gz' 'paired_run_name_2.fastq.gz'"/>
<has_text_matching expression="--action 'add' --center 'Some research center'"/>
</assert_command>
</test>
<!--Test 5: build tables from user input fields NON-VIRAL samples-->
<!--Test 4: build tables from user input fields NON-VIRAL samples-->
<test expect_num_outputs="5">
<conditional name="action_options">
<param name="action" value="add"/>
Expand Down Expand Up @@ -616,7 +549,7 @@ $samples.append({'title':str($sample.sample_title),'description':str($sample.sam
</assert_contents>
</output>
</test>
<!--Test 6: with submit_test to skip credentials checksRUN failing build tables from user input fields NON-VIRAL samples
<!--Test 5: with submit_test to skip credentials checksRUN failing build tables from user input fields NON-VIRAL samples
also tests compression of uncompressed inputs and adding the .gz suffix -->
<test expect_failure="False" expect_num_outputs="5">
<conditional name="action_options">
Expand Down Expand Up @@ -672,7 +605,7 @@ $samples.append({'title':str($sample.sample_title),'description':str($sample.sam
<not_has_text text="modify" />
</assert_command>
</test>
<!--Test 7: viral submission - User input metadata - Add extension = False-->
<!--Test 6: viral submission - User input metadata - Add extension = False-->
<test expect_failure="False" expect_num_outputs="5">
<conditional name="action_options">
<param name="action" value="add"/>
Expand Down Expand Up @@ -735,7 +668,7 @@ $samples.append({'title':str($sample.sample_title),'description':str($sample.sam
<has_text_matching expression="--checklist ERC000033"/>
</assert_command>
</test>
<!--Test 8: modify option and auto compression - viral submission - User input metadata-->
<!--Test 7: modify option and auto compression - viral submission - User input metadata-->
<test expect_failure="False" expect_num_outputs="5">
<conditional name="action_options">
<param name="action" value="modify"/>
Expand Down Expand Up @@ -799,7 +732,7 @@ $samples.append({'title':str($sample.sample_title),'description':str($sample.sam
<not_has_text text="--action 'add'" />
</assert_command>
</test>
<!--Test 9: isa JSON input of NON-VIRAL samples-->
<!--Test 8: isa JSON input of NON-VIRAL samples-->
<test expect_num_outputs="5">
<conditional name="action_options">
<param name="action" value="add"/>
Expand Down