@@ -371,7 +371,7 @@ def download_and_generate_annotation(genomes_dir, annot_url, localname):
371371 elif "gff" in ext :
372372 cmd = "gff3ToGenePred -geneNameAttr=gene {0} {1}"
373373 elif "gtf" in ext :
374- cmd = "gtfToGenePred {0} {1}"
374+ cmd = "gtfToGenePred -ignoreGroupsWithoutExons {0} {1}"
375375 elif "txt" in ext :
376376 # UCSC annotations only
377377 with open (annot_file ) as f :
@@ -524,7 +524,7 @@ def search(self, term):
524524 term = safe (str (term ))
525525 if term .startswith ("GCA_" ) and self .name != "NCBI" :
526526 for row in self ._search_accessions (term ):
527- yield ( row )
527+ yield row
528528
529529 elif is_number (term ):
530530 for name in genomes :
@@ -1055,14 +1055,10 @@ def get_genome_download_link(self, name, mask="soft", **kwargs):
10551055 ------
10561056 str with the http/ftp download link.
10571057 """
1058- genome = self .genomes [safe (name )]
1058+ # only soft masked genomes available. can be (un)masked in _post_process_download
1059+ link = self ._ftp_or_html_link (name , file_suffix = "_genomic.fna.gz" )
10591060
1060- # only soft masked genomes available. can be (un)masked in _post _process_download
1061- link = genome ["ftp_path" ]
1062- link = link .replace ("ftp://" , "https://" )
1063- link += "/" + link .split ("/" )[- 1 ] + "_genomic.fna.gz"
1064-
1065- if check_url (link , 2 ):
1061+ if link :
10661062 return link
10671063
10681064 raise GenomeDownloadError (
@@ -1092,10 +1088,9 @@ def _post_process_download(self, name, localname, out_dir, mask="soft"):
10921088 masking level: soft/hard/none, default=soft
10931089 """
10941090 # Create mapping of accessions to names
1095- genome = self .genomes [safe (name )]
1096- url = genome ["ftp_path" ]
1097- url += f"/{ url .split ('/' )[- 1 ]}
1098- url = url .replace ("ftp://" , "https://" )
1091+ url = self ._ftp_or_html_link (
1092+ name , file_suffix = "_assembly_report.txt" , skip_check = True
1093+ )
10991094
11001095 tr = {}
11011096 urlcleanup ()
@@ -1148,13 +1143,21 @@ def get_annotation_download_link(self, name, **kwargs):
11481143 name : str
11491144 Genome name
11501145 """
1146+ return self ._ftp_or_html_link (name , file_suffix = "_genomic.gff.gz" )
1147+
1148+ def _ftp_or_html_link (self , name , file_suffix , skip_check = False ):
1149+ """
1150+ NCBI's files are accessible over FTP and HTTPS
1151+ Try HTTPS first and return the first functioning link
1152+ """
11511153 genome = self .genomes [safe (name )]
1152- link = genome ["ftp_path" ]
1153- link = link .replace ("ftp://" , "https://" )
1154- link += "/" + link .split ("/" )[- 1 ] + "_genomic.gff.gz"
1154+ ftp_link = genome ["ftp_path" ]
1155+ html_link = ftp_link .replace ("ftp://" , "https://" )
1156+ for link in [html_link , ftp_link ]:
1157+ link += "/" + link .split ("/" )[- 1 ] + file_suffix
11551158
1156- if check_url (link , 2 ):
1157- return link
1159+ if skip_check or check_url (link , max_tries = 2 , timeout = 10 ):
1160+ return link
11581161
11591162
11601163@register_provider ("URL" )
@@ -1188,6 +1191,9 @@ def search(self, term):
11881191 same as if no genomes were found at the other providers"""
11891192 yield from ()
11901193
1194+ def _genome_info_tuple (self , name ):
1195+ return tuple ()
1196+
11911197 def get_genome_download_link (self , url , mask = None , ** kwargs ):
11921198 return url
11931199
@@ -1203,55 +1209,41 @@ def get_annotation_download_link(self, name, **kwargs):
12031209 "Only (gzipped) gtf, gff and bed files are supported.\n "
12041210 )
12051211
1206- if check_url (link ):
1207- return link
1212+ return link
12081213
12091214 @staticmethod
1210- def search_url_for_annotation (url ):
1211- """Attempts to find a gtf or gff3 file in the same location as the genome url"""
1215+ def search_url_for_annotations (url , name ):
1216+ """Attempts to find gtf or gff3 files in the same location as the genome url"""
12121217 urldir = os .path .dirname (url )
12131218 sys .stderr .write (
1214- "You have requested gene annotation to be downloaded.\n "
1219+ "You have requested the gene annotation to be downloaded.\n "
12151220 "Genomepy will check the remote directory:\n "
12161221 f"{ urldir } \n "
12171222 )
12181223
1219- # try to find a GTF or GFF3 file
1220- name = get_localname (url )
1221- with urlopen (urldir ) as f :
1222- for urlline in f .readlines ():
1223- urlstr = str (urlline )
1224- if any (
1225- substring in urlstr .lower () for substring in [".gtf" , name + ".gff" ]
1226- ):
1227- break
1224+ def fuzzy_annotation_search (search_name , search_list ):
1225+ """Returns all files containing both name and an annotation extension"""
1226+ hits = []
1227+ for ext in ["gtf" , "gff" ]:
1228+ # .*? = non greedy filler. 3? = optional 3 (for gff3). (\.gz)? = optional .gz
1229+ expr = f"{ search_name } { ext } # noqa: W605
1230+ for line in search_list :
1231+ hit = re .search (expr , line , flags = re .IGNORECASE )
1232+ if hit :
1233+ hits .append (hit [0 ])
1234+ return hits
12281235
1229- # retrieve the filename from the HTML line
1230- fname = ""
1231- for split in re .split ('>|<|><|/|"' , urlstr ):
1232- if split .lower ().endswith (
1233- (
1234- ".gtf" ,
1235- ".gtf.gz" ,
1236- name + ".gff" ,
1237- name + ".gff.gz" ,
1238- name + ".gff3" ,
1239- name + ".gff3.gz" ,
1240- )
1241- ):
1242- fname = split
1243- break
1244- else :
1236+ # try to find a GTF or GFF3 file
1237+ dirty_list = [str (line ) for line in urlopen (urldir ).readlines ()]
1238+ fnames = fuzzy_annotation_search (name , dirty_list )
1239+ if not fnames :
12451240 raise FileNotFoundError (
12461241 "Could not parse the remote directory. "
12471242 "Please supply a URL using --url-to-annotation.\n "
12481243 )
12491244
1250- # set variables for downloading
1251- link = urldir + "/" + fname
1252-
1253- if check_url (link ):
1254- return link
1245+ links = [urldir + "/" + fname for fname in fnames ]
1246+ return links
12551247
12561248 def download_annotation (self , url , genomes_dir = None , localname = None , ** kwargs ):
12571249 """
@@ -1279,8 +1271,20 @@ def download_annotation(self, url, genomes_dir=None, localname=None, **kwargs):
12791271 genomes_dir = get_genomes_dir (genomes_dir , check_exist = False )
12801272
12811273 if kwargs .get ("to_annotation" ):
1282- link = self .get_annotation_download_link (None , ** kwargs )
1274+ links = [ self .get_annotation_download_link (None , ** kwargs )]
12831275 else :
1284- link = self .search_url_for_annotation (url )
1276+ # can return multiple possible hits
1277+ links = self .search_url_for_annotations (url , name )
12851278
1286- self .attempt_and_report (name , localname , link , genomes_dir )
1279+ for link in links :
1280+ try :
1281+ self .attempt_and_report (name , localname , link , genomes_dir )
1282+ break
1283+ except GenomeDownloadError as e :
1284+ if not link == links [- 1 ]:
1285+ sys .stdout .write (
1286+ "\n One of the potential annotations was incompatible with genomepy."
1287+ + "\n Attempting another...\n \n "
1288+ )
1289+ continue
1290+ return e
0 commit comments