Add Python versions 3.13 and 3.14 to CI workflow (#828)

* Add Python versions 3.13 and 3.14 to CI workflow

Upper limit removed in #826

* Migrate ete3 → ete4 for Python 3.13+ compatibility (#829)

* Initial plan

* Migrate from ete3 to ete4 for Python 3.13+ compatibility

Co-authored-by: tanghaibao <106987+tanghaibao@users.noreply.github.com>

* Style fixes by Black

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: tanghaibao <106987+tanghaibao@users.noreply.github.com>

* Fix bam2mat tests failing on Python 3.13 due to pysam iterator ValueError (#830)

* Initial plan

* Fix bam2mat ValueError with Python 3.13 + pysam + pytest-cov

Co-authored-by: tanghaibao <106987+tanghaibao@users.noreply.github.com>

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: tanghaibao <106987+tanghaibao@users.noreply.github.com>

---------

Co-authored-by: Copilot <198982749+Copilot@users.noreply.github.com>
Co-authored-by: tanghaibao <106987+tanghaibao@users.noreply.github.com>

Author: 3 people

.github/workflows/pytest.yml

@@ -11,7 +11,7 @@ jobs:
       max-parallel: 4
       matrix:
         os: [ubuntu-latest, macos-latest]
-        python-version: ["3.10", "3.11", "3.12"]
+        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
     steps:
       - uses: actions/checkout@v6
       - name: Set up Python ${{ matrix.python-version }}
@@ -48,7 +48,7 @@ jobs:
 # Only upload coverage for the latest Python version on Ubuntu
 #- name: Upload coverage reports to Codecov
 #  uses: codecov/codecov-action@v5
-#  if: matrix.python-version == '3.12' && matrix.os == 'ubuntu-latest'
+#  if: matrix.python-version == '3.14' && matrix.os == 'ubuntu-latest'
 #  with:
 #    token: ${{ secrets.CODECOV_TOKEN }}
 #    slug: tanghaibao/jcvi

pyproject.toml

@@ -37,7 +37,7 @@ dependencies = [
     "CrossMap",
     "cython",
     "deap",
-    "ete3",
+    "ete4",
     "ftpretty",
     "genomepy",
     "gffutils",
@@ -125,7 +125,6 @@ addopts = "-v --cov=jcvi --cov-branch --cov-report=xml --cov-report=term"
 testpaths = ["tests"]
 python_files = ["test_*.py"]
 filterwarnings = [
-    "ignore:invalid escape sequence.*:SyntaxWarning:ete3\\..*",
 ]
 
 [tool.isort]

src/jcvi/apps/phylo.py

@@ -41,7 +41,7 @@
     FSeqBootCommandline,
 )
 from Bio.Phylo.Applications import PhymlCommandline, RaxmlCommandline
-from ete3 import Tree
+from ete4 import Tree
 import numpy as np
 
 from ..compara.ks import (
@@ -252,8 +252,8 @@ def smart_reroot(treefile, outgroupfile, outfile, format=0):
     Tree reading format options see here:
     http://packages.python.org/ete2/tutorial/tutorial_trees.html#reading-newick-trees
     """
-    tree = Tree(treefile, format=format)
-    leaves = [t.name for t in tree.get_leaves()][::-1]
+    tree = Tree(treefile, parser=format)
+    leaves = [t.name for t in list(tree.leaves())][::-1]
     outgroup = []
     for o in must_open(outgroupfile):
         o = o.strip()
@@ -271,12 +271,12 @@ def smart_reroot(treefile, outgroupfile, outfile, format=0):
         return treefile
 
     try:
-        tree.set_outgroup(tree.get_common_ancestor(*outgroup))
+        tree.set_outgroup(tree.common_ancestor(*outgroup))
     except ValueError:
         assert type(outgroup) == list
         outgroup = outgroup[0]
         tree.set_outgroup(outgroup)
-    tree.write(outfile=outfile, format=format)
+    tree.write(outfile=outfile, parser=format)
 
     logger.debug("Rerooted tree printed to {0}".format(outfile))
     return outfile
@@ -374,9 +374,9 @@ def build_nj_phylip(alignment, outfile, outgroup, work_dir="."):
             nodesupport[node_children] = node.dist / 100.0
 
     for k, v in nodesupport.items():
-        ct_b.get_common_ancestor(*k).support = v
+        ct_b.common_ancestor(*k).support = v
     print(ct_b)
-    ct_b.write(format=0, outfile=outfile)
+    ct_b.write(outfile=outfile, parser=0)
 
     try:
         s = op.getsize(outfile)

src/jcvi/assembly/hic.py

@@ -1007,42 +1007,50 @@ def distbin_number(dist, start=minsize, ratio=1.01):
     # Check all reads, rules borrowed from LACHESIS
     # https://github.com/shendurelab/LACHESIS/blob/master/src/GenomeLinkMatrix.cc#L1476
     j = k = 0
-    for c in bamfile:
-        j += 1
-        if j % 100000 == 0:
-            print("{} reads counted".format(j), file=sys.stderr)
+    try:
+        for c in bamfile:
+            j += 1
+            if j % 100000 == 0:
+                print("{} reads counted".format(j), file=sys.stderr)
 
-        if c.is_qcfail and c.is_duplicate:
-            continue
-        if c.is_secondary and c.is_supplementary:
-            continue
-        if c.mapping_quality == 0:
-            continue
-        if not c.is_paired:
-            continue
-        if c.is_read2:  # Take only one read
-            continue
-
-        # pysam v0.8.3 does not support keyword reference_name
-        achr = bamfile.getrname(c.reference_id)
-        apos = c.reference_start
-        bchr = bamfile.getrname(c.next_reference_id)
-        bpos = c.next_reference_start
-        if achr not in seqstarts or bchr not in seqstarts:
-            continue
-        if achr == bchr:
-            dist = abs(apos - bpos)
-            if dist < minsize:
+            if c.is_qcfail and c.is_duplicate:
+                continue
+            if c.is_secondary and c.is_supplementary:
+                continue
+            if c.mapping_quality == 0:
+                continue
+            if not c.is_paired:
+                continue
+            if c.is_read2:  # Take only one read
                 continue
-            db = distbin_number(dist)
-            B[db] += 1
-
-        abin, bbin = bin_number(achr, apos), bin_number(bchr, bpos)
-        A[abin, bbin] += 1
-        if abin != bbin:
-            A[bbin, abin] += 1
 
-        k += 1
+            # pysam v0.8.3 does not support keyword reference_name
+            achr = bamfile.getrname(c.reference_id)
+            apos = c.reference_start
+            bchr = bamfile.getrname(c.next_reference_id)
+            bpos = c.next_reference_start
+            if achr not in seqstarts or bchr not in seqstarts:
+                continue
+            if achr == bchr:
+                dist = abs(apos - bpos)
+                if dist < minsize:
+                    continue
+                db = distbin_number(dist)
+                B[db] += 1
+
+            abin, bbin = bin_number(achr, apos), bin_number(bchr, bpos)
+            A[abin, bbin] += 1
+            if abin != bbin:
+                A[bbin, abin] += 1
+
+            k += 1
+    except ValueError as e:
+        # pysam >= 0.23.1 raises ValueError("Firing event 10 with no exception set")
+        # at iterator exhaustion on Python 3.13 when pytest-cov is active.
+        # All records have been processed at this point, so we only re-raise
+        # unexpected errors. See: https://github.com/pysam-developers/pysam/issues/1350
+        if "Firing event 10 with no exception set" not in str(e):
+            raise
 
     logger.debug("Total reads counted: %s", percentage(2 * k, j))
     bamfile.close()

src/jcvi/formats/cdt.py

@@ -45,7 +45,7 @@ def get_names(self):
 
     def get_gtr_tree(self):
 
-        from ete3 import Tree
+        from ete4 import Tree
 
         fp = open(self.gtrfile)
         reader = csv.reader(fp, delimiter="\t")
@@ -68,7 +68,7 @@ def get_gtr_tree(self):
 
     def print_newick(self, nwk_file):
 
-        self.gtr_tree.write(format=5, outfile=nwk_file)
+        self.gtr_tree.write(outfile=nwk_file, parser=5)
         logger.debug("Newick tree written to `%s`", nwk_file)
 
     def iter_partitions(self, cutoff=0.3, gtr=True):

src/jcvi/graphics/tree.py

@@ -5,7 +5,7 @@
 from itertools import groupby
 import sys
 
-from ete3 import Tree
+from ete4 import Tree
 
 from ..apps.base import OptionParser, glob, logger
 from ..formats.base import LineFile
@@ -177,12 +177,20 @@ def draw_tree(
 
     if reroot:
         if outgroup:
-            R = t.get_common_ancestor(*outgroup)
+            R = t.common_ancestor(*outgroup)
         else:
             # Calculate the midpoint node
             R = t.get_midpoint_outgroup()
 
         if R is not t:
+            # ete4 requires consistent support at root before rerooting
+            root_children_support = [c.support for c in t.children]
+            if None in root_children_support and any(
+                s is not None for s in root_children_support
+            ):
+                for child in t.children:
+                    if child.support is None:
+                        child.support = 1.0
             t.set_outgroup(R)
 
         # By default, the distance to outgroup and non-outgroup is the same
@@ -192,7 +200,7 @@ def draw_tree(
             a, b = t.children
             # Avoid even split
             total = a.dist + b.dist
-            newR = t.get_common_ancestor(*outgroup)
+            newR = t.common_ancestor(*outgroup)
             a.dist = 0.9 * total
             b.dist = total - a.dist
 
@@ -210,7 +218,7 @@ def rescale(dist):
     def rescale_divergence(divergence):
         return rescale(max_dist - divergence)
 
-    num_leaves = len(t.get_leaf_names())
+    num_leaves = len(list(t.leaf_names()))
     yinterval = (1 - ystart) / num_leaves
     ytop = ystart + (num_leaves - 0.5) * yinterval
 
@@ -228,10 +236,10 @@ def rescale_divergence(divergence):
     i = 0
     color_groups = []  # Used to plot groups to the right of the tree
     for n in t.traverse("postorder"):
-        dist = n.get_distance(t)
+        dist = t.get_distance(n, t)
         xx = rescale(dist)
 
-        if n.is_leaf():
+        if n.is_leaf:
             yy = ystart + i * yinterval
             i += 1
 
@@ -304,10 +312,10 @@ def rescale_divergence(divergence):
                     alpha=0.4,
                     lw=2,
                 )
-            support = n.support
+            support = n.support if n.support is not None else 0
             if support > 1:
                 support = support / 100.0
-            if not n.is_root() and supportcolor:
+            if not n.is_root and supportcolor:
                 if support > scutoff / 100.0:
                     ax.text(
                         xx,
@@ -535,7 +543,7 @@ def repl(match):
     if repl.hpd:
         print(repl.hpd, file=sys.stderr)
 
-    return (Tree(treedata, format=1), repl.hpd) if changed else (Tree(treedata), None)
+    return (Tree(treedata, parser=1), repl.hpd) if changed else (Tree(treedata), None)
 
 
 def main(args):

src/jcvi/utils/taxonomy.py

@@ -35,7 +35,7 @@
 
 from BeautifulSoup import BeautifulSoup
 from ClientForm import ParseResponse
-from ete3 import Tree
+from ete4 import Tree
 
 from ..apps.base import ActionDispatcher, OptionParser, logger
 
@@ -85,7 +85,7 @@ def __str__(self):
         return self.newick
 
     def print_tree(self):
-        t = Tree(self.newick, format=8)
+        t = Tree(self.newick, parser=8)
         print(t)
 
 
@@ -122,9 +122,9 @@ def MRCA(list_of_taxids):
     """
 
     t = TaxIDTree(list_of_taxids)
-    t = Tree(str(t), format=8)
+    t = Tree(str(t), parser=8)
 
-    ancestor = t.get_common_ancestor(*t.get_leaves())
+    ancestor = t.common_ancestor(*list(t.leaves()))
 
     return ancestor.name
 

tests/graphics/test_tree.py

@@ -3,15 +3,81 @@
 
 import os
 import os.path as op
+import tempfile
 
+import matplotlib
 
-def disabled_test_tree_main():
+matplotlib.use("Agg")
+
+import pytest
+
+
+def test_tree_main(tmp_path):
+    """Render the built-in demo tree to a PNG and check the file is created."""
     from jcvi.apps.base import cleanup
     from jcvi.graphics.tree import main as tree_main
 
-    demo = "demo.png"
+    demo = str(tmp_path / "demo.png")
     cleanup(demo)
 
+    os.chdir(tmp_path)
     tree_main(["demo", "--format", "png"])
-    assert op.exists(demo)
-    cleanup(demo)
+    assert op.exists(str(tmp_path / "demo.png"))
+
+
+def test_tree_parse():
+    """Test that the ete4 Tree can be created from a newick string."""
+    from ete4 import Tree
+
+    newick = "(((A:0.1,B:0.2)90:0.3,(C:0.15,D:0.25)80:0.2)100:0.05,E:0.4);"
+    t = Tree(newick)
+    assert t is not None
+    leaves = list(t.leaf_names())
+    assert sorted(leaves) == ["A", "B", "C", "D", "E"]
+
+
+def test_tree_parse_tree_function():
+    """Test parse_tree handles a plain newick file (no HPD)."""
+    from jcvi.graphics.tree import parse_tree
+
+    newick = "(((A:0.1,B:0.2)90:0.3,(C:0.15,D:0.25)80:0.2)100:0.05,E:0.4);"
+    with tempfile.NamedTemporaryFile(mode="w", suffix=".nwk", delete=False) as f:
+        f.write(newick)
+        fname = f.name
+    try:
+        t, hpd = parse_tree(fname)
+        assert hpd is None
+        leaves = list(t.leaf_names())
+        assert sorted(leaves) == ["A", "B", "C", "D", "E"]
+    finally:
+        os.unlink(fname)
+
+
+def test_tree_truncate_name():
+    """Test the truncate_name helper function."""
+    from jcvi.graphics.tree import truncate_name
+
+    assert truncate_name("ABCDEF", rule=None) == "ABCDEF"
+    assert truncate_name("ABCDEF", rule="head3") == "DEF"
+    assert truncate_name("ABCDEF", rule="ohead3") == "ABC"
+    assert truncate_name("ABCDEF", rule="tail2") == "ABCD"
+    assert truncate_name("ABCDEF", rule="otail2") == "EF"
+
+
+def test_tree_draw(tmp_path):
+    """Test that draw_tree runs without error on a simple tree."""
+    import matplotlib.pyplot as plt
+    from ete4 import Tree
+
+    from jcvi.graphics.tree import draw_tree
+
+    newick = "(((A:0.1,B:0.2)0.9:0.3,(C:0.15,D:0.25)0.8:0.2)1.0:0.05,E:0.4);"
+    t = Tree(newick)
+
+    fig = plt.figure(figsize=(8, 6))
+    ax = fig.add_axes([0, 0, 1, 1])
+    draw_tree(ax, t)
+    out = str(tmp_path / "tree_draw.png")
+    fig.savefig(out)
+    plt.close(fig)
+    assert op.exists(out)