Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7,24 +7,24 @@ repos:
- id: fix-future-annotations

- repo: https://github.com/psf/black
rev: 24.4.2
rev: 26.5.1
hooks:
- id: black

- repo: https://github.com/PyCQA/flake8
rev: 7.0.0
rev: 7.3.0
hooks:
- id: flake8
additional_dependencies:
- flake8-bugbear

- repo: https://github.com/pycqa/isort
rev: 5.13.2
rev: 9.0.0b2
hooks:
- id: isort

- repo: https://github.com/pre-commit/mirrors-mypy
rev: v1.10.0
rev: v2.3.0
hooks:
- id: mypy
exclude: ^docs/
Expand Down
12 changes: 12 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,15 @@
## v2.2.4(2026-08-12)

### Added

- Add source mapping for AST elements. Every element now carries a `source_span` attribute (a `(start, end)` tuple of indices into the source text) plus the `start_pos`/`end_pos` convenience properties, and inline elements expose their pure-syntax parts (e.g. emphasis markers, code span backticks, link brackets) via `syntax_spans`. This enables features like syntax highlighting. [#273](https://github.com/frostming/marko/issues/273)

### Fixed

- Fix a quadratic-time linebreak parsing issue when the input contains no newline character. [#219](https://github.com/frostming/marko/issues/219)
- Preserve loose list spacing when re-rendering Markdown with `MarkdownRenderer`. [#272](https://github.com/frostming/marko/pull/272)
- Fix the broken tests badge link in the docs index page. [#207](https://github.com/frostming/marko/issues/207)

## v2.2.3(2026-05-28)

### Changed
Expand Down
6 changes: 3 additions & 3 deletions examples/highlight.py
Original file line number Diff line number Diff line change
Expand Up @@ -351,12 +351,12 @@ def render_element(el: Element, text: str, b: HtmlBuilder) -> None:
for a, c in zip(cuts, cuts[1:]):
if a >= c:
continue
child = next(
matching_child = next(
(ch for ch, (cs, ce) in zip(children, child_spans) if cs <= a and ce >= c),
None,
)
if child is not None:
render_element(child, text, b)
if matching_child is not None:
render_element(matching_child, text, b)
continue
spcls = next((k for x, y, k in special if x <= a and y >= c), None)
if spcls is not None:
Expand Down
15 changes: 8 additions & 7 deletions marko/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@

from __future__ import annotations

from typing import TYPE_CHECKING, Iterable, cast
from collections.abc import Iterable
from typing import TYPE_CHECKING, cast

from .helpers import MarkoExtension, load_extension
from .html_renderer import HTMLRenderer
Expand All @@ -23,7 +24,7 @@
from .block import Document
from .parser import ElementType

__version__ = "2.2.3"
__version__ = "2.2.4"


class SetupDone(Exception):
Expand Down Expand Up @@ -166,13 +167,13 @@ def render(parsed: Document) -> str:


__all__ = [
"MarkoExtension",
"HTMLRenderer",
"Markdown",
"MarkoExtension",
"Parser",
"Renderer",
"convert",
"load_extension",
"parse",
"render",
"load_extension",
"Parser",
"Renderer",
"HTMLRenderer",
]
11 changes: 4 additions & 7 deletions marko/ext/footnote.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
print(markdown(text))

"""

from __future__ import annotations

import re
Expand Down Expand Up @@ -76,10 +77,8 @@ def render_footnote_ref(self, element):
self.footnotes.append(element.label)
idx = self.footnotes.index(element.label) + 1
return (
'<sup class="footnote-ref" id="fnref-{lab}">'
'<a href="#fn-{lab}">{id}</a></sup>'.format(
lab=self.escape_url(element.label), id=idx
)
f'<sup class="footnote-ref" id="fnref-{self.escape_url(element.label)}">'
f'<a href="#fn-{self.escape_url(element.label)}">{idx}</a></sup>'
)

@render_footnote_ref.dispatch(MarkdownRenderer)
Expand All @@ -101,9 +100,7 @@ def _render_footnote_def(self, element):
children = re.sub(r"</p>$", f"{back}</p>", children)
else:
children = f"{children}<p>{back}</p>\n"
return '<li id="fn-{}">\n{}</li>\n'.format(
self.escape_url(element.label), children
)
return f'<li id="fn-{self.escape_url(element.label)}">\n{children}</li>\n'

@helpers.render_dispatch((HTMLRenderer, MarkdownRenderer))
def render_document(self, element):
Expand Down
25 changes: 16 additions & 9 deletions marko/helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,13 @@
import re
from functools import partial
from importlib import import_module
from typing import TYPE_CHECKING, Callable, Generic, TypeVar, overload
from typing import TYPE_CHECKING, Callable, Generic, TypeVar, cast, overload

from marko.renderer import Renderer

if TYPE_CHECKING:
from typing import Any, Container, Iterable
from collections.abc import Container, Iterable
from typing import Any

from marko.element import Element

Expand All @@ -23,6 +24,7 @@
ElementT = TypeVar("ElementT", bound="Element")
RendererFunc = Callable[[T, ElementT], U]


def camel_to_snake_case(name: str) -> str:
"""Takes a camelCased string and converts to snake_case."""
pattern = r"[A-Z][a-z]+|[A-Z]+(?![a-z])"
Expand Down Expand Up @@ -110,7 +112,7 @@ def partition_by_spaces(text: str, spaces: str = " \t") -> tuple[str, str, str]:
class MarkoExtension:
parser_mixins: list[type] = dataclasses.field(default_factory=list)
renderer_mixins: list[type] = dataclasses.field(default_factory=list)
elements: list[type["Element"]] = dataclasses.field(default_factory=list)
elements: list[type[Element]] = dataclasses.field(default_factory=list)


def load_extension(name: str, **kwargs: Any) -> MarkoExtension:
Expand Down Expand Up @@ -154,7 +156,9 @@ def dispatch(
self: _RendererDispatcher[T, ElementT, U],
types: type[Renderer] | tuple[type[Renderer], ...],
) -> Callable[[RendererFunc[T, ElementT, U]], _RendererDispatcher[T, ElementT, U]]:
def decorator(func: RendererFunc[T, ElementT, U]) -> _RendererDispatcher[T, ElementT, U]:
def decorator(
func: RendererFunc[T, ElementT, U],
) -> _RendererDispatcher[T, ElementT, U]:
self._mapping[types] = func
return self

Expand All @@ -164,10 +168,10 @@ def __set_name__(self, owner: type, name: str) -> None:
self.name = name

@staticmethod
def render_ast(self, element: "Element") -> Any:
def render_ast(self, element: Element) -> Any:
return self.render_children(element)

def super_render(self, r: Any, element: "Element") -> Any:
def super_render(self, r: Any, element: Element) -> Any:
"""Call on the next class in the MRO which has the same method."""
klasses = (c for c in type(r).mro() if self.name in c.__dict__)
try:
Expand All @@ -194,20 +198,23 @@ def __get__(

def __get__(
self: _RendererDispatcher[T, ElementT, U],
obj: Renderer | None, owner: type,
obj: Renderer | None,
owner: type,
) -> RendererFunc[T, ElementT, U] | _RendererDispatcher[T, ElementT, U]:
if obj is None:
return self
for types, func in self._mapping.items():
if isinstance(obj, types):
return partial(func, obj)
return partial(func, cast(T, obj))
return partial(self.super_render, obj)


def render_dispatch(
types: type[Renderer] | tuple[type[Renderer], ...],
) -> Callable[[RendererFunc[T, ElementT, U]], _RendererDispatcher[T, ElementT, U]]:
def decorator(func: RendererFunc[T, ElementT, U]) -> _RendererDispatcher[T, ElementT, U]:
def decorator(
func: RendererFunc[T, ElementT, U],
) -> _RendererDispatcher[T, ElementT, U]:
return _RendererDispatcher(types, func)

return decorator
2 changes: 1 addition & 1 deletion marko/inline.py
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ def find(cls, text: str, *, source: Source) -> Iterator[_Match]:
# ideally the regex pattern should be rewritten, but this works for now.
# see issue #219
if "\n" not in text:
return []
return iter(())
return super().find(text, source=source)


Expand Down
2 changes: 1 addition & 1 deletion marko/parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ def _build_inline_element_list(self) -> list[InlineElementType]:
return [e for e in self.inline_elements.values() if not e.virtual]


from . import block, element, inline, inline_parser
from . import block, element, inline, inline_parser # noqa: E402

if TYPE_CHECKING:
BlockElementType = type[block.BlockElement]
Expand Down
3 changes: 1 addition & 2 deletions marko/renderer.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,8 +95,7 @@ def render_children(self, element: Any) -> Any:
return "".join(rendered)


_FT = TypeVar("_FT")
_F = TypeVar("_F", bound=Callable[..., _FT])
_F = TypeVar("_F", bound=Callable[..., Any])


def force_delegate(func: _F) -> _F:
Expand Down
31 changes: 14 additions & 17 deletions tests/test_basic.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#! -*- coding: utf-8 -*-
from __future__ import annotations

import re
import textwrap

Expand Down Expand Up @@ -62,21 +63,21 @@ def test_markdown_renderer_loose_list(self):
):
markdown = marko.Markdown(renderer=MarkdownRenderer)
rerendered = markdown(text)
assert rerendered == text, f"loose list not preserved: {text!r} -> {rerendered!r}"
assert (
rerendered == text
), f"loose list not preserved: {text!r} -> {rerendered!r}"
assert normalize_html(marko.convert(rerendered)) == normalize_html(
marko.convert(text)
)

def test_markdown_renderer_preserve_link_refs(self):
text = textwrap.dedent(
"""\
text = textwrap.dedent("""\
This is a [link ref][1], [1] and a [link](abc "title").
This is a footnote [^1].

[1]: def
[^1]: ghi
"""
)
""")
markdown = marko.Markdown(renderer=MarkdownRenderer, extensions=["footnote"])
rerendered = markdown.convert(text)
assert rerendered == text
Expand Down Expand Up @@ -157,8 +158,7 @@ def render_paragraph(self, element):
assert raw_text["children"] == "Hello world"

def test_gfm_markdown_renderer(self):
text = textwrap.dedent(
"""\
text = textwrap.dedent("""\
## TODO

- [x] Shopping[^1]
Expand All @@ -175,8 +175,7 @@ def test_gfm_markdown_renderer(self):
> Upcoming bugs

[^1]: go to https://example.com
"""
)
""")

md = marko.Markdown(extensions=["gfm", "footnote"], renderer=MarkdownRenderer)
res = md.convert(text)
Expand All @@ -185,29 +184,27 @@ def test_gfm_markdown_renderer(self):
def test_paragraph_breaking_element(self):
class CustomElement(block.BlockElement):
breaks_paragraph = True
pattern = re.compile(r" {,3}@broken (.*)", flags=re.M)
pattern = re.compile(r" {,3}@broken (.*)", flags=re.MULTILINE)

def __init__(self, match: re.Match[str]) -> None:
self.inline_body = match.group(1).strip()

@classmethod
def match(cls, source: Source) -> "re.Match[str] | None":
def match(cls, source: Source) -> re.Match[str] | None:
return source.expect_re(cls.pattern)

@classmethod
def parse(cls, source: Source) -> "re.Match[str] | None":
def parse(cls, source: Source) -> re.Match[str] | None:
m = source.match
source.consume()
return m

my_extension = marko.MarkoExtension(elements=[CustomElement])

text = textwrap.dedent(
"""
text = textwrap.dedent("""
some text that is
@broken by a custom element
"""
)
""")

md = marko.Markdown(extensions=[my_extension])
res = md.parse(text)
Expand Down
Loading