make_dashboards.py

#!/usr/bin/env python3
#
# Copyright (C) 2017 ScyllaDB
#

#
# This file is part of Scylla.
#
# Scylla is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Scylla is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Scylla.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import print_function

import argparse
import copy
import json
import re
import os
import yaml

strip_class = True


def help(args):
    print("""
The utility can be used to create dashboards from templates or templates from dashboards.

types files holds type definitions.
Type is a json object, that will be added (but not replace) to the values in the template.
Types support inheritance, when a type holds a class field, it would inherit the fields from
the base class.

Type examples:

{
    "base_row": {
        "collapse": false,
        "editable": true
    },
    "small_row": {
        "class": "base_row",
        "height": "25px"
    },
    "row": {
        "class": "base_row",
        "height": "150px"
    }
}


Template example:

{
    "dashboard": {
        "class": "dashboard",
        "rows": [
            {
                "class": "small_row",
                "panels": [
                    {
                        "class": "text_panel",
                        "content": "<img src=\"http://www.scylladb.com/wp-content/uploads/logo-scylla-white-simple.png\" height=\"70\">\n<hr style=\"border-top: 3px solid #5780c1;\">",
                        "id": "auto",
                    }
                ],
                "title": "New row"
            },
            {
                "class": "row"
            }
        ]
    }
}

When creating templates, the -kt is useful to find conflicts.
    """
    )

#TRACE=["version", "version-reject"]
TRACE=[]
MASTER_VERSION=666

def trace(part, *str):
    if part in TRACE:
        print(*str)
def is_version_bigger(version, cmp_version):
    cmp_op = 0
    m = re.match(r"([^\d]+)\s*([\d\.]+)\s*", cmp_version)
    trace("version",cmp_version)
    if m:
        cmp_version = m.group(2)
        if m.group(1) == ">":
            cmp_op = 1
        elif m.group(1) == "<":
            cmp_op = -1
    if version[0] == MASTER_VERSION:
        return cmp_op == 1
    cmp = cmp_version.split('.')
    if len(cmp) == 0 or (version[0] > 1900) != (int(cmp[0]) > 1900):
        trace("version","wrong type returning false", cmp_version, version, cmp_op, cmp[0], version[0] > 1900, int(cmp[0]) > 1900)
        return False
    ln = min(len(cmp), len(version))
    for i in range(ln):
        if (cmp_op == 0 and version[i] != int(cmp[i])) or (cmp_op > 0 and version[i] < int(cmp[i])) or (cmp_op < 0 and version[i] > int(cmp[i])):
            trace("version","not bigger/smaller, returning False", cmp_version, version, cmp[i], cmp_op, version[i])
            return False
        if (cmp_op >0 and version[i] > int(cmp[i])) or (cmp_op < 0 and version[i] < int(cmp[i])):
            return True
    # If we got here version=cmp_version
    trace("version","all is equal", cmp_version, version, cmp[i], version[i], cmp_op)
    return cmp_op >= 0

def should_version_reject(version, obj):
    if not version or "dashversion" not in obj:
        return False
    if isinstance(obj["dashversion"], list):
        for v in obj["dashversion"]:
            if is_version_bigger(version, v):
                return False
        return True
    return not is_version_bigger(version, obj["dashversion"])

def get_type(name, types):
    if name not in types:
        return {}
    if "class" not in types[name]:
        return types[name]
    result = types[name].copy()
    cls = get_type(types[name]["class"], types)
    for k in cls:
        if k not in result:
            result[k] = cls[k]
    return result

def get_json_file(name):
    try:
        return json.load(open(name))
    except Exception as inst:
        print("Failed opening file:", name, inst)
        exit(0)
def get_yaml_file(name):
    with open(name, "r") as stream:
        try:
            return yaml.safe_load(stream)
        except yaml.YAMLError as exc:
            print("Failed opening replace file", name, exc)
            exit(0)

def get_file(f):
    if f.endswith('json'):
        return get_json_file(f)
    elif f.endswith('yml') or f.endswith('yaml'):
        return get_yaml_file(f)
    else:
        print("unsupported file extension ", f)
        exit(0)

def get_exact_match(replace_file):
    if not replace_file:
        return {}
    res = {}
    for f in replace_file:
        res.update(get_file(f))
    return res

def write_json(name, obj, replace_strings=[]):
    y = json.dumps(obj, sort_keys = True, separators=(',', ': '), indent = 4)
    for r in replace_strings:
        y = y.replace(r[0], r[1] if len(r) > 1 else '')
        if r[0].endswith('_DOT__'):
            y = y.replace(r[0].replace('_DOT__','_DASHED__'), r[1].replace('.', '-'))
    with open(name, 'w') as outfile:
        outfile.write(y)

def merge_json_files(files):
    results = {}
    for name in files:
        results.update(get_file(name))
    return results

def make_replace_strings(replace):
    results = []
    if (replace):
        for v in replace:
            results.append(v.split('=', 1))
    return results

def should_product_reject(products, obj):
    return ("dashproduct" in obj) and (obj["dashproduct"] == "" and len(products)>0 or obj["dashproduct"] != "" and obj["dashproduct"] not in products) or ("dashproductreject" in obj and obj["dashproductreject"] in products)

def apply_path_overrides(obj, types=None, version=None, products=None, exact_match_replace=None):
    """Process keys containing '/' as deep-path overrides.
    e.g. "a/b/c": val sets obj["a"]["b"]["c"] = val
         "a/0/b": val sets obj["a"][0]["b"] = val (integer part = list index)
    Values that are dicts or lists are resolved via update_object before being placed.
    """
    path_keys = [k for k in list(obj.keys()) if '/' in k]
    for key in path_keys:
        val = obj.pop(key)
        # Resolve the value being placed (class expansion, etc.)
        if types is not None:
            if isinstance(val, dict):
                val = update_object(val, types, version or [], products or [], exact_match_replace or {})
            elif isinstance(val, list):
                val = [update_object(v, types, version or [], products or [], exact_match_replace or {}) if isinstance(v, dict) else v for v in val]
                val = [v for v in val if v is not None]
        parts = key.split('/')
        target = obj
        for i, part in enumerate(parts[:-1]):
            next_part = parts[i + 1]
            if isinstance(target, list):
                target = target[int(part)]
            else:
                # create the next level as list or dict based on whether next part is numeric
                try:
                    int(next_part)
                    target = target.setdefault(part, [])
                except ValueError:
                    target = target.setdefault(part, {})
        last = parts[-1]
        if isinstance(target, list):
            target[int(last)] = val
        else:
            target[last] = val

def update_object(obj, types, version, products, exact_match_replace):
    global id
    if not isinstance(obj, dict):
        return obj
    if "class" in obj:
        extra = get_type(obj["class"], types)
        for key in extra:
            if key not in obj:
                obj[key] = copy.deepcopy(extra[key])
        if strip_class:
            del obj["class"]
    # First pass: recursively resolve all non-path-override keys so the full
    # structure is materialized before path overrides try to navigate into it.
    for v in list(obj.keys()):
        if '/' in v:
            continue  # path override keys handled below
        if v == "id" and obj[v] == "auto":
            obj[v] = id
            id = id + 1
        elif isinstance(obj[v], list):
            obj[v] = [m for m in [update_object(o, types, version, products, exact_match_replace) for o in obj[v]] if m is not None]
        elif isinstance(obj[v], dict):
            obj[v] = update_object(obj[v], types, version, products, exact_match_replace)
        else:
            if obj[v] in exact_match_replace:
                obj[v] = exact_match_replace[obj[v]]
    # Apply path overrides on the fully-materialized structure; values being
    # placed are themselves resolved via update_object.
    apply_path_overrides(obj, types, version, products, exact_match_replace)
    if (version and should_version_reject(version, obj)) or should_product_reject(products, obj):
        trace("version-reject", "rejecting obj", obj)
        return None
    return obj

def compact_obj(obj, types, args):
    if not isinstance(obj, dict):
        return obj
    for v in obj:
        if isinstance(obj[v], list):
            if obj[v] and isinstance(obj[v][0], dict) and obj[v][0]:
                obj[v] = [compact_obj(o, types, args) for o in obj[v]]
        elif isinstance(obj[v], dict):
            obj[v] = compact_obj(obj[v], types, args)

    if "class" in obj:
        extra = get_type(obj["class"], types)
        for key in extra:
            if key != "class" and key in obj:
                if key != "id" and obj[key] != extra[key]:
                    if args.key_tips:
                        obj["**" + key + "**"] = extra[key]
                else:
                    obj.pop(key)
    return obj

def get_space_panel(size):
    global id
    id = id + 1
    return {
  "class": "text_panel",
  "content": "##  ",
  "editable": True,
  "error": False,
  "id": id,
  "links": [],
  "mode": "markdown",
  "span": size,
  "style": {},
  "title": "",
  "transparent": True,
  "type": "text"
}

def panel_width(gridpos, panel):
    if "w" in gridpos:
        return gridpos["w"]
    if "span" in panel:
        return panel["span"] * 2
    return 6

def get_height(value, default):
    m = re.match(r"(\d+)", value)
    if m:
        return int(int(m.group(1))/30)
    return default

def set_grid_pos(x, y, panel, h, gridpos):
    if "x" not in gridpos:
        gridpos["x"] = x
    if "y" not in gridpos:
        gridpos["y"] = y
    if "h" not in gridpos:
        if "height" in panel:
            gridpos["h"] = get_height(panel["height"], h)
        else:
            gridpos["h"] = h
    if "w" not in gridpos:
        gridpos["w"] = panel_width(gridpos, panel)
    panel["gridPos"] = gridpos
    return gridpos["h"]

def add_row(y, panels, row, args):
    # total_span = 0
    h = 6
    x = 0
    max_h = 0
    if "height" in row:
        if row["height"] != "auto":
            h = get_height(row["height"], h)
    if "gridPos" in row:
        if "h" in row["gridPos"]:
            h = row["gridPos"]["h"]
    for p in row["panels"]:
        gridpos = {}
        if "gridPos" in p:
            gridpos = dict(p["gridPos"])
        else:
            gridpos = {}
        w = panel_width(gridpos, p)
        if  w + x > 24:
            x = 0
            y = y + max_h
            max_h = 0
        height = set_grid_pos(x, y, p, h, gridpos)
        x = x + w
        if height > max_h:
            max_h = height
        panels.append(p)
    return y + max_h

def is_collapsed_row(row):
    if "panels" not in row or len(row["panels"]) == 0:
        print("row has no panels, treating as row, please update the template to add an empty panel to this row", row)
        return False
    return len(row["panels"]) == 1 and ("type" in row["panels"][0] and row["panels"][0]["type"] == "row" or "class" in row["panels"][0] and row["panels"][0]["class"] in ["row", "collapsible_row_panel"]) and "collapsed" in row["panels"][0] and row["panels"][0]["collapsed"]

def is_collapsable_row(row):
    if "panels" not in row or len(row["panels"]) == 0:
        print("row has no panels, treating as row, please update the template to add an empty panel to this row", row)
        return False
    if len(row["panels"]) == 1 and "type" not in row["panels"][0]:
        print("type is missing, treating as row, please update the template to add type:row to this panel", row)
    return len(row["panels"]) == 1 and ("type" in row["panels"][0] and row["panels"][0]["type"] == "row" or "class" in row["panels"][0] and row["panels"][0]["class"] in ["row", "collapsible_row_panel"])

def make_grafana_5(results, args):
    rows = results["dashboard"]["rows"]
    panels = []
    y = 0
    in_collapsable_panel = False
    collapsible_row = []
    collapsible_panels = []
    for row in rows:
        if is_collapsable_row(row) and in_collapsable_panel:
            collapsible_row[0]["panels"] = collapsible_panels
            panels.append(collapsible_row[0])
            collapsible_row = []
            collapsible_panels = []
            in_collapsable_panel = False
        if is_collapsed_row(row):
            in_collapsable_panel = True
            y = add_row(y, collapsible_row, row, args)
        else:
            if in_collapsable_panel:
                y = add_row(y, collapsible_panels, row, args)
            else:
                y = add_row(y, panels, row, args)
    del results["dashboard"]["rows"]
    if in_collapsable_panel:
        collapsible_row[0]["panels"] = collapsible_panels
        panels.append(collapsible_row[0])
    results["dashboard"]["panels"] = panels

def make_grafana_13(results, args):
    # Remove old Grafana 5/8 top-level keys not valid in Grafana 13 schema
    for old_key in ("templating", "time", "tags", "overwrite", "version"):
        results["dashboard"].pop(old_key, None)

    rows = results["dashboard"]["rows"]
    elements = {}
    _auto_id = [1]  # mutable so nested helpers can increment it

    def _alloc_panel_id():
        while f"panel-{_auto_id[0]}" in elements:
            _auto_id[0] += 1
        pid = _auto_id[0]
        _auto_id[0] += 1
        return pid

    def _process_panels(panels, row_layout_kind, default_height):
        """Register panels in the shared elements dict and return layout items."""
        layout_items = []
        x = 0
        y = 0
        for panel in panels:
            panel_id = panel.get("id")
            if panel_id is None:
                panel_id = _alloc_panel_id()
            element_name = f"panel-{panel_id}"
            conditional_rendering = panel.pop("conditionalRendering", None)

            # Extract layout hints before wrapping/stripping
            panel_body_pre = panel.get("spec", panel)
            panel_repeat = panel.pop("repeat", None)
            if panel_repeat is None and isinstance(panel_body_pre, dict):
                panel_repeat = panel_body_pre.pop("repeat", None)
            gp_raw = panel.get("gridPos") or panel_body_pre.get("gridPos")
            gp = gp_raw if isinstance(gp_raw, dict) else {}
            span = panel_body_pre.get("span") or panel.get("span")

            if args.panel_as_spec and not ("kind" in panel and "spec" in panel):
                panel = {"kind": "Panel", "spec": panel}
            elements[element_name] = panel

            # Strip template-engine keys that must not appear in Grafana output.
            # gridPos and span are consumed for layout above; directive keys were
            # used for filtering and must not leak into the rendered dashboard.
            _STRIP_FROM_PANEL = ("gridPos", "span", "dashproductreject", "dashproduct", "dashproduc", "dashversion")
            panel_body = panel.get("spec", panel)  # inside spec for wrapped panels
            for _k in _STRIP_FROM_PANEL:
                panel.pop(_k, None)
                panel_body.pop(_k, None)
            # For G13-format panels (kind+spec), move recognised content keys
            # from panel top-level into spec (template may set them at top level
            # when the class already provides kind/spec), then strip everything
            # else at the panel top level since only kind+spec are valid in G13.
            if "kind" in panel and "spec" in panel:
                for _move_key in ("title", "description"):
                    if _move_key in panel and not panel["spec"].get(_move_key):
                        panel["spec"][_move_key] = panel.pop(_move_key)
                for _k in list(panel.keys()):
                    if _k not in ("kind", "spec"):
                        panel.pop(_k)

            if row_layout_kind == "GridLayout":
                item = {
                    "kind": "GridLayoutItem",
                    "spec": {
                        "element": {"kind": "ElementReference", "name": element_name}
                    }
                }
                if span is not None:
                    gp.setdefault("w", int(span) * 2)
                item["spec"]["x"] = gp.get("x", x)
                item["spec"]["y"] = gp.get("y", y)
                item["spec"]["width"] = gp.get("w", 6)
                item["spec"]["height"] = gp.get("h", default_height)
                x = x + item["spec"]["width"]
                if x >= 24:
                    x = 0
                    y = y + item["spec"]["height"]
                if conditional_rendering is not None:
                    item["spec"]["conditionalRendering"] = conditional_rendering
                if panel_repeat is not None:
                    item["spec"]["repeat"] = panel_repeat if isinstance(panel_repeat, dict) else {
                        "mode": "variable",
                        "value": panel_repeat,
                        "direction": "h"
                    }
                layout_items.append(item)
            else:
                item_spec = {
                    "element": {"kind": "ElementReference", "name": element_name}
                }
                if conditional_rendering is not None:
                    item_spec["conditionalRendering"] = conditional_rendering
                if panel_repeat is not None:
                    item_spec["repeat"] = panel_repeat if isinstance(panel_repeat, dict) else {
                        "mode": "variable",
                        "value": panel_repeat,
                        "direction": "h"
                    }
                layout_items.append({
                    "kind": "AutoGridLayoutItem",
                    "spec": item_spec
                })
        return layout_items

    def _build_leaf_layout(row, row_layout_kind):
        """Build a GridLayout/AutoGridLayout for a leaf row (one that has panels)."""
        default_height = 6
        if "height" in row:
            try:
                default_height = int(row["height"].replace("px", "")) / 30
            except ValueError:
                print("Warning: row height is not a number, using default 6", row["height"])
                print(row)
        if "gridPos" in row and "h" in row.get("gridPos", {}):
            default_height = row["gridPos"]["h"]
        panels = row.get("panels", [])
        layout_items = _process_panels(panels, row_layout_kind, default_height)
        if row_layout_kind == "GridLayout":
            return layout_items, {"kind": "GridLayout", "spec": {"items": layout_items}}
        else:
            return layout_items, {
                "kind": "AutoGridLayout",
                "spec": {
                    "maxColumnCount": 3,
                    "columnWidthMode": "standard",
                    "rowHeightMode": "standard",
                    "items": layout_items
                }
            }

    def _process_row_or_tab(row):
        """Recursively convert a template row/tab to a G13 RowsLayoutRow or TabsLayoutTab.

        A row/tab in the template may contain:
          - ``panels``  — leaf panels → GridLayout or AutoGridLayout
          - ``rows``    — nested child rows/tabs; if the first child has
                          ``type: "tab"`` a TabsLayout is produced, otherwise
                          a RowsLayout is produced.

        An item with ``type: "tab"`` becomes a TabsLayoutTab; everything else
        becomes a RowsLayoutRow.
        """
        row_title = row.get("title", "")
        row_type = row.get("type", "row")
        row_collapse = row.get("collapse", False)
        row_layout_kind = row.get("layout", "AutoGridLayout")
        row_conditional_rendering = row.pop("conditionalRendering", None)
        row_repeat = row.pop("repeat", None)
        hide_header = row.get("hideHeader", False)

        sub_rows = row.get("rows", [])
        if sub_rows:
            # Detect whether children are tabs or rows based on the first child.
            if sub_rows and sub_rows[0].get("type") == "tab":
                children = [_process_row_or_tab(child) for child in sub_rows]
                inner_layout = {"kind": "TabsLayout", "spec": {"tabs": children}}
            else:
                children = [r for r in [_process_row_or_tab(child) for child in sub_rows] if r is not None]
                inner_layout = {"kind": "RowsLayout", "spec": {"rows": children}}
            items_check = children
        else:
            items_check, inner_layout = _build_leaf_layout(row, row_layout_kind)

        if row_type == "tab":
            tab_spec = {
                "title": row_title,
                "layout": inner_layout
            }
            if row_conditional_rendering is not None:
                tab_spec["conditionalRendering"] = row_conditional_rendering
            return {"kind": "TabsLayoutTab", "spec": tab_spec}
        else:
            # Skip empty rows — Grafana 13 can crash on rows with zero items
            if not items_check:
                return None
            row_spec = {
                "title": row_title,
                "collapse": row_collapse,
                "layout": inner_layout
            }
            if row_conditional_rendering is not None:
                row_spec["conditionalRendering"] = row_conditional_rendering
            if row_repeat is not None:
                row_spec["repeat"] = row_repeat if isinstance(row_repeat, dict) else {"mode": "variable", "value": row_repeat}
            if hide_header:
                row_spec["hideHeader"] = True
            return {"kind": "RowsLayoutRow", "spec": row_spec}

    layout_rows = [r for r in [_process_row_or_tab(row) for row in rows] if r is not None]

    del results["dashboard"]["rows"]
    results["dashboard"]["spec"]["elements"] = elements
    results["dashboard"]["spec"]["layout"]["spec"]["rows"] = layout_rows

    # Strip template-engine directive keys from variables; only kind+spec are
    # valid at the variable top level in Grafana 13 schema.
    for var in results["dashboard"]["spec"].get("variables", []):
        for _k in list(var.keys()):
            if _k not in ("kind", "spec"):
                var.pop(_k)

    # Grafana v2beta1 Dashboard schema requires a `status` field.
    results["dashboard"].setdefault("status", {})

def write_as_file(name_path, result, dir, replace_strings):
    name = os.path.basename(name_path)
    write_json(os.path.join(dir, name), result["dashboard"], replace_strings)

def parse_version(v):
    if v == 'master':
        return MASTER_VERSION
    return int(v)

def get_dashboard(name, types, args, replace_strings, exact_match_replace):
    global id, strip_class
    id = 1
    strip_class = args.strip_class
    version_name = ""
    version = []
    if args.dash_version != "":
        version_name =  "." + args.dash_version
        version = [parse_version(v) for v in args.dash_version.split('.')]
    new_name = name.replace("grafana/", "grafana/build/").replace(".template.json", version_name + ".json")
    result = get_json_file(name)
    for r in args.add_row:
        [row_number, row_name] = r.split(",")
        row = get_file(row_name)
        result["dashboard"]["rows"].insert(int(row_number), row)

    update_object(result, types, version, args.product, exact_match_replace)
    if not args.grafana4:
        if args.grafana13:
            make_grafana_13(result, args)
        elif args.grafana5:
            make_grafana_5(result, args)
        else:
            make_grafana_13(result, args)
    if args.as_file:
        write_as_file(new_name, result, args.as_file, replace_strings)
    else:
        write_json(new_name, result, replace_strings)

def compact_dashboard(name, type, args):
    new_name = name.replace(".json", ".template.json")
    result = get_json_file(name)
    result = compact_obj(result, types, args)
    write_json(new_name, result)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Dashboards creating tool', conflict_handler="resolve")
    parser.add_argument('-t', '--type', action='append', help='Types file')
    parser.add_argument('-R', '--replace', action='append', help='Search and replace a value, it should be in a format of old_value=new_value')
    parser.add_argument('-rf', '--replace-file', action='append', help='Search and replace a value from file')
    parser.add_argument('-d', '--dashboards', action='append', help='dashbaords file')
    parser.add_argument('-ar', '--add-row', action='append', help='merge a templated row, format number:file', default=[])
    parser.add_argument('-r', '--reverse', action='store_true', default=False, help='Reverse mode, take a dashboard and try to minimize it')
    parser.add_argument('-G', '--grafana4', action='store_true', default=False, help='Do not Migrate the dashboard to the grafa 5 format, if not set the script will remove and emulate the rows with a single panels')
    parser.add_argument('-G5', '--grafana5', action='store_true', default=False, help='Generate dashboard in Grafana 5 format (panels with gridPos)')
    parser.add_argument('-G13', '--grafana13', action='store_true', default=False, help='Generate dashboard in Grafana 13 format (RowsLayout with elements)')
    parser.add_argument('--panel-as-spec', action='store_true', default=True, help='Wrap panels in {kind: Panel, spec: {...}} when generating Grafana 13 format (default: true)')
    parser.add_argument('--no-panel-as-spec', dest='panel_as_spec', action='store_false', help='Disable panel-as-spec wrapping')
    parser.add_argument('--strip-class', dest='strip_class', action='store_true', default=True, help='Remove class keys from generated output (default: true)')
    parser.add_argument('--no-strip-class', dest='strip_class', action='store_false', help='Keep class keys in generated output')
    parser.add_argument('-h', '--help', action='store_true', default=False, help='Print help information')
    parser.add_argument('-kt', '--key-tips', action='store_true', default=False, help='Add key tips when there are conflict values between the template and the value')
    parser.add_argument('-af', '--as-file', type=str, default="", help='Make the dashboard ready to be loaded as files and not with http, when not empty, state the directory the file will be written to')
    parser.add_argument('-V', '--dash-version', type=str, default="", help='When set, create a dashboard for a specific version, looking at the dashversion tags')
    parser.add_argument('-P', '--product', action='append', default=[], help='when added will look at the dashproduct tag')

    args = parser.parse_args()
    if args.help:
        parser.print_help()
        help(args)
        exit(0)

    exact_match_replace = get_exact_match(args.replace_file)

    types = merge_json_files(args.type)
    replace_strings = make_replace_strings(args.replace)
    for d in args.dashboards:
        if args.reverse:
            compact_dashboard(d, types, args)
        else:
            get_dashboard(d, types, args, replace_strings, exact_match_replace)