scripts/build.py

# A script to add footers and check for dead links
import yaml
import os
import re
import sys
import pathlib
import difflib
import os.path


class LOG:
    FATAL = 0
    ERROR = 1
    WARN = 2
    INFO = 3
    DEBUG = 4


COLOR = True
LOGLEVEL = LOG.INFO


def log(severity, context, message):
    if severity > LOGLEVEL:
        return
    color = {
        LOG.FATAL: '\u001b[31m',
        LOG.ERROR: '\u001b[35m',
        LOG.WARN: '\u001b[33m',
        LOG.INFO: '\u001b[34m',
        LOG.DEBUG: '\u001b[37m'
    }[severity]
    seve = {
        LOG.FATAL: 'FATL',
        LOG.ERROR: 'EROR',
        LOG.WARN: 'WARN',
        LOG.INFO: 'INFO',
        LOG.DEBUG: 'DBUG'
    }[severity]

    if COLOR == False:
        print(f"{seve} {context}: {message}")
    else:
        print(f"\u001b[1m{color}{seve}\u001b[0m {context}: {message}")


class ContextFilters:

    def warn_unlinked(self, context):
        """Warns about any pages which have been created, but not put into the navigation"""
        for file in context['all_files']:
            if file in context['nav_files']:
                continue
            log(LOG.WARN, "warn_unlinked",
                f"{file} has been created, but not linked in nav")


class Filters:

    def warn_deadlink(self, content, context):
        """Checks for dead links between pages"""
        links = [link.replace(context['mkdocs']['site_url'], "") for _, link in re.findall(
            r"\[([\w\W]+?)\]\(([\w\W]+?)\)", content, re.MULTILINE) if ((link.startswith(
                "https://") or link.startswith("http://")) and link.startswith(context['mkdocs']['site_url']) or link.startswith("/") or "://" not in link) and "images" not in link]
        for l in links:
            link = l
            if l == '' or l == '/':
                link = "index.md"
            link = link.split("#")[0]

            path = context['docs_dir'] + link
            if not link.startswith("/"):
                path = os.path.join(
                    context['docs_dir'], os.path.dirname(context['file_name']), link)

            if not path.endswith(".md"):
                if path.endswith("/"):
                    path = path[:-1] + ".md"
                else:
                    path = path + ".md"

            abs_path = pathlib.Path(path).resolve()

            if not abs_path.exists():
                log(LOG.WARN, "warn_deadlink",
                    f"{context['file_name']} contains a dead link to {link} ({path})")

        return content

    # disabled for now because we're using snippets
    # def footer(self, content, context):
    #     """Appends a footer to the end of every manuscript file"""
    #     if not hasattr(self, "footer_content"):
    #         self.footer_path = os.path.join(
    #             os.path.dirname(__file__), "recipe-footer.md")
    #         footer_file = open(self.footer_path)
    #         self.footer_content = footer_file.read()
    #         self.footer_search = self.footer_content.split("\n")[
    #             0].replace("#", "")
    #         footer_file.close()
    #     if "index.md" in context['file_name']:
    #         log(LOG.DEBUG, "footer",
    #             f"ignoring {context['file_name']} as it contains index.md")
    #         return content

    #     if self.footer_search in content:
    #         log(LOG.WARN, "footer",
    #             f"Footer is hard-coded in {context['file_name']}")
    #         return content
    #     return content + "\n" + self.footer_content


def flattern(obj):
    resp = []

    if type(obj) == dict:
        for key, value in obj.items():
            if type(value) == dict or type(value) == list:
                resp.extend(flattern(value))
            else:
                resp.append(value)
    elif type(obj) == list:
        for value in obj:
            if type(value) == dict or type(value) == list:
                resp.extend(flattern(value))
            else:
                resp.append(value)
    return resp


def execute(dry_run, show_diffs, mkdocs_yaml):
    log(LOG.DEBUG, "", "Loading mkdocs.yml")
    mkdocs = yaml.load(open(mkdocs_yaml).read(), Loader=yaml.Loader)
    log(LOG.DEBUG, "mkdocs", mkdocs)
    log(LOG.DEBUG, "", "Loaded mkdocs.yml")

    log(LOG.DEBUG, "", "Registering filters")
    filters = [getattr(Filters(), func) for func in dir(
        Filters) if callable(getattr(Filters, func)) and not func.endswith("__")]
    for filter in filters:
        log(LOG.INFO, "Filter Info", f"{filter.__name__}: {filter.__doc__}")
    log(LOG.DEBUG, "", "Registered filters")

    log(LOG.DEBUG, "", "Registering context filters")
    context_filters = [getattr(ContextFilters(), func) for func in dir(
        ContextFilters) if callable(getattr(ContextFilters, func)) and not func.endswith("__")]

    for filter in context_filters:
        log(LOG.INFO, "Filter Info", f"{filter.__name__}: {filter.__doc__}")
    log(LOG.DEBUG, "", "Registered context filters")

    log(LOG.DEBUG, "", "Walking docs_dir for manuscript files")
    docs_dir = os.path.join(os.path.dirname(mkdocs_yaml), mkdocs['docs_dir'])
    all_files = flattern([[os.path.join(root, file).replace(docs_dir + "/", "") for file in files if file.endswith(".md")]
                          for root, dirs, files in os.walk(docs_dir)])
    log(LOG.DEBUG, "all_files", all_files)
    log(LOG.DEBUG, "", "Walked docs_dir")

    log(LOG.DEBUG, "", "Loading nav entries")
    nav_files = flattern(mkdocs['nav'])
    log(LOG.DEBUG, "nav_files", nav_files)
    log(LOG.DEBUG, "", "Loaded nav entries")

    log(LOG.DEBUG, "", "Constructing context")
    context = {
        'docs_dir': docs_dir,
        'all_files': all_files,
        'nav_files': nav_files,
        'mkdocs': mkdocs
    }
    log(LOG.DEBUG, "", "Constructed context")

    log(LOG.DEBUG, "", "Running context filters")
    for filter in context_filters:
        log(LOG.INFO, "Global", f"Running {filter.__name__}")
        filter(context)

    log(LOG.DEBUG, "", "Running filters per file")

    for file in all_files:
        log(LOG.INFO, "", f"Processing {file}")
        path = os.path.join(docs_dir, file)
        context['file_name'] = file
        file_open = open(path, 'r')
        original = file_open.read()
        content = original
        file_open.close()
        for filter in filters:
            log(LOG.DEBUG, file, f"Running \"{filter.__name__}\" filter")
            content = filter(content, context)
        if not dry_run:
            fo = open(path, 'w')
            fo.write(content)
            fo.close()

        if show_diffs:
            a = original
            b = content

            diffs = []
            diff_count = 0
            matcher = difflib.SequenceMatcher(None, a, b)
            for opcode, a0, a1, b0, b1 in matcher.get_opcodes():
                diff_count = diff_count + 1
                if opcode == "equal":
                    diff_count = diff_count - 1
                    diffs.append(a[a0:a1])
                elif opcode == "insert":
                    diffs.append("\u001b[42m" + b[b0:b1])
                elif opcode == "delete":
                    diffs.append("\u001b[41m" + a[a0:a1])
                elif opcode == "replace":
                    diffs.append("\u001b[42m" + b[b0:b1])
                    diffs.append("\u001b[41m" + a[a0:a1])
            if diff_count != 0:
                log(LOG.INFO, file, "\u001b[0m".join(diffs) + "\u001b[0m")


def main():
    import argparse
    parser = argparse.ArgumentParser(description="Stir up a cookbook")
    parser.add_argument(
        "mkdocs", help="Path to the mkdocs.yml file", type=str)
    parser.add_argument(
        "-d", "--dry", help="Run the build in dry mode (does not write changes)", action="store_true")
    parser.add_argument(
        "-i", "--diffs", help="Show diffs for changed files", action="store_true")

    args = parser.parse_args()

    execute(args.dry, args.diffs, args.mkdocs)


if __name__ == "__main__":
    main()