Merge pull request #3224 from mkdocs/ignore

New `exclude_docs` config: gitignore-like patterns of files to exclude
mkdocs · Jun 11, 2023 · 82aa863 · 82aa863
2 parents f2d14c5 + 9d56fa2
commit 82aa863
Show file tree

Hide file tree

Showing 23 changed files with 554 additions and 197 deletions.
diff --git a/.flake8 b/.flake8
@@ -1,3 +1,2 @@
 [flake8]
-max-line-length = 119
-extend-ignore = E203, TC004
+extend-ignore = E501, E203, TC004
diff --git a/docs/user-guide/configuration.md b/docs/user-guide/configuration.md
@@ -291,6 +291,69 @@ server root and effectively points to `https://example.com/bugs/`. Of course, th
 list of all the Markdown files found within the `docs_dir` and its
 sub-directories. Index files will always be listed first within a sub-section.
 
+### exclude_docs
+
+NEW: **New in version 1.5.**
+
+This config defines patterns of files (under [`docs_dir`](#docs_dir)) to not be picked up into the built site.
+
+Example:
+
+```yaml
+exclude_docs: |
+    api-config.json    # A file with this name anywhere.
+    drafts/            # A "drafts" directory anywhere.
+    /requirements.txt  # Top-level "docs/requirements.txt".
+    *.py               # Any file with this extension anywhere.
+    !/foo/example.py   # But keep this particular file.
+```
+
+This follows the [.gitignore pattern format](https://git-scm.com/docs/gitignore#_pattern_format).
+
+Note that `mkdocs serve` does *not* follow this setting and instead displays excluded documents but with a "DRAFT" mark. To prevent this effect, you can run `mkdocs serve --clean`.
+
+The following defaults are always implicitly prepended - to exclude dot-files (and directories) as well as the top-level `templates` directory:
+
+```yaml
+exclude_docs: |
+    .*
+    /templates/
+```
+
+So, in order to really start this config fresh, you'd need to specify a negated version of these entries first.
+
+Otherwise you could for example opt only certain dot-files back into the site:
+
+```yaml
+exclude_docs: |
+    !.assets  # Don't exclude '.assets' although all other '.*' are excluded
+```
+
+### not_in_nav
+
+NEW: **New in version 1.5.**
+
+NOTE: This option does *not* actually exclude anything from the nav.
+
+If you want to include some docs into the site but intentionally exclude them from the nav, normally MkDocs warns about this.
+
+Adding such patterns of files (relative to [`docs_dir`](#docs_dir)) into the `not_in_nav` config will prevent such warnings.
+
+Example:
+
+```yaml
+nav:
+    - Foo: foo.md
+    - Bar: bar.md
+
+not_in_nav: |
+    /private.md
+```
+
+As the previous option, this follows the .gitignore pattern format.
+
+NOTE: Adding a given file to [`exclude_docs`](#exclude_docs) takes precedence over and implies `not_in_nav`.
+
 ## Build directories
 
 ### theme

diff --git a/docs/user-guide/writing-your-docs.md b/docs/user-guide/writing-your-docs.md
@@ -27,10 +27,7 @@ extensions may be used for your Markdown source files: `markdown`, `mdown`,
 directory will be rendered in the built site regardless of any settings.
 
 NOTE:
-Files and directories with names which begin with a dot (for example:
-`.foo.md` or `.bar/baz.md`) are ignored by MkDocs, which matches the
-behavior of most web servers. There is no option to override this
-behavior.
+Files and directories with names which begin with a dot (for example: `.foo.md` or `.bar/baz.md`) are ignored by MkDocs. This can be overridden with the [`exclude_docs` config](configuration.md#exclude_docs).
 
 You can also create multi-page documentation, by creating several Markdown
 files:

diff --git a/mkdocs/__main__.py b/mkdocs/__main__.py
@@ -46,6 +46,10 @@ def _showwarning(message, category, filename, lineno, file=None, line=None):
 
 
 def _enable_warnings():
+    from mkdocs.commands import build
+
+    build.log.addFilter(utils.DuplicateFilter())
+
     warnings.simplefilter('module', DeprecationWarning)
     warnings.showwarning = _showwarning
 
@@ -111,8 +115,9 @@ def __del__(self):
 use_directory_urls_help = "Use directory URLs when building pages (the default)."
 reload_help = "Enable the live reloading in the development server (this is the default)"
 no_reload_help = "Disable the live reloading in the development server."
-dirty_reload_help = (
-    "Enable the live reloading in the development server, but only re-build files that have changed"
+serve_dirty_help = "Only re-build files that have changed."
+serve_clean_help = (
+    "Build the site without any effects of `mkdocs serve` - pure `mkdocs build`, then serve."
 )
 commit_message_help = (
     "A commit message to use when committing to the "
@@ -220,21 +225,23 @@ def cli():
 
 @cli.command(name="serve")
 @click.option('-a', '--dev-addr', help=dev_addr_help, metavar='<IP:PORT>')
-@click.option('--livereload', 'livereload', flag_value='livereload', help=reload_help, default=True)
+@click.option('--livereload', 'livereload', flag_value='livereload', default=True, hidden=True)
 @click.option('--no-livereload', 'livereload', flag_value='no-livereload', help=no_reload_help)
-@click.option('--dirtyreload', 'livereload', flag_value='dirty', help=dirty_reload_help)
+@click.option('--dirtyreload', 'build_type', flag_value='dirty', hidden=True)
+@click.option('--dirty', 'build_type', flag_value='dirty', help=serve_dirty_help)
+@click.option('-c', '--clean', 'build_type', flag_value='clean', help=serve_clean_help)
 @click.option('--watch-theme', help=watch_theme_help, is_flag=True)
 @click.option(
     '-w', '--watch', help=watch_help, type=click.Path(exists=True), multiple=True, default=[]
 )
 @common_config_options
 @common_options
-def serve_command(dev_addr, livereload, watch, **kwargs):
+def serve_command(**kwargs):
     """Run the builtin development server"""
     from mkdocs.commands import serve
 
     _enable_warnings()
-    serve.serve(dev_addr=dev_addr, livereload=livereload, watch=watch, **kwargs)
+    serve.serve(**kwargs)
 
 
 @cli.command(name="build")

diff --git a/mkdocs/commands/build.py b/mkdocs/commands/build.py
@@ -5,36 +5,26 @@
 import os
 import time
 from typing import TYPE_CHECKING, Any, Sequence
-from urllib.parse import urlsplit
+from urllib.parse import urljoin, urlsplit
 
 import jinja2
 from jinja2.exceptions import TemplateNotFound
 
 import mkdocs
 from mkdocs import utils
 from mkdocs.exceptions import Abort, BuildError
-from mkdocs.structure.files import File, Files, get_files
+from mkdocs.structure.files import File, Files, InclusionLevel, _set_exclusions, get_files
 from mkdocs.structure.nav import Navigation, get_navigation
+from mkdocs.structure.pages import Page
+from mkdocs.utils import DuplicateFilter  # noqa - legacy re-export
 
 if TYPE_CHECKING:
     from mkdocs.config.defaults import MkDocsConfig
-    from mkdocs.structure.pages import Page
-
-
-class DuplicateFilter:
-    """Avoid logging duplicate messages."""
-
-    def __init__(self) -> None:
-        self.msgs: set[str] = set()
-
-    def __call__(self, record: logging.LogRecord) -> bool:
-        rv = record.msg not in self.msgs
-        self.msgs.add(record.msg)
-        return rv
 
+if TYPE_CHECKING:
+    from mkdocs.livereload import LiveReloadServer
 
 log = logging.getLogger(__name__)
-log.addFilter(DuplicateFilter())
 
 
 def get_context(
@@ -202,6 +192,7 @@ def _build_page(
     nav: Navigation,
     env: jinja2.Environment,
     dirty: bool = False,
+    excluded: bool = False,
 ) -> None:
     """Pass a Page to theme template and write output to site_dir."""
 
@@ -229,6 +220,13 @@ def _build_page(
             'page_context', context, page=page, config=config, nav=nav
         )
 
+        if excluded:
+            page.content = (
+                '<div class="mkdocs-draft-marker" title="This page will not be included into the built site.">'
+                'DRAFT'
+                '</div>' + (page.content or '')
+            )
+
         # Render the template.
         output = template.render(context)
 
@@ -254,7 +252,9 @@ def _build_page(
         raise
 
 
-def build(config: MkDocsConfig, live_server: bool = False, dirty: bool = False) -> None:
+def build(
+    config: MkDocsConfig, live_server: LiveReloadServer | None = None, dirty: bool = False
+) -> None:
     """Perform a full site build."""
 
     logger = logging.getLogger('mkdocs')
@@ -265,6 +265,8 @@ def build(config: MkDocsConfig, live_server: bool = False, dirty: bool = False)
     if config.strict:
         logging.getLogger('mkdocs').addHandler(warning_counter)
 
+    inclusion = InclusionLevel.all if live_server else InclusionLevel.is_included
+
     try:
         start = time.monotonic()
 
@@ -297,17 +299,30 @@ def build(config: MkDocsConfig, live_server: bool = False, dirty: bool = False)
 
         # Run `files` plugin events.
         files = config.plugins.run_event('files', files, config=config)
+        # If plugins have added files but haven't set their inclusion level, calculate it again.
+        _set_exclusions(files._files, config)
 
         nav = get_navigation(files, config)
 
         # Run `nav` plugin events.
         nav = config.plugins.run_event('nav', nav, config=config, files=files)
 
         log.debug("Reading markdown pages.")
-        for file in files.documentation_pages():
+        excluded = []
+        for file in files.documentation_pages(inclusion=inclusion):
             log.debug(f"Reading: {file.src_uri}")
+            if file.page is None and file.inclusion.is_excluded():
+                if live_server:
+                    excluded.append(urljoin(live_server.url, file.url))
+                Page(None, file, config)
             assert file.page is not None
             _populate_page(file.page, config, files, dirty)
+        if excluded:
+            log.info(
+                "The following pages are being built only for the preview "
+                "but will be excluded from `mkdocs build` per `exclude_docs`:\n  - "
+                + "\n  - ".join(excluded)
+            )
 
         # Run `env` plugin events.
         env = config.plugins.run_event('env', env, config=config, files=files)
@@ -316,7 +331,7 @@ def build(config: MkDocsConfig, live_server: bool = False, dirty: bool = False)
         # with lower precedence get written first so that files with higher precedence can overwrite them.
 
         log.debug("Copying static assets.")
-        files.copy_static_files(dirty=dirty)
+        files.copy_static_files(dirty=dirty, inclusion=inclusion)
 
         for template in config.theme.static_templates:
             _build_theme_template(template, env, files, config, nav)
@@ -325,10 +340,12 @@ def build(config: MkDocsConfig, live_server: bool = False, dirty: bool = False)
             _build_extra_template(template, files, config, nav)
 
         log.debug("Building markdown pages.")
-        doc_files = files.documentation_pages()
+        doc_files = files.documentation_pages(inclusion=inclusion)
         for file in doc_files:
             assert file.page is not None
-            _build_page(file.page, config, doc_files, nav, env, dirty)
+            _build_page(
+                file.page, config, doc_files, nav, env, dirty, excluded=file.inclusion.is_excluded()
+            )
 
         # Run `post_build` plugin events.
         config.plugins.run_event('post_build', config=config)

diff --git a/mkdocs/commands/serve.py b/mkdocs/commands/serve.py
@@ -28,6 +28,7 @@ def serve(
     theme=None,
     theme_dir=None,
     livereload='livereload',
+    build_type=None,
     watch_theme=False,
     watch=[],
     **kwargs,
@@ -58,8 +59,13 @@ def mount_path(config: MkDocsConfig):
         **kwargs,
     )
 
-    live_server = livereload in ('dirty', 'livereload')
-    dirty = livereload == 'dirty'
+    is_clean = build_type == 'clean'
+    is_dirty = build_type == 'dirty'
+
+    config = get_config()
+    config['plugins'].run_event(
+        'startup', command=('build' if is_clean else 'serve'), dirty=is_dirty
+    )
 
     def builder(config: MkDocsConfig | None = None):
         log.info("Building documentation...")
@@ -75,31 +81,28 @@ def builder(config: MkDocsConfig | None = None):
         # Override a few config settings after validation
         config.site_url = f'http://{config.dev_addr}{mount_path(config)}'
 
-        build(config, live_server=live_server, dirty=dirty)
+        build(config, live_server=None if is_clean else server, dirty=is_dirty)
 
-    config = get_config()
-    config['plugins'].run_event('startup', command='serve', dirty=dirty)
+    host, port = config.dev_addr
+    server = LiveReloadServer(
+        builder=builder, host=host, port=port, root=site_dir, mount_path=mount_path(config)
+    )
+
+    def error_handler(code) -> bytes | None:
+        if code in (404, 500):
+            error_page = join(site_dir, f'{code}.html')
+            if isfile(error_page):
+                with open(error_page, 'rb') as f:
+                    return f.read()
+        return None
+
+    server.error_handler = error_handler
 
     try:
         # Perform the initial build
         builder(config)
 
-        host, port = config.dev_addr
-        server = LiveReloadServer(
-            builder=builder, host=host, port=port, root=site_dir, mount_path=mount_path(config)
-        )
-
-        def error_handler(code) -> bytes | None:
-            if code in (404, 500):
-                error_page = join(site_dir, f'{code}.html')
-                if isfile(error_page):
-                    with open(error_page, 'rb') as f:
-                        return f.read()
-            return None
-
-        server.error_handler = error_handler
-
-        if live_server:
+        if livereload:
             # Watch the documentation files, the config file and the theme files.
             server.watch(config.docs_dir)
             server.watch(config.config_file_path)

diff --git a/mkdocs/config/config_options.py b/mkdocs/config/config_options.py
@@ -28,6 +28,8 @@
 from urllib.parse import urlsplit, urlunsplit
 
 import markdown
+import pathspec
+import pathspec.gitignore
 
 from mkdocs import plugins, theme, utils
 from mkdocs.config.base import (
@@ -1118,3 +1120,15 @@ def post_validation(self, config: Config, key_name: str):
         plugins = config[self.plugins_key]
         for name, hook in config[key_name].items():
             plugins[name] = hook
+
+
+class PathSpec(BaseConfigOption[pathspec.gitignore.GitIgnoreSpec]):
+    """A path pattern based on gitignore-like syntax."""
+
+    def run_validation(self, value: object) -> pathspec.gitignore.GitIgnoreSpec:
+        if not isinstance(value, str):
+            raise ValidationError(f'Expected a multiline string, but a {type(value)} was given.')
+        try:
+            return pathspec.gitignore.GitIgnoreSpec.from_lines(lines=value.splitlines())
+        except ValueError as e:
+            raise ValidationError(str(e))
diff --git a/mkdocs/config/defaults.py b/mkdocs/config/defaults.py
@@ -26,6 +26,12 @@ class MkDocsConfig(base.Config):
     """Defines the structure of the navigation."""
     pages = c.Deprecated(removed=True, moved_to='nav')
 
+    exclude_docs = c.Optional(c.PathSpec())
+    """Gitignore-like patterns of files (relative to docs dir) to exclude from the site."""
+
+    not_in_nav = c.Optional(c.PathSpec())
+    """Gitignore-like patterns of files (relative to docs dir) that are not intended to be in the nav."""
+
     site_url = c.Optional(c.URL(is_dir=True))
     """The full URL to where the documentation will be hosted."""