From ef4e6114714f4fdcc2181bc46f1fa67ed0b31c06 Mon Sep 17 00:00:00 2001 From: Tobias Schmidt Date: Wed, 14 Oct 2020 20:04:54 +0200 Subject: [PATCH] Automate repository docs inclusion (#1762) So far every prometheus/alertmanater/... release branch had to be manually configured in the nanoc.yaml config file. With this change the most recent release branches will be checked out automatically if a corresponding semver tag exists. As the Prometheus git repository includes several hundreds of megabytes of vendored assets, the repository is cloned bare and all blobs are filtered by default. Each version is then checked out in an individual working tree and git's spare-chekcout feature is used to reduce the checkout to the `docs/` folder. The git data is cached in `tmp/repo_docs/` and will be recreated automatically if removed. Signed-off-by: Tobias Schmidt --- .gitignore | 3 +- Makefile | 2 +- Rules | 20 +-- layouts/header.html | 2 +- lib/data_sources/repo_docs.rb | 184 ++++++++++++++++++++------- lib/filters/normalize_links.rb | 5 +- lib/filters/outdated_content.rb | 23 ---- lib/filters/prerelease_content.rb | 23 ---- lib/filters/version_warning.rb | 44 +++++++ lib/helpers/nav.rb | 13 +- nanoc.yaml | 201 +----------------------------- scripts/checkout.sh | 62 --------- 12 files changed, 202 insertions(+), 380 deletions(-) delete mode 100644 lib/filters/outdated_content.rb delete mode 100644 lib/filters/prerelease_content.rb create mode 100644 lib/filters/version_warning.rb delete mode 100755 scripts/checkout.sh diff --git a/.gitignore b/.gitignore index 0a8ba4ea..1fef6b37 100644 --- a/.gitignore +++ b/.gitignore @@ -5,8 +5,7 @@ output/ # Temporary file directory tmp/ -/downloads/ -/repositories/ +downloads/ # Crash Log crash.log diff --git a/Makefile b/Makefile index 3addfd8a..0eccfea1 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ bundle: bundle install --path vendor clean: - rm -rf output downloads repositories + rm -rf output downloads compile: $(NANOC) diff --git a/Rules b/Rules index b2a381b8..c67cff2e 100644 --- a/Rules +++ b/Rules @@ -14,21 +14,8 @@ # because “*” matches zero or more characters. passthrough '/assets/**' - -compile '/_redirects/' do -end - -route '/_redirects/' do - '/_redirects' -end - -# TODO(ts): Remove these hacks once the nanoc4 upgrade is done. -compile '*/images/*' do -end - -route '*/images/*' do - item.identifier.chop + '.' + item[:extension] -end +passthrough '/_redirects' +passthrough '*/images/*' # RSS Feed compile '/blog/feed/' do @@ -45,8 +32,7 @@ compile '*' do if item[:extension] == 'md' filter :redcarpet, options: {filter_html: true, autolink: true, no_intraemphasis: true, fenced_code_blocks: true, gh_blockcode: true, tables: true}, renderer_options: {with_toc_data: true} filter :normalize_links, item[:repo_docs] if item[:repo_docs] - filter :outdated_content, item[:repo_docs] if item[:repo_docs] && item[:repo_docs][:outdated] - filter :prerelease_content, item[:repo_docs] if item[:repo_docs] && item[:repo_docs][:prerelease] + filter :version_warning, item[:repo_docs] if item[:repo_docs] filter :add_anchors filter :bootstrappify filter :admonition diff --git a/layouts/header.html b/layouts/header.html index 5dda1e4e..73895b29 100644 --- a/layouts/header.html +++ b/layouts/header.html @@ -25,7 +25,7 @@ - <% if (c = @item[:repo_docs]) && c[:canonical] %><% end %> + <% if (c = @item[:repo_docs]) && c[:canonical_root] %><% end %> diff --git a/lib/data_sources/repo_docs.rb b/lib/data_sources/repo_docs.rb index ab3b161b..0daf4094 100644 --- a/lib/data_sources/repo_docs.rb +++ b/lib/data_sources/repo_docs.rb @@ -1,60 +1,158 @@ -# TODO(ts): Rewrite data source and use one single instance to combine all -# different versions for a given path. -class RepoDocsDataSource < ::Nanoc::DataSources::Filesystem +require 'uri' + +# The RepoDocs data source provides items sourced from other Git repositories. +# For a given repository_url, all git version tags are fetched and for the most +# recent (in order to save compilation time) tags the `docs/` folder in the +# respective `release-` is checked out and its content mounted under +# the given `items_root`. +# +# As the Prometheus git repository includes several hundreds of megabytes of +# vendored assets, the repository is cloned bare and all blobs are filtered by +# default. Each version is then checked out in an individual working tree and +# git's spare-checkout feature is used to reduce the checkout to the `docs/` +# folder. The git data is cached in `tmp/repo_docs/`. +class RepoDocsDataSource < ::Nanoc::DataSource identifier :repo_docs - PATH = "repositories" + DOCS_DIRECTORY = 'docs'.freeze + BRANCH_PATTERN = 'release-*'.freeze + VERSION_REGEXP = /\Av\d+\.\d+\.\d+(?:-[a-z0-9.]+)?\z/.freeze + TMPDIR = 'tmp/repo_docs/'.freeze def up - c = config[:config] - - %x( - scripts/checkout.sh \ - -d "#{docs_root}" \ - -t "#{repo_path}" \ - "#{c[:repository]}" "#{c[:refspec]}" - ) - if $?.exitstatus != 0 - raise "Couldn't checkout repository #{c.inspect}" - end - - super + validate + sync_repository end def items - c = config.fetch(:config) - super.map do |item| - attrs = item.attributes.dup - attrs[:repo_docs] = c - attrs[:repo_docs][:items_root] = config.fetch(:items_root) - # TODO(ts): Remove assumptions about the path layout, rewrite datasource. - attrs[:repo_docs][:version_root] = config.fetch(:items_root).sub(%r{(.+/)[^/]+/\Z}, '\\1') - # TODO(ts): Document that repo doc index.md will be ignored. - if item.identifier == '/' - attrs[:nav] = { strip: true } + items_root = config.fetch(:items_root, '/') + latest = latest_version + + versions.inject([]) do |list, version| + branch = "release-#{version}" + dir = git_checkout(branch, DOCS_DIRECTORY) + fs_config = { content_dir: dir, encoding: 'utf-8', identifier_type: 'legacy' } + fs = ::Nanoc::DataSources::Filesystem.new(@site_config, '/', '/', fs_config) + + fs.items.each do |item| + attrs = item.attributes.dup + attrs[:nav] = { strip: true } if item.identifier == '/' + attrs[:repo_docs] = { + name: version, + refspec: branch, + version: version, + latest: latest, + items_root: items_root, + version_root: File.join(items_root, version, '/'), + canonical_root: File.join(items_root, 'latest', '/'), + repository_url: git_remote, + entrypoint: config[:config][:entrypoint], + } + + if version == latest + lattrs = attrs.dup + lattrs[:repo_docs] = attrs[:repo_docs].dup + lattrs[:repo_docs][:name] = "latest (#{version})" + lattrs[:repo_docs][:version_root] = lattrs[:repo_docs][:canonical_root] + list << new_item(item.content, lattrs, item.identifier.prefix('/latest')) + end + + list << new_item(item.content, attrs, item.identifier.prefix('/' + version)) end - new_item(item.content, attrs, item.identifier) + + list end end - def content_dir_name - File.join(repo_path, docs_root) - end - - def layouts_dir_name - 'unsupported' - end - private - def docs_root - c = config.fetch(:config) - c.fetch(:root, 'docs/') + def validate + if !config[:config].has_key?(:entrypoint) + fail ArgumentError, 'entrypoint config option must be set' + end + if !config[:config].has_key?(:repository_url) + fail ArgumentError, 'repository config option must be set' + end + URI(config[:config][:repository_url]) # raises an exception if invalid end - def repo_path - c = config.fetch(:config) - base = c.fetch(:repo_base, 'repositories') - File.join(base, File.basename(c[:repository]), c[:name]) + def git_remote + config[:config][:repository_url] + end + + def git_dir + basename = File.basename(git_remote) + basename += '.git' unless basename.end_with?('.git') + File.join(TMPDIR, basename) + end + + def git_branches + output = `cd #{git_dir} && git branch --format='%(refname:short)' --list '#{BRANCH_PATTERN}'` + fail "Could not list git branches" if $?.exitstatus != 0 + output.split("\n") + end + + def git_tags + output = `cd #{git_dir} && git tag` + fail "Could not list git tags" if $?.exitstatus != 0 + output.split("\n") + end + + # git_checkout checks out the directory in the specified branch using git's + # sparse checkout and returns the path to the location in the workingtree. + def git_checkout(branch, directory) + worktree = File.absolute_path(File.join(git_dir.delete_suffix('.git'), branch)) + if !File.exist?(File.join(worktree, '.git')) + run_command("cd #{git_dir} && git worktree add --no-checkout #{worktree} #{branch}") + end + + worktree_info = File.join(git_dir, 'worktrees', branch, 'info') + Dir.mkdir(worktree_info) if !Dir.exist?(worktree_info) + File.write(File.join(worktree_info, 'sparse-checkout'), "/#{directory}\n") + + run_command("cd #{worktree} && git reset --hard --quiet && git clean --force") + File.join(worktree, directory) + end + + # sync_repository clones or updates a bare git repository and enables the + # sparse checkout feature. + def sync_repository + if !Dir.exist?(git_dir) + run_command("git clone --bare --filter=blob:none #{git_remote} #{git_dir}") + run_command("cd #{git_dir} && git config core.sparseCheckout true") + else + run_command("cd #{git_dir} && git fetch --quiet") + end + end + + # versions returns an ordered list of major.minor version names for which + # documentation should be published. Only the most recent versions for which a + # corresponding release-* branch exists are returned. + def versions + branches = git_branches + all = git_tags + .select { |v| v.match(VERSION_REGEXP) } + .map { |v| v.delete_prefix('v').split('.')[0, 2].join('.') } + .uniq + .select { |v| branches.include?('release-' + v) } + .sort_by { |v| v.split('.').map(&:to_i) } + .reverse + + # Number of versions is reduced to speed up site compilation time. + grouped = all.group_by { |v| v.split('.').first } + grouped.inject([]) do |list, (major, versions)| + size = major == grouped.keys.first ? 10 : 1 + list += versions[0, size] + end + end + + # latest_version returns the latest released version. + def latest_version + tags = git_tags + versions.find { |v| tags.any? { |t| t.start_with?('v' + v) && !t.include?('-') } } + end + + def run_command(cmd) + fail "Running command '#{cmd}' failed" if !system(cmd) end end diff --git a/lib/filters/normalize_links.rb b/lib/filters/normalize_links.rb index 4e9ea528..631290d0 100644 --- a/lib/filters/normalize_links.rb +++ b/lib/filters/normalize_links.rb @@ -35,10 +35,7 @@ class NormalizeLinks < ::Nanoc::Filter end def github_link_to(file, config) - base = config[:repository] - if base.end_with?('.git') - base = base[0..-5] - end + base = config[:repository_url].delete_suffix('.git') File.join(base, 'blob', config[:refspec], file) end diff --git a/lib/filters/outdated_content.rb b/lib/filters/outdated_content.rb deleted file mode 100644 index 536463bf..00000000 --- a/lib/filters/outdated_content.rb +++ /dev/null @@ -1,23 +0,0 @@ -# encoding: utf-8 - -require 'nokogiri' - -class OutdatedContent < ::Nanoc::Filter - identifier :outdated_content - - def run(content, params = {}) - doc = Nokogiri::HTML(content) - # TODO(ts): We need to link to the same page or the first child without hardcoding /getting_started/. - warning = %(

CAUTION: This page documents an old version of #{params[:repository].split("/")[-1].split(".")[0].capitalize()}. - Check out the latest stable version.

) - - body = doc.css('body') - if first = body.children.first - first.add_previous_sibling(warning) - else - body << Nokogiri::HTML::DocumentFragment.parse(warning) - end - - doc.to_s - end -end diff --git a/lib/filters/prerelease_content.rb b/lib/filters/prerelease_content.rb deleted file mode 100644 index 37d1946c..00000000 --- a/lib/filters/prerelease_content.rb +++ /dev/null @@ -1,23 +0,0 @@ -# encoding: utf-8 - -require 'nokogiri' - -class PrerelaseContent < ::Nanoc::Filter - identifier :prerelease_content - - def run(content, params = {}) - doc = Nokogiri::HTML(content) - # TODO(ts): We need to link to the same page or the first child without hardcoding /getting_started/. - warning = %(

CAUTION: This page documents a pre-release version of #{params[:repository].split("/")[-1].split(".")[0].capitalize()}. - Check out the latest stable version.

) - - body = doc.css('body') - if first = body.children.first - first.add_previous_sibling(warning) - else - body << Nokogiri::HTML::DocumentFragment.parse(warning) - end - - doc.to_s - end -end diff --git a/lib/filters/version_warning.rb b/lib/filters/version_warning.rb new file mode 100644 index 00000000..f7ccfa48 --- /dev/null +++ b/lib/filters/version_warning.rb @@ -0,0 +1,44 @@ +# encoding: utf-8 + +require 'nokogiri' + +# VersionWarning adds a warning to the top of pre-release or outdated versioned +# documentation pages. +class VersionWarning < ::Nanoc::Filter + identifier :version_warning + + def run(content, params = {}) + case version_compare(params[:version], params[:latest]) + when 1 + type = 'a pre-release version' + when 0 + return content + when -1 + type = 'an old version' + end + + href = File.join(params[:canonical_root], params[:entrypoint]) + repo = File.basename(params[:repository_url], '.git').capitalize + warning = %(

CAUTION: This page documents #{type} of #{repo}. + Check out the latest stable version.

) + + prepend_warning(content, warning) + end + + private + + def prepend_warning(content, warning) + doc = Nokogiri::HTML(content) + body = doc.css('body') + if first = body.children.first + first.add_previous_sibling(warning) + else + body << Nokogiri::HTML::DocumentFragment.parse(warning) + end + doc.to_s + end + + def version_compare(a, b) + a.split('.').map(&:to_i) <=> b.split('.').map(&:to_i) + end +end diff --git a/lib/helpers/nav.rb b/lib/helpers/nav.rb index a1bc6d58..7fda2a5d 100644 --- a/lib/helpers/nav.rb +++ b/lib/helpers/nav.rb @@ -90,21 +90,16 @@ module Versioned !item[:repo_docs].nil? end - # latest? returns true if the item is part of the version group "latest". - def self.latest?(opts) - opts[:name].include?('latest') - end - # current? returns true if the item is part of the selected version group. If # no group is selected (e.g. when a page outside of the versioned docs is # viewed), the latest version will be shown. def self.current?(opts, page) return false if opts.nil? || !page.respond_to?(:path) - if page.path.start_with?(opts[:version_root]) - page.path.start_with?(opts[:items_root]) + if page.path.start_with?(opts[:items_root]) + page.path.start_with?(opts[:version_root]) else - latest?(opts) + opts[:version_root] == opts[:canonical_root] end end @@ -115,7 +110,7 @@ module Versioned selected = current?(v, page) ? 'selected="selected"' : '' # TODO(ts): Refactor and think about linking directly to the page of the same version. first = items - .find { |i| i.path.start_with?(v[:items_root]) } + .find { |i| i.path.start_with?(v[:version_root]) } .children.sort_by { |c| c[:sort_rank] || 0 }.first %() end diff --git a/nanoc.yaml b/nanoc.yaml index fa458b52..9730c8f6 100644 --- a/nanoc.yaml +++ b/nanoc.yaml @@ -72,207 +72,18 @@ data_sources: type: repo_docs encoding: utf-8 identifier_type: legacy - items_root: /docs/prometheus/latest/ + items_root: /docs/prometheus/ config: - name: 'latest (2.21)' - repository: https://github.com/prometheus/prometheus.git - refspec: release-2.21 - # Use an entry like below to show documentation for release candidates. - # - - # type: repo_docs - # encoding: utf-8 - # items_root: /docs/prometheus/2.20/ - # identifier_type: legacy - # config: - # name: '2.20-rc' - # repository: https://github.com/prometheus/prometheus.git - # refspec: release-2.20 - # canonical: /docs/prometheus/latest/ - # prerelease: /docs/prometheus/latest/getting_started/ - - - type: repo_docs - encoding: utf-8 - items_root: /docs/prometheus/2.21/ - identifier_type: legacy - config: - name: '2.21' - repository: https://github.com/prometheus/prometheus.git - refspec: release-2.21 - canonical: /docs/prometheus/latest/ - - - type: repo_docs - encoding: utf-8 - items_root: /docs/prometheus/2.20/ - identifier_type: legacy - config: - name: '2.20' - repository: https://github.com/prometheus/prometheus.git - refspec: release-2.20 - canonical: /docs/prometheus/latest/ - outdated: /docs/prometheus/latest/getting_started/ - - - type: repo_docs - encoding: utf-8 - items_root: /docs/prometheus/2.19/ - identifier_type: legacy - config: - name: '2.19' - repository: https://github.com/prometheus/prometheus.git - refspec: release-2.19 - canonical: /docs/prometheus/latest/ - outdated: /docs/prometheus/latest/getting_started/ - - - type: repo_docs - encoding: utf-8 - items_root: /docs/prometheus/2.18/ - identifier_type: legacy - config: - name: '2.18' - repository: https://github.com/prometheus/prometheus.git - refspec: release-2.18 - canonical: /docs/prometheus/latest/ - outdated: /docs/prometheus/latest/getting_started/ + entrypoint: /getting_started/ + repository_url: https://github.com/prometheus/prometheus.git - type: repo_docs encoding: utf-8 identifier_type: legacy - items_root: /docs/prometheus/2.17/ + items_root: /docs/alerting/ config: - name: '2.17' - repository: https://github.com/prometheus/prometheus.git - refspec: release-2.17 - canonical: /docs/prometheus/latest/ - outdated: /docs/prometheus/latest/getting_started/ - - - type: repo_docs - encoding: utf-8 - identifier_type: legacy - items_root: /docs/prometheus/2.16/ - config: - name: '2.16' - repository: https://github.com/prometheus/prometheus.git - refspec: release-2.16 - canonical: /docs/prometheus/latest/ - outdated: /docs/prometheus/latest/getting_started/ - - - type: repo_docs - encoding: utf-8 - identifier_type: legacy - items_root: /docs/prometheus/2.15/ - config: - name: '2.15' - repository: https://github.com/prometheus/prometheus.git - refspec: release-2.15 - canonical: /docs/prometheus/latest/ - outdated: /docs/prometheus/latest/getting_started/ - - - type: repo_docs - encoding: utf-8 - identifier_type: legacy - items_root: /docs/prometheus/2.14/ - config: - name: '2.14' - repository: https://github.com/prometheus/prometheus.git - refspec: release-2.14 - canonical: /docs/prometheus/latest/ - outdated: /docs/prometheus/latest/getting_started/ - - - type: repo_docs - encoding: utf-8 - identifier_type: legacy - items_root: /docs/prometheus/2.13/ - config: - name: '2.13' - repository: https://github.com/prometheus/prometheus.git - refspec: release-2.13 - canonical: /docs/prometheus/latest/ - outdated: /docs/prometheus/latest/getting_started/ - - - type: repo_docs - encoding: utf-8 - identifier_type: legacy - items_root: /docs/prometheus/2.12/ - config: - name: '2.12' - repository: https://github.com/prometheus/prometheus.git - refspec: release-2.12 - canonical: /docs/prometheus/latest/ - outdated: /docs/prometheus/latest/getting_started/ - - - type: repo_docs - encoding: utf-8 - identifier_type: legacy - items_root: /docs/prometheus/2.11/ - config: - name: '2.11' - repository: https://github.com/prometheus/prometheus.git - refspec: release-2.11 - canonical: /docs/prometheus/latest/ - outdated: /docs/prometheus/latest/getting_started/ - - - type: repo_docs - encoding: utf-8 - identifier_type: legacy - items_root: /docs/prometheus/2.10/ - config: - name: '2.10' - repository: https://github.com/prometheus/prometheus.git - refspec: release-2.10 - canonical: /docs/prometheus/latest/ - outdated: /docs/prometheus/latest/getting_started/ - - - type: repo_docs - encoding: utf-8 - identifier_type: legacy - items_root: /docs/prometheus/2.9/ - config: - name: '2.9' - repository: https://github.com/prometheus/prometheus.git - refspec: release-2.9 - canonical: /docs/prometheus/latest/ - outdated: /docs/prometheus/latest/getting_started/ - - - type: repo_docs - encoding: utf-8 - identifier_type: legacy - items_root: /docs/prometheus/1.8/ - config: - name: '1.8' - repository: https://github.com/prometheus/prometheus.git - refspec: release-1.8 - canonical: /docs/prometheus/latest/ - outdated: /docs/prometheus/latest/getting_started/ - - - type: repo_docs - encoding: utf-8 - identifier_type: legacy - items_root: /docs/alerting/latest/ - config: - name: 'latest (0.21)' - repository: https://github.com/prometheus/alertmanager.git - refspec: release-0.21 - canonical: /docs/alerting/latest/ - - - type: repo_docs - encoding: utf-8 - identifier_type: legacy - items_root: /docs/alerting/0.21/ - config: - name: '0.21' - repository: https://github.com/prometheus/alertmanager.git - refspec: release-0.21 - canonical: /docs/alerting/latest/ - - - type: repo_docs - encoding: utf-8 - identifier_type: legacy - items_root: /docs/alerting/0.20/ - config: - name: '0.20' - repository: https://github.com/prometheus/alertmanager.git - refspec: release-0.20 - canonical: /docs/alerting/latest/ - outdated: /docs/alerting/latest/overview/ + entrypoint: /overview/ + repository_url: https://github.com/prometheus/alertmanager.git - type: filesystem items_root: /assets diff --git a/scripts/checkout.sh b/scripts/checkout.sh deleted file mode 100755 index 550b67df..00000000 --- a/scripts/checkout.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash - -usage() { -me=$(basename $0) -cat < [ ] - -Options: - -d Remote directory name of the sparse-checkout. Default: docs/ - -t Target path of the checkout. Default: repository basename - -Example: - ./checkout.sh https://github.com/prometheus/docs.git -EOF - -exit 1 -} - -while getopts 'd:t:' OPT -do - case ${OPT} in - d) - DIRECTORY="${OPTARG}" - ;; - t) - TARGET="${OPTARG}" - ;; - *) - usage - ;; - esac -done - -shift $((OPTIND-1)) - -[ $# -ge 1 ] || usage - -REPOSITORY="$1" -REFSPEC="$2" - -if [[ -z "${DIRECTORY}" ]]; then - DIRECTORY="docs/" -fi - -if [[ -z "${TARGET}" ]]; then - TARGET=$(basename "${REPOSITORY}") -fi - -mkdir -p "${TARGET}" -cd "${TARGET}" - -git init - -git config core.sparsecheckout true -echo "${DIRECTORY}" > .git/info/sparse-checkout - -if ! git remote | grep -q origin; then - git remote add origin "${REPOSITORY}" -fi -git fetch --depth=1 origin "${REFSPEC}" -git reset --hard origin/"${REFSPEC}"