import { execSync } from "child_process"; import * as fs from "fs"; import * as path from "path"; import docsConfig from "../docs-config"; import { GithubMarkdownSource } from "../src/docs-config-types"; import { DocsCollection, AllRepoVersions, RepoDocMetadata, } from "@/docs-collection-types"; import matter from "gray-matter"; import { octokit } from "./githubClient"; import { compareFullVersion, filterUnique, majorMinor } from "./utils"; const OUTDIR = "./generated"; const docsCollection: DocsCollection = {}; const allRepoVersions: AllRepoVersions = {}; // Find all files (.md and others) recursively in a directory. const findFiles = (dir: string): string[] => { let results: string[] = []; const entries = fs.readdirSync(dir, { withFileTypes: true }); for (const entry of entries) { const fullPath = path.join(dir, entry.name); if (entry.isDirectory()) { results = results.concat(findFiles(fullPath)); } else if (entry.isFile()) { results.push(fullPath); } } return results; }; const syncRepo = (owner: string, repo: string, repoDir: string) => { console.log(`Syncing ${repo} repo to ${repoDir}...`); if (!fs.existsSync(repoDir)) { execSync( `git clone --bare --filter=blob:none https://github.com/${owner}/${repo}.git ${repoDir}` ); execSync(`git -C ${repoDir} config core.sparseCheckout true`); } else { execSync(`git -C ${repoDir} fetch --prune --quiet`); } }; const checkoutVersionDocs = ( owner: string, repo: string, repoDir: string, version: string, workingTreeBase: string, repoDocsDir: string ) => { const workingTree = path.resolve( `${workingTreeBase}/${owner}/${repo}/${version}` ); const checkoutConfig = `${repoDir}/worktrees/${version}/info/sparse-checkout`; const branch = `release-${version}`; console.log( `Checking out ${branch} of ${owner}/${repo} from ${repoDir} into ${workingTree}...` ); if (!fs.existsSync(checkoutConfig) || !fs.existsSync(workingTree)) { console.log("Creating new worktree..."); // Just a safety check before running rm -rf. // TODO: Maybe remove this. if (workingTree.length > 1) { execSync(`rm -rf ${workingTree}`); } execSync( `cd ${repoDir} && git worktree prune && git worktree add --no-checkout ${workingTree} ${branch}` ); if (!fs.existsSync(path.dirname(checkoutConfig))) { fs.mkdirSync(path.dirname(checkoutConfig), { recursive: true }); } fs.writeFileSync(checkoutConfig, `/${repoDocsDir}\n`); } else { console.log("Worktree already exists..."); } execSync(`git -C ${workingTree} reset --hard --quiet`); execSync(`git -C ${workingTree} clean --force`); }; const fetchRepoDocs = async ({ owner, repo, repoDocsDir, minNumVersions, slugPrefix, }: GithubMarkdownSource) => { console.log(`Fetching releases and repo docs for ${owner}/${repo}...`); // Clone a bare repo with sparse checkout so we can get the docs at specific // versions from it later on. const repoCheckoutDir = `${OUTDIR}/repos/${owner}/${repo}.git`; syncRepo(owner, repo, repoCheckoutDir); // List all GitHub releases for the repo. const iterator = octokit.paginate.iterator(octokit.rest.repos.listReleases, { owner, repo, per_page: 100, }); const allReleaseTags: string[] = []; for await (const { data: releases } of iterator) { allReleaseTags.push(...releases.map((r) => r.tag_name)); // For the Prometheus repo for efficieny-reasons, stop once we have found at least // one release starting with "v1.". // Note: releases are sorted by date, so this works ok. // TODO: Do we even still want to show the latest v1 release? if ( owner === "prometheus" && repo === "prometheus" && releases.find((release) => release.tag_name.startsWith("v1.")) ) { break; } } // We still need to sort the releases by version number, as e.g. "2.53.4" // was released after "3.2.1".. allReleaseTags.sort(compareFullVersion).reverse(); // Get all . versions, regardless of the patch version. const allVersions = allReleaseTags .filter((tag) => tag.startsWith("v")) // Ignore prehistoric release tags like "0.1.0" .map((tag) => majorMinor(tag)) .filter(filterUnique); // Remove dupes (e.g. "3.4.0" and "3.4.1" both become "3.4") // First, get the last 10 versions, regardless of the major version. const recentVersions = allVersions.slice(0, minNumVersions); // Then, ensure we include at least one release for each major version. const groupedByMajor = Object.groupBy( allVersions, (version) => version.split(".")[0] ); for (const major in groupedByMajor) { const latestInMajor = groupedByMajor[major]![0]; if (!recentVersions.includes(latestInMajor)) { recentVersions.push(latestInMajor); } } const latestTag = allReleaseTags.find((tag) => !tag.includes("-")); if (!latestTag) { throw new Error(`No latest version found for ${owner}/${repo}.`); } const latestVersion = majorMinor(latestTag); // Store metadata about the repo and its versions. if (!allRepoVersions[owner]) { allRepoVersions[owner] = {}; } allRepoVersions[owner][repo] = { versions: recentVersions, latestVersion, ltsVersions: (owner === "prometheus" && docsConfig.ltsVersions[repo]) || [], }; console.log( `Found ${allReleaseTags.length} releases, ${allVersions.length} major/minor versions, ${recentVersions.length} recent versions, and the latest version ${latestVersion} for ${owner}/${repo}.` ); // For each recent version, check out the corresponding commit and // produce a directory containing its docs, then add it to the metadata. for (const version of recentVersions) { checkoutVersionDocs( owner, repo, repoCheckoutDir, version, `${OUTDIR}/repo-docs`, repoDocsDir ); const docsDir = `${OUTDIR}/repo-docs/${owner}/${repo}/${version}/${repoDocsDir}`; // Store metadata about Markdown page files, copy non-Markdown // assets to the docs assets directory. const assetsRoot = `/repo-docs-assets/${owner}/${repo}/${version}`; for (const file of findFiles(docsDir)) { const filePath = path.relative(docsDir, file); if ( owner === "prometheus" && ["prometheus", "alertmanager"].includes(repo) && filePath === "index.md" ) { // Skip the index.md file in the external repo, as it is not a real or conformant page. console.log("Skipping Prometheus index.md file:", filePath); continue; } if (file.endsWith(".md")) { console.log("Found Markdown file:", filePath); const { data: { title, nav_title: navTitle, sort_rank: sortRank }, content, } = matter(fs.readFileSync(file, "utf-8")); if (!title) { throw new Error(`Missing title in ${file}`); } if (!sortRank) { // Docs in https://github.com/prometheus/prometheus/tree/main/docs/command-line // are currently missing sort_rank 😤 if (!filePath.includes("command-line")) { throw new Error(`Missing sort_rank in ${file}`); } } const newDoc: RepoDocMetadata = { type: "repo-doc", filePath: file, content: "", owner, repo, version, slugPrefix, latestVersion, versionRoot: path.join(slugPrefix, version), assetsRoot, title, navTitle, sortRank: sortRank ?? 0, }; docsCollection[ path.join(slugPrefix, version, filePath.replace(/\.md$/, "")) ] = newDoc; if (version === latestVersion) { // Also add the latest version to the collection with // "latest" as the version in the slug. docsCollection[ path.join(slugPrefix, "latest", filePath.replace(/\.md$/, "")) ] = newDoc; } } else { console.log("Found non-Markdown asset file:", filePath); const destDir = `${OUTDIR}/${assetsRoot}/${path.dirname(filePath)}`; if (!fs.existsSync(destDir)) { fs.mkdirSync(destDir, { recursive: true }); } fs.copyFileSync(file, `${destDir}/${path.basename(filePath)}`); } } } }; for (const sourceConfig of docsConfig.githubMarkdownSources) { await fetchRepoDocs(sourceConfig); } for (const sourceConfig of docsConfig.localMarkdownSources) { const { docsDir, slugPrefix } = sourceConfig; console.log(`Integrating local docs from ${docsDir}...`); for (const file of findFiles(docsDir)) { if (!file.endsWith(".md")) { throw new Error( `Found non-Markdown file ${file} in local docs dir, please put assets into public/.` ); } const filePath = path.relative(docsDir, file); const { data: { title, sort_rank: sortRank, nav_icon: navIcon }, content, } = matter(fs.readFileSync(file, "utf-8")); if (!title) { throw new Error(`Missing title in ${file}`); } if (!sortRank) { // A number of guides in https://github.com/prometheus/docs/tree/main/content/docs/guides // are currently missing sort_rank 😤 if (!filePath.includes("guides")) { throw new Error(`Missing sort_rank in ${file}`); } } if (file.endsWith(".md")) { console.log("Found Markdown file:", filePath); docsCollection[path.join(slugPrefix, filePath.replace(/\.md$/, ""))] = { type: "local-doc", filePath: file, content: "", title, sortRank: sortRank ?? 0, navIcon, }; } // Don't need to care about assets for local docs here, local doc authors // should store them in public/ and link to them there. } } // Write out the docs collection metadata object to a JSON file. const docsCollectionFile = `${OUTDIR}/docs-collection.json`; const allRepoVersionsFile = `${OUTDIR}/repo-versions.json`; fs.writeFileSync(docsCollectionFile, JSON.stringify(docsCollection, null, 2)); fs.writeFileSync(allRepoVersionsFile, JSON.stringify(allRepoVersions, null, 2));