mirror of
https://github.com/prometheus/docs.git
synced 2026-02-05 15:45:27 +01:00
Initial docs site commit
Signed-off-by: Julius Volz <julius.volz@gmail.com>
This commit is contained in:
Julius Volz
334
scripts/fetch-repo-docs.ts
Normal file
334
scripts/fetch-repo-docs.ts
Normal file
@@ -0,0 +1,334 @@
|
||||
import { Octokit } from "octokit";
|
||||
import { execSync } from "child_process";
|
||||
import * as fs from "fs";
|
||||
import * as path from "path";
|
||||
import docsConfig from "../docs-config";
|
||||
import { GithubMarkdownSource } from "../src/docs-config-types";
|
||||
import {
|
||||
DocsCollection,
|
||||
AllRepoVersions,
|
||||
RepoDocMetadata,
|
||||
} from "@/docs-collection-types";
|
||||
import matter from "gray-matter";
|
||||
import dotenv from "dotenv";
|
||||
|
||||
dotenv.config();
|
||||
|
||||
const OUTDIR = "./generated";
|
||||
|
||||
const octokit = new Octokit({
|
||||
auth: `${process.env.GITHUB_TOKEN}`,
|
||||
});
|
||||
|
||||
const docsCollection: DocsCollection = {};
|
||||
const allRepoVersions: AllRepoVersions = {};
|
||||
|
||||
// Find all files (.md and others) recursively in a directory.
|
||||
const findFiles = (dir: string): string[] => {
|
||||
let results: string[] = [];
|
||||
const entries = fs.readdirSync(dir, { withFileTypes: true });
|
||||
|
||||
for (const entry of entries) {
|
||||
const fullPath = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
results = results.concat(findFiles(fullPath));
|
||||
} else if (entry.isFile()) {
|
||||
results.push(fullPath);
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
};
|
||||
|
||||
const syncRepo = (owner: string, repo: string, repoDir: string) => {
|
||||
console.log(`Syncing ${repo} repo to ${repoDir}...`);
|
||||
if (!fs.existsSync(repoDir)) {
|
||||
execSync(
|
||||
`git clone --bare --filter=blob:none https://github.com/${owner}/${repo}.git ${repoDir}`
|
||||
);
|
||||
execSync(`git -C ${repoDir} config core.sparseCheckout true`);
|
||||
} else {
|
||||
execSync(`git -C ${repoDir} fetch --prune --quiet`);
|
||||
}
|
||||
};
|
||||
|
||||
const checkoutVersionDocs = (
|
||||
owner: string,
|
||||
repo: string,
|
||||
repoDir: string,
|
||||
version: string,
|
||||
workingTreeBase: string,
|
||||
repoDocsDir: string
|
||||
) => {
|
||||
const workingTree = path.resolve(
|
||||
`${workingTreeBase}/${owner}/${repo}/${version}`
|
||||
);
|
||||
const checkoutConfig = `${repoDir}/worktrees/${version}/info/sparse-checkout`;
|
||||
const branch = `release-${version}`;
|
||||
|
||||
console.log(
|
||||
`Checking out ${branch} of ${owner}/${repo} from ${repoDir} into ${workingTree}...`
|
||||
);
|
||||
|
||||
if (!fs.existsSync(checkoutConfig) || !fs.existsSync(workingTree)) {
|
||||
console.log("Creating new worktree...");
|
||||
// Just a safety check before running rm -rf.
|
||||
// TODO: Maybe remove this.
|
||||
if (workingTree.length > 1) {
|
||||
execSync(`rm -rf ${workingTree}`);
|
||||
}
|
||||
execSync(
|
||||
`cd ${repoDir} && git worktree prune && git worktree add --no-checkout ${workingTree} ${branch}`
|
||||
);
|
||||
if (!fs.existsSync(path.dirname(checkoutConfig))) {
|
||||
fs.mkdirSync(path.dirname(checkoutConfig), { recursive: true });
|
||||
}
|
||||
fs.writeFileSync(checkoutConfig, `/${repoDocsDir}\n`);
|
||||
} else {
|
||||
console.log("Worktree already exists...");
|
||||
}
|
||||
|
||||
execSync(`git -C ${workingTree} reset --hard --quiet`);
|
||||
execSync(`git -C ${workingTree} clean --force`);
|
||||
};
|
||||
|
||||
const fetchRepoDocs = async ({
|
||||
owner,
|
||||
repo,
|
||||
repoDocsDir,
|
||||
minNumVersions,
|
||||
slugPrefix,
|
||||
}: GithubMarkdownSource) => {
|
||||
console.log(`Fetching releases and repo docs for ${owner}/${repo}...`);
|
||||
|
||||
// Clone a bare repo with sparse checkout so we can get the docs at specific
|
||||
// versions from it later on.
|
||||
const repoCheckoutDir = `${OUTDIR}/repos/${owner}/${repo}.git`;
|
||||
syncRepo(owner, repo, repoCheckoutDir);
|
||||
|
||||
// List all GitHub releases for the repo.
|
||||
const iterator = octokit.paginate.iterator(octokit.rest.repos.listReleases, {
|
||||
owner,
|
||||
repo,
|
||||
per_page: 100,
|
||||
});
|
||||
|
||||
const allReleaseTags: string[] = [];
|
||||
|
||||
for await (const { data: releases } of iterator) {
|
||||
allReleaseTags.push(...releases.map((r) => r.tag_name));
|
||||
// For the Prometheus repo for efficieny-reasons, stop once we have found at least
|
||||
// one release starting with "v1.".
|
||||
// Note: releases are sorted by date, so this works ok.
|
||||
// TODO: Do we even still want to show the latest v1 release?
|
||||
if (
|
||||
owner === "prometheus" &&
|
||||
repo === "prometheus" &&
|
||||
releases.find((release) => release.tag_name.startsWith("v1."))
|
||||
) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
// We still need to sort the releases by version number, as e.g. "2.53.4"
|
||||
// was released after "3.2.1".
|
||||
//
|
||||
// Sorting by major + minor should be sufficient, as patch versions and
|
||||
// other suffixes should be in the expected order already when sorting by
|
||||
// release date.
|
||||
allReleaseTags
|
||||
.sort((a, b) => {
|
||||
const [majorA, minorA] = a.replace(/^v/, "").split(".").map(Number);
|
||||
const [majorB, minorB] = b.replace(/^v/, "").split(".").map(Number);
|
||||
return majorA === majorB ? minorA - minorB : majorA - majorB;
|
||||
})
|
||||
.reverse();
|
||||
|
||||
function onlyUnique(value: string, index: number, array: string[]) {
|
||||
return array.indexOf(value) === index;
|
||||
}
|
||||
|
||||
// "v3.4.1" -> "3.4"
|
||||
function shortVersion(version: string) {
|
||||
return version.replace(/^v/, "").split(".").slice(0, 2).join(".");
|
||||
}
|
||||
|
||||
// Get all <major>.<minor> versions, regardless of the patch version.
|
||||
const allVersions = allReleaseTags
|
||||
.filter((tag) => tag.startsWith("v")) // Ignore prehistoric release tags like "0.1.0"
|
||||
.map((tag) => shortVersion(tag))
|
||||
.filter(onlyUnique); // Remove dupes (e.g. "3.4.0" and "3.4.1" both become "3.4")
|
||||
|
||||
// First, get the last 10 versions, regardless of the major version.
|
||||
const recentVersions = allVersions.slice(0, minNumVersions);
|
||||
|
||||
// Then, ensure we include at least one release for each major version.
|
||||
const groupedByMajor = Object.groupBy(
|
||||
allVersions,
|
||||
(version) => version.split(".")[0]
|
||||
);
|
||||
for (const major in groupedByMajor) {
|
||||
const latestInMajor = groupedByMajor[major]![0];
|
||||
if (!recentVersions.includes(latestInMajor)) {
|
||||
recentVersions.push(latestInMajor);
|
||||
}
|
||||
}
|
||||
|
||||
const latestTag = allReleaseTags.find((tag) => !tag.includes("-"));
|
||||
if (!latestTag) {
|
||||
throw new Error(`No latest version found for ${owner}/${repo}.`);
|
||||
}
|
||||
const latestVersion = shortVersion(latestTag);
|
||||
|
||||
// Store metadata about the repo and its versions.
|
||||
if (!allRepoVersions[owner]) {
|
||||
allRepoVersions[owner] = {};
|
||||
}
|
||||
allRepoVersions[owner][repo] = {
|
||||
versions: recentVersions,
|
||||
latestVersion,
|
||||
ltsVersions: (owner === "prometheus" && docsConfig.ltsVersions[repo]) || [],
|
||||
};
|
||||
|
||||
console.log(
|
||||
`Found ${allReleaseTags.length} releases, ${allVersions.length} major/minor versions, ${recentVersions.length} recent versions, and the latest version ${latestVersion} for ${owner}/${repo}.`
|
||||
);
|
||||
|
||||
// For each recent version, check out the corresponding commit and
|
||||
// produce a directory containing its docs, then add it to the metadata.
|
||||
for (const version of recentVersions) {
|
||||
checkoutVersionDocs(
|
||||
owner,
|
||||
repo,
|
||||
repoCheckoutDir,
|
||||
version,
|
||||
`${OUTDIR}/repo-docs`,
|
||||
repoDocsDir
|
||||
);
|
||||
const docsDir = `${OUTDIR}/repo-docs/${owner}/${repo}/${version}/${repoDocsDir}`;
|
||||
|
||||
// Store metadata about Markdown page files, copy non-Markdown
|
||||
// assets to the docs assets directory.
|
||||
const assetsRoot = `/repo-docs-assets/${owner}/${repo}/${version}`;
|
||||
for (const file of findFiles(docsDir)) {
|
||||
const filePath = path.relative(docsDir, file);
|
||||
|
||||
if (
|
||||
owner === "prometheus" &&
|
||||
["prometheus", "alertmanager"].includes(repo) &&
|
||||
filePath === "index.md"
|
||||
) {
|
||||
// Skip the index.md file in the external repo, as it is not a real or conformant page.
|
||||
console.log("Skipping Prometheus index.md file:", filePath);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (file.endsWith(".md")) {
|
||||
console.log("Found Markdown file:", filePath);
|
||||
|
||||
const {
|
||||
data: { title, nav_title: navTitle, sort_rank: sortRank },
|
||||
content,
|
||||
} = matter(fs.readFileSync(file, "utf-8"));
|
||||
|
||||
if (!title) {
|
||||
throw new Error(`Missing title in ${file}`);
|
||||
}
|
||||
if (!sortRank) {
|
||||
// Docs in https://github.com/prometheus/prometheus/tree/main/docs/command-line
|
||||
// are currently missing sort_rank 😤
|
||||
if (!filePath.includes("command-line")) {
|
||||
throw new Error(`Missing sort_rank in ${file}`);
|
||||
}
|
||||
}
|
||||
|
||||
const newDoc: RepoDocMetadata = {
|
||||
type: "repo-doc",
|
||||
filePath: file,
|
||||
content: "",
|
||||
owner,
|
||||
repo,
|
||||
version,
|
||||
slugPrefix,
|
||||
latestVersion,
|
||||
versionRoot: path.join(slugPrefix, version),
|
||||
assetsRoot,
|
||||
title,
|
||||
navTitle,
|
||||
sortRank: sortRank ?? 0,
|
||||
};
|
||||
|
||||
docsCollection[
|
||||
path.join(slugPrefix, version, filePath.replace(/\.md$/, ""))
|
||||
] = newDoc;
|
||||
|
||||
if (version === latestVersion) {
|
||||
// Also add the latest version to the collection with
|
||||
// "latest" as the version in the slug.
|
||||
docsCollection[
|
||||
path.join(slugPrefix, "latest", filePath.replace(/\.md$/, ""))
|
||||
] = newDoc;
|
||||
}
|
||||
} else {
|
||||
console.log("Found non-Markdown asset file:", filePath);
|
||||
const destDir = `${OUTDIR}/${assetsRoot}/${path.dirname(filePath)}`;
|
||||
if (!fs.existsSync(destDir)) {
|
||||
fs.mkdirSync(destDir, { recursive: true });
|
||||
}
|
||||
fs.copyFileSync(file, `${destDir}/${path.basename(filePath)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
for (const sourceConfig of docsConfig.githubMarkdownSources) {
|
||||
await fetchRepoDocs(sourceConfig);
|
||||
}
|
||||
|
||||
for (const sourceConfig of docsConfig.localMarkdownSources) {
|
||||
const { docsDir, slugPrefix } = sourceConfig;
|
||||
console.log(`Integrating local docs from ${docsDir}...`);
|
||||
for (const file of findFiles(docsDir)) {
|
||||
if (!file.endsWith(".md")) {
|
||||
throw new Error(
|
||||
`Found non-Markdown file ${file} in local docs dir, please put assets into public/.`
|
||||
);
|
||||
}
|
||||
|
||||
const filePath = path.relative(docsDir, file);
|
||||
const {
|
||||
data: { title, sort_rank: sortRank, nav_icon: navIcon },
|
||||
content,
|
||||
} = matter(fs.readFileSync(file, "utf-8"));
|
||||
if (!title) {
|
||||
throw new Error(`Missing title in ${file}`);
|
||||
}
|
||||
if (!sortRank) {
|
||||
// A number of guides in https://github.com/prometheus/docs/tree/main/content/docs/guides
|
||||
// are currently missing sort_rank 😤
|
||||
if (!filePath.includes("guides")) {
|
||||
throw new Error(`Missing sort_rank in ${file}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (file.endsWith(".md")) {
|
||||
console.log("Found Markdown file:", filePath);
|
||||
docsCollection[path.join(slugPrefix, filePath.replace(/\.md$/, ""))] = {
|
||||
type: "local-doc",
|
||||
filePath: file,
|
||||
content: "",
|
||||
title,
|
||||
sortRank: sortRank ?? 0,
|
||||
navIcon,
|
||||
};
|
||||
}
|
||||
|
||||
// Don't need to care about assets for local docs here, local doc authors
|
||||
// should store them in public/ and link to them there.
|
||||
}
|
||||
}
|
||||
|
||||
// Write out the docs collection metadata object to a JSON file.
|
||||
const docsCollectionFile = `${OUTDIR}/docs-collection.json`;
|
||||
const allRepoVersionsFile = `${OUTDIR}/repo-versions.json`;
|
||||
fs.writeFileSync(docsCollectionFile, JSON.stringify(docsCollection, null, 2));
|
||||
fs.writeFileSync(allRepoVersionsFile, JSON.stringify(allRepoVersions, null, 2));
|
||||
Reference in New Issue
Block a user