From c2a7d606e2fa088014e117cb2afe0ddd5fc0ea10 Mon Sep 17 00:00:00 2001 From: Antheas Kapenekakis Date: Wed, 11 Dec 2024 21:16:54 +0100 Subject: [PATCH] docs: Describe `--progress-fd`, add rendered JSON schema Signed-off-by: Antheas Kapenekakis Signed-off-by: Colin Walters --- docs/src/SUMMARY.md | 1 + docs/src/bootc-via-api.md | 2 +- docs/src/experimental-progress-fd.md | 33 ++++ docs/src/progress-v1.schema.json | 230 +++++++++++++++++++++++++++ lib/src/cli.rs | 18 ++- lib/src/progress_jsonl.rs | 7 +- xtask/src/xtask.rs | 12 +- 7 files changed, 292 insertions(+), 11 deletions(-) create mode 100644 docs/src/experimental-progress-fd.md create mode 100644 docs/src/progress-v1.schema.json diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index 659bf211..3c289037 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -55,6 +55,7 @@ # Experimental features - [bootc image](experimental-bootc-image.md) +- [--progress-fd](experimental-progress-fd.md) # More information diff --git a/docs/src/bootc-via-api.md b/docs/src/bootc-via-api.md index fdfdf793..c2a5ea77 100644 --- a/docs/src/bootc-via-api.md +++ b/docs/src/bootc-via-api.md @@ -7,7 +7,7 @@ are stable and will not change. ## Using `bootc edit` and `bootc status --json` While bootc does not depend on Kubernetes, it does currently -also offere a Kubernetes *style* API, especially oriented +also offer a Kubernetes *style* API, especially oriented towards the [spec and status and other conventions](https://kubernetes.io/docs/reference/using-api/api-concepts/). In general, most use cases of driving bootc via API are probably diff --git a/docs/src/experimental-progress-fd.md b/docs/src/experimental-progress-fd.md new file mode 100644 index 00000000..b2d6c73c --- /dev/null +++ b/docs/src/experimental-progress-fd.md @@ -0,0 +1,33 @@ + +# Interactive progress with `--progress-fd` + +This is an experimental feature; tracking issue: + +While the `bootc status` tooling allows a client to discover the state +of the system, during interactive changes such as `bootc upgrade` +or `bootc switch` it is possible to monitor the status of downloads +or other operations at a fine-grained level with `-progress-fd`. + +The format of data output over `--progress-fd` is [JSON Lines](https://jsonlines.org) +which is a series of JSON objects separated by newlines (the intermediate +JSON content is guaranteed not to contain a literal newline). + +The current API version is `org.containers.bootc/progress/v1`. You can find +the JSON schema describing this version here: +[progress-v1.schema.json](progress-v1.schema.json). + +Deploying a new image with either switch or upgrade consists +of three stages: `pulling`, `importing`, and `staging`. The `pulling` step +downloads the image from the registry, offering per-layer and progress in +each message. The `importing` step imports the image into storage and consists +of a single step. Finally, `staging` runs a variety of staging +tasks. Currently, they are staging the image to disk, pulling bound images, +and removing old images. + +Note that new stages or fields may be added at any time. + +Importing and staging are affected by disk speed and the total image size. Pulling +is affected by network speed and how many layers invalidate between pulls. +Therefore, a large image with a good caching strategy will have longer +importing and staging times, and a small bespoke container image will have +negligible importing and staging times. \ No newline at end of file diff --git a/docs/src/progress-v1.schema.json b/docs/src/progress-v1.schema.json new file mode 100644 index 00000000..aeba668d --- /dev/null +++ b/docs/src/progress-v1.schema.json @@ -0,0 +1,230 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Event", + "description": "An event emitted as JSON.", + "oneOf": [ + { + "description": "An incremental update to a container image layer download", + "type": "object", + "required": [ + "api_version", + "bytes", + "bytes_cached", + "bytes_total", + "description", + "id", + "steps", + "steps_cached", + "steps_total", + "subtasks", + "task", + "type" + ], + "properties": { + "api_version": { + "description": "The version of the progress event format.", + "type": "string" + }, + "bytes": { + "description": "The number of bytes already fetched.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "bytes_cached": { + "description": "The number of bytes fetched by a previous run.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "bytes_total": { + "description": "Total number of bytes. If zero, then this should be considered \"unspecified\".", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "description": { + "description": "A human readable description of the task if i18n is not available.", + "type": "string" + }, + "id": { + "description": "A human and machine readable unique identifier for the task (e.g., the image name). For tasks that only happen once, it can be set to the same value as task.", + "type": "string" + }, + "steps": { + "description": "The initial position of progress.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "steps_cached": { + "description": "The number of steps fetched by a previous run.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "steps_total": { + "description": "The total number of steps (e.g. container image layers, RPMs)", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "subtasks": { + "description": "The currently running subtasks.", + "type": "array", + "items": { + "$ref": "#/definitions/SubTaskBytes" + } + }, + "task": { + "description": "A machine readable type (e.g., pulling) for the task (used for i18n and UI customization).", + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "ProgressBytes" + ] + } + } + }, + { + "description": "An incremental update with discrete steps", + "type": "object", + "required": [ + "api_version", + "description", + "id", + "steps", + "steps_cached", + "steps_total", + "subtasks", + "task", + "type" + ], + "properties": { + "api_version": { + "description": "The version of the progress event format.", + "type": "string" + }, + "description": { + "description": "A human readable description of the task if i18n is not available.", + "type": "string" + }, + "id": { + "description": "A human and machine readable unique identifier for the task (e.g., the image name). For tasks that only happen once, it can be set to the same value as task.", + "type": "string" + }, + "steps": { + "description": "The initial position of progress.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "steps_cached": { + "description": "The number of steps fetched by a previous run.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "steps_total": { + "description": "The total number of steps (e.g. container image layers, RPMs)", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "subtasks": { + "description": "The currently running subtasks.", + "type": "array", + "items": { + "$ref": "#/definitions/SubTaskStep" + } + }, + "task": { + "description": "A machine readable type (e.g., pulling) for the task (used for i18n and UI customization).", + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "ProgressSteps" + ] + } + } + } + ], + "definitions": { + "SubTaskBytes": { + "description": "An incremental update to e.g. a container image layer download. The first time a given \"subtask\" name is seen, a new progress bar should be created. If bytes == bytes_total, then the subtask is considered complete.", + "type": "object", + "required": [ + "bytes", + "bytesCached", + "bytesTotal", + "description", + "id", + "subtask" + ], + "properties": { + "bytes": { + "description": "Updated byte level progress", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "bytesCached": { + "description": "The number of bytes fetched by a previous run (e.g., zstd_chunked).", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "bytesTotal": { + "description": "Total number of bytes", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "description": { + "description": "A human readable description of the task if i18n is not available. (e.g., \"OSTree Chunk:\", \"Derived Layer:\")", + "type": "string" + }, + "id": { + "description": "A human and machine readable identifier for the task (e.g., ostree chunk/layer hash).", + "type": "string" + }, + "subtask": { + "description": "A machine readable type for the task (used for i18n). (e.g., \"ostree_chunk\", \"ostree_derived\")", + "type": "string" + } + } + }, + "SubTaskStep": { + "description": "Marks the beginning and end of a dictrete step", + "type": "object", + "required": [ + "completed", + "description", + "id", + "subtask" + ], + "properties": { + "completed": { + "description": "Starts as false when beginning to execute and turns true when completed.", + "type": "boolean" + }, + "description": { + "description": "A human readable description of the task if i18n is not available. (e.g., \"OSTree Chunk:\", \"Derived Layer:\")", + "type": "string" + }, + "id": { + "description": "A human and machine readable identifier for the task (e.g., ostree chunk/layer hash).", + "type": "string" + }, + "subtask": { + "description": "A machine readable type for the task (used for i18n). (e.g., \"ostree_chunk\", \"ostree_derived\")", + "type": "string" + } + } + } + } +} \ No newline at end of file diff --git a/lib/src/cli.rs b/lib/src/cli.rs index 808dd301..63bb9caf 100644 --- a/lib/src/cli.rs +++ b/lib/src/cli.rs @@ -359,6 +359,12 @@ pub(crate) enum ImageOpts { Cmd(ImageCmdOpts), } +#[derive(Debug, Clone, clap::ValueEnum, PartialEq, Eq)] +pub(crate) enum SchemaType { + Host, + Progress, +} + /// Hidden, internal only options #[derive(Debug, clap::Subcommand, PartialEq, Eq)] pub(crate) enum InternalsOpts { @@ -371,7 +377,10 @@ pub(crate) enum InternalsOpts { }, FixupEtcFstab, /// Should only be used by `make update-generated` - PrintJsonSchema, + PrintJsonSchema { + #[clap(long)] + of: SchemaType, + }, /// Perform cleanup actions Cleanup, /// Proxy frontend for the `ostree-ext` CLI. @@ -1090,8 +1099,11 @@ async fn run_from_opt(opt: Opt) -> Result<()> { .await } InternalsOpts::FixupEtcFstab => crate::deploy::fixup_etc_fstab(&root), - InternalsOpts::PrintJsonSchema => { - let schema = schema_for!(crate::spec::Host); + InternalsOpts::PrintJsonSchema { of } => { + let schema = match of { + SchemaType::Host => schema_for!(crate::spec::Host), + SchemaType::Progress => schema_for!(crate::progress_jsonl::Event), + }; let mut stdout = std::io::stdout().lock(); serde_json::to_writer_pretty(&mut stdout, &schema)?; Ok(()) diff --git a/lib/src/progress_jsonl.rs b/lib/src/progress_jsonl.rs index acee1284..5fb3b577 100644 --- a/lib/src/progress_jsonl.rs +++ b/lib/src/progress_jsonl.rs @@ -2,6 +2,7 @@ //! see . use anyhow::Result; +use schemars::JsonSchema; use serde::Serialize; use std::borrow::Cow; use std::os::fd::{FromRawFd, OwnedFd, RawFd}; @@ -20,7 +21,7 @@ pub const API_VERSION: &str = "org.containers.bootc.progress/v1"; /// An incremental update to e.g. a container image layer download. /// The first time a given "subtask" name is seen, a new progress bar should be created. /// If bytes == bytes_total, then the subtask is considered complete. -#[derive(Debug, serde::Serialize, serde::Deserialize, Default, Clone)] +#[derive(Debug, serde::Serialize, serde::Deserialize, Default, Clone, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct SubTaskBytes<'t> { /// A machine readable type for the task (used for i18n). @@ -44,7 +45,7 @@ pub struct SubTaskBytes<'t> { } /// Marks the beginning and end of a dictrete step -#[derive(Debug, serde::Serialize, serde::Deserialize, Default, Clone)] +#[derive(Debug, serde::Serialize, serde::Deserialize, Default, Clone, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct SubTaskStep<'t> { /// A machine readable type for the task (used for i18n). @@ -64,7 +65,7 @@ pub struct SubTaskStep<'t> { } /// An event emitted as JSON. -#[derive(Debug, serde::Serialize, serde::Deserialize)] +#[derive(Debug, serde::Serialize, serde::Deserialize, JsonSchema)] #[serde( tag = "type", rename_all = "PascalCase", diff --git a/xtask/src/xtask.rs b/xtask/src/xtask.rs index 0cb93901..6132d055 100644 --- a/xtask/src/xtask.rs +++ b/xtask/src/xtask.rs @@ -147,10 +147,14 @@ fn update_generated(sh: &Shell) -> Result<()> { ) .run()?; } - let schema = cmd!(sh, "cargo run -q -- internals print-json-schema").read()?; - let target = "docs/src/host-v1.schema.json"; - std::fs::write(target, &schema)?; - println!("Updated {target}"); + for (of, target) in [ + ("host", "docs/src/host-v1.schema.json"), + ("progress", "docs/src/progress-v1.schema.json"), + ] { + let schema = cmd!(sh, "cargo run -q -- internals print-json-schema --of={of}").read()?; + std::fs::write(target, &schema)?; + println!("Updated {target}"); + } Ok(()) }