diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 000000000..98953e3ce --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,54 @@ +name: Deploy Documentation + +on: + push: + branches: [master] + tags: + - 'v*.*.*' + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: pages + cancel-in-progress: false + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install mdBook + uses: cargo-bins/cargo-binstall@v1 + with: + packages: mdbook + + - name: Build book + run: mdbook build book/ + + - name: Copy init script + run: cp website/init book/book/init + + - name: Setup Pages + uses: actions/configure-pages@v5 + + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 + with: + path: book/book/ + + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + needs: build + if: startsWith(github.ref, 'refs/tags/v') + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/README.md b/README.md index 3947c274a..1f6c034a4 100644 --- a/README.md +++ b/README.md @@ -1,717 +1,84 @@ -# bender +# Bender -Bender is a dependency management tool for hardware design projects. It provides a way to define dependencies among IPs, execute unit tests, and verify that the source files are valid input for various simulation and synthesis tools. +**A dependency management tool for hardware design projects.** + +Bender manages HDL package dependencies across Git repositories and local paths, resolves compatible versions, records the exact result in `Bender.lock`, and collects ordered source sets for downstream tools. It is designed to fit existing hardware flows rather than impose a registry, directory layout, or simulator choice. ![Build Status](https://github.com/pulp-platform/bender/actions/workflows/ci.yml/badge.svg) [![Crates.io](https://img.shields.io/crates/v/bender.svg)](https://crates.io/crates/bender) -[![dependency status](https://deps.rs/repo/github/pulp-platform/bender/status.svg)](https://deps.rs/repo/github/pulp-platform/bender) -![Crates.io](https://img.shields.io/crates/l/bender) - - -## Table of Contents - -- [Principles](#principles) -- [Installation](#installation) -- [Workflow](#workflow) -- [Package Structure](#package-structure) -- [Manifest Format (`Bender.yml`)](#manifest-format-benderyml) - - [Dependencies](#dependencies) - - [Sources](#sources) - - [Targets](#targets) -- [Configuration Format (`bender.yml`, `Bender.local`)](#configuration-format-benderyml-benderlocal) -- [Commands](#commands) - - -## Principles - -Bender is built around the following core principles: +[![License](https://img.shields.io/crates/l/bender)](#license) -- **Be as opt-in as possible.** We do not assume any specific EDA tool, workflow, or directory layout (besides a few key files). All features are designed to be as modular as possible, such that the user can integrate them into their respective flow. +## Why Bender -- **Allow for reproducible builds.** Bender maintains a precise *lock file* which tracks the exact git hash a dependency has been resolved to. This allows the source code of a package to be reliable reconstructed after the fact. +- **Reproducible dependency resolution:** Resolve IP dependencies from Git tags, revisions, or local paths and record the exact state in `Bender.lock`. +- **HDL-aware source collection:** Keep source ordering, include directories, defines, and target-specific file groups in one place. +- **Tool-agnostic workflow:** Generate inputs for QuestaSim, VCS, Vivado, Verilator, and other flows without forcing a particular setup. +- **Practical local development:** Work on dependencies locally and modify them conveniently as part of your development flow. -- **Collect source files.** The first feature tier of Bender is to collect the source files in a hardware IP. In doing this, it shall do the following: - - Maintain the required order across source files, e.g. for package declarations before their use. - - Be as language-agnostic as possible, supporting both SystemVerilog and VHDL. - - Allow source files to be organized into recursive groups. - - Track defines and include directories individually for each group. +## Quick Start -- **Manage dependencies.** The second feature tier of Bender is to maintain other packages an IP may depend on, and to provide a local checkout of the necessary source files. Specifically, it shall: - - Support transitive dependencies - - Not rely on a central package registry, unlike e.g. npm, cargo, or brew (necessary because parts of a project are usually under NDA) - - Enforce strict use of [semantic versioning](https://semver.org/) +Install Bender: -- **Generate tool scripts.** The third feature tier of Bender is the ability to generate source file listings and compilation scripts for various tools. - - -## Installation - -The recommended way to install Bender is via the shell installer, which detects your platform and places the `bender` binary in `~/.cargo/bin` (on your `PATH`): ```sh curl --proto '=https' --tlsv1.2 https://pulp-platform.github.io/bender/init -sSf | sh ``` -To install a specific version (e.g., `0.28.0`), append ` -s -- 0.28.0` to that command. Alternatively, you can manually download a precompiled binary from [our Releases on GitHub][releases]. -To build from source, [install Rust][rust-installation] and run: +Create a package: + ```sh -cargo install bender +mkdir my_ip +cd my_ip +bender init ``` -The Slang-backed `pickle` command is included by default. It requires a C++20 compiler and increases build time significantly. To build without it, use `--no-default-features`. - -Some Linux distributions also have Bender in their repositories: - -### [ArchLinux ![aur-shield](https://img.shields.io/aur/version/bender)][aur-bender] - -Please extend this list through a PR if you know additional distributions. - - -## Workflow - -The workflow of bender is based on a configuration and a lock file. The configuration file lists the sources, dependencies, and tests of the package at hand. The lock file is used by the tool to track which exact version of a package is being used. Adding this file to version control, e.g. for chips that will be taped out, makes it easy to reconstruct the exact IPs that were used during a simulation, synthesis, or tapeout. - -Upon executing any command, bender checks to see if dependencies have been added to the configuration file that are not in the lock file. It then tries to find a revision for each added dependency that is compatible with the other dependencies and add that to the lock file. In a second step, bender tries to ensure that the checked out revisions match the ones in the lock file. If not possible, appropriate errors are generated. - -The update command reevaluates all dependencies in the configuration file and tries to find for each a revision that satisfies all recursive constraints. If semantic versioning is used, this will update the dependencies to newer versions within the bounds of the version requirement provided in the configuration file. - - -## Package Structure - -Bender looks for the following three files in a package: - -- `Bender.yml`: This is the main **package manifest**, and the only required file for a directory to be recognized as a Bender package. It contains metadata, dependencies, and source file lists. - -- `Bender.lock`: The **lock file** is generated once all dependencies have been successfully resolved. It contains the exact revision of each dependency. This file *may* be put under version control to allow for reproducible builds. This is handy for example upon taping out a design. If the lock file is missing or a new dependency has been added, it is regenerated. - -- `Bender.local`: This optional file contains **local configuration overrides**. It should be ignored in version control, i.e. added to `.gitignore`. This file can be used to override dependencies with local variants. It is also used when the user asks for a local working copy of a dependency. - -[Relevant code](https://github.com/pulp-platform/bender/blob/master/src/cli.rs) - -## Manifest Format (`Bender.yml`) - -The package manifest describes the package, its metadata, its dependencies, and its source files. All paths in the manifest may be relative, in which case they are understood to be relative to the directory that contains the manifest. +Add a dependency to `Bender.yml`: ```yaml -# Package metadata. Required. -package: - # The name of the package. Required. - name: magic-chip - - # The list of package authors and contributors. Optional. - # By convention, authors should be listed in the form shown below. - authors: ["John Doe "] - - # A short description of the package. Optional. - description: "This is a magical chip" - -# Specify git remotes for dependencies. Optional. -remotes: - pulp: - url: "https://github.com/pulp-platform" - default: true # Only required if multiple remotes are specified. - - # Additional non-default remotes (HTTP or SSH). - openhw: "https://github.com/openhwgroup" - # Template remote URL where `{}` is a placeholder for dependency name. - # If no placeholder is found, "/{}.git" is used. - internal: "git@gitlab.company.com:internal-repo/{}/release" - -# Other packages this package depends on. Optional. dependencies: - # Path dependency. - axi: { path: "../axi" } - - # Git version dependency from default remote. - apb: "0.2" - - # Git version dependency from non-default remote. - fll: { version: "0.8", remote: "internal" } - - # Git version dependency with explicit git url. - ara: { git: "https://github.com/john_doe/ara.git", version: "2" } - - # Git revision dependency (always requires explicit git url). - spatz: {git: "https://github.com/pulp-platform/spatz.git", rev: "fixes" } - - # Git version dependency, only included if target "test" or "regression_test" is set. - common_verification: { version: "0.2", target: "any(test, regression_test)" } - - # Git revision dependency, passing a custom target. - # (equivalent to `-t common_cells:cc_custom_target`). - common_cells: { version: "1.39", pass_targets: ["cc_custom_target"] } - - # Git version dependency, passing conditional targets to a dependency - # (equivalent to `-t cva6:cv64a6_imafdcv_sv39` if target 64bit is set, - # `-t cva6:cv32a6_imac_sv32` if target 32bit is set) - ariane: - remote: openhw - version: 5.3.0 - pass_targets: - - {target: 64bit, pass: "cv64a6_imafdcv_sv39"} - - {target: 32bit, pass: "cv32a6_imac_sv32"} - -# Freeze any dependency updates. Optional. False if omitted. -# Useful for chip packages. Once the chip is in final tapeout mode, and -# dependency updates would require disastrous amounts of re-verification. -frozen: true - -# List of source files in this package. Optional. -sources: - # Individual source files are simple string entries: - - src/package.sv - - src/file1.vhd - - src/file2.vhd - - # Source files can be grouped: - - files: - - src/stuff/pkg.sv - - src/stuff/top.sv - - # Grouped source files may have additional include dirs, defines, and target: - - include_dirs: - - src/include - - src/stuff/include - # Include directories can be additionally filtered with targets - - target: core_target - dir: core_target/include - defines: - # Define without a value. - EXCLUDE_MAGIC: ~ - # Define with a value. - PREFIX_NAME: stuff - # Define dependent on a target - TEST_COMPONENTS: {target: "any(test, magic_test)", value: "all"} - target: all(asic, synthesis, freepdk45) - files: - - src/core/pkg.sv - - src/core/alu.sv - - src/core/top.sv - - # Source files can use glob patterns to include all matching files: - - src/more_stuff/**/*.sv - - # Source files can have custom fileendings - - sv: vendor/encrypted_sv_src.svp - - v: vendor/encrypted_v_src.vp - - vhd: vendor/encrypted_vhd_src.e - - # File list in another external file, supporting simple file names, `+define+` and `+incdir+` - - external_flists: - - other_file_list.f - files: [] - -# A list of include directories which should implicitly be added to source -# file groups of packages that have the current package as a dependency. -# Export include directories are also added to the current package. -# Optional. -export_include_dirs: - - include - - uvm/magic/include - # Include directories can be filtered with targets - - target: target - dir: target/include - -# Additional workspace configuration. Optional. -workspace: - # Create symlinks to dependencies. - # A list of paths at which bender will create a symlink to the checked-out - # version of the corresponding package. - package_links: - links/axi: axi - common: common_cells - - # A directory where the dependencies will be checked out. Optional. - # If specified, bender will check out the dependencies once and leave them - # for the user to modify and keep up to date. - # CAUTION: Bender will not touch these after the initial checkout. - # Useful for chip packages, if the intent is to commit all dependencies into - # the chip's version control. - checkout_dir: deps - -# Map of package-provided commands that can be called as `bender `. -# Optional. Only available in dependent packages. -plugins: - hello: scripts/hello.sh - -# List of vendorized files from external repositories not supporting bender. Optional. -vendor_package: - # package name - - name: lowrisc_opentitan - # target directory - target_dir: vendor/lowrisc_opentitan - # upstream dependency (i.e. git repository similar to dependencies, only supports commit hash) - upstream: { git: "https://github.com/lowRISC/opentitan.git", rev: "47a0f4798febd9e53dd131ef8c8c2b0255d8c139" } - # paths to include from upstream dependency. Per default, all paths are included. Optional. - include_from_upstream: - - "src/*" - # paths to exclude from upstream dependency. Paths that also match a pattern in include_from_upstream are excluded. Optional. - exclude_from_upstream: - - "ci/*" - # directory containing patch files. Optional. - patch_dir: "vendor/patches" - # custom file mapping from remote repository to local repository, with optional patch_dir containing patches. Optional. Note: mappings make upstreaming patches slightly more complicated. Avoid if not necessary. - mapping: - - {from: 'hw/ip/prim/rtl/prim_subreg.sv', to: 'src/prim_subreg.sv' } - - {from: 'hw/ip/prim/rtl/prim_subreg_arb.sv', to: 'src/prim_subreg_arb.sv' } - - {from: 'hw/ip/prim/rtl/prim_subreg_ext.sv', to: 'src/prim_subreg_ext.sv', patch_dir: 'lowrisc_opentitan' } - - {from: 'hw/ip/prim/rtl/prim_subreg_shadow.sv', to: 'src/prim_subreg_shadow.sv' } -``` - -[Relevant code](https://github.com/pulp-platform/bender/blob/master/src/config.rs) - - -### Dependencies - -Dependencies are specified in the `dependencies` section of the package manifest, or the `overrides` section in the configuration file. There are different kinds of dependencies, as described in the following. - -#### Path - - mydep: { path: "../path/to/mydep" } - -Path dependencies are not considered versioned. Either all versions of dependency `mydep` point to the same path, or otherwise the resolution will fail. - -#### Git - - mydep: { git: "git@github.com:pulp-platform/common_verification.git", rev: "" } - mydep: { git: "git@github.com:pulp-platform/common_verification.git", version: "1.1" } - -Git dependencies are automatically checked out and cloned, and are considered for version resolution. The `rev` field can be a git "commit-ish", which essentially is a commit hash, a tag name, or a branch name, where the newest name that starts with the indicated revision is selected. The `version` field can be any of the [semver predicates](https://docs.rs/semver/#requirements), such as a simple version `X.Y.Z` (or `X.Y`), prefixing `=` to only allow that specific version, `~` to limit updates to patches, or defining custom ranges with `>=U.V.W, Note: Git tags without the `v` prefix will not be detected by bender. eg: use `v1.2.3`, and **NOT** `1.2.3` - -[Relevant dependency resolution code](https://github.com/pulp-platform/bender/blob/master/src/resolver.rs) - -#### Git LFS Support - -Bender detects if a repository requires Git LFS and if the `git-lfs` tool is installed on your system. - -- If the repository uses LFS (detected via `.gitattributes`) and `git-lfs` is installed, Bender will automatically configure LFS and pull the required files. -- If the repository appears to use LFS but `git-lfs` is **not** installed, Bender will print a warning (`W33`) but proceed with the checkout. In this case, you may end up with pointer files instead of the actual large files, which can cause build failures. -- If the repository does not use LFS, Bender skips LFS operations entirely to save time. - -#### Target handling - -Specified dependencies can be filtered, similar to the sources below. For consistency, this filtering does **NOT** apply during an update, i.e., all dependencies will be accounted for in the Bender.lock file. The target filtering only applies for sources and script outputs. This can be used e.g., to include specific IP only for testing. - -#### Passing targets - -For sources and script generation, targets can be passed from a package to its dependency directly in the `Bender.yml` file. This allows for enabling and disabling of specific features. Furthermore, these passed targets can be again filtered with a target specification applied to the specific target. This can be used e.g., to enable specific features of dependencies. - - -### Sources - -The source files listed in the `sources` section of the package manifest are a recursive structure. Each entry in the list can either be a single source file, or a group of source files: - -```yaml -# Format of the `sources` section in the manifest: -sources: - - - - - - ... - - # A source file is formatted as follows: - - src/top.sv - - # A source group is formatted as follows. - # Be careful about the `-`, which may appear on the same line as the first - # field of the source group. - - - # List of include directories. Optional. - include_dirs: - - - - - - ... - # List of defines. Optional. - defines: - # Defines without value: - : ~ - : ~ - # Defines with value: - : - : - ... - # Target specifier. Optional. - target: - # Optional setting to override other files in any source that have the same file basename. - override_files: true - # Recursive list of source files and groups: - files: - - - - - - ... -``` - -The `target` specification configures a source group to be included or excluded under certain circumstances. See below for details. The `include_dirs` field specifies the `+incdir+...` statements to be added to any compilation command for the group. The `defines` field specifies the `+define+...` statements to be added add to any compilation command for this group. - - -### Targets - -Targets are flags that can be used to filter source files and dependencies. They are used to differentiate the steps in the ASIC/FPGA design flow, the EDA tools, technology targets, and more. They can also be used to have different versions of an IP optimized for different chips or technologies. - -Targets specify a simple expression language, as follows: - -- `*` matches any target -- `name` matches the target "name" -- `all(T1, ..., TN)` matches if all of the targets T1 to TN match (boolean *AND*) -- `any(T1, ..., TN)` matches if any of the targets T1 to TN match (boolean *OR*) -- `not(T)` matches if target T does *not* match (boolean *NOT*) - -The following targets are automatically set by various bender subcommands: - -- `synthesis` for synthesis tool script generation -- `simulation` for simulation tool script generation - -Individual commands may also set tool-specific targets: - -- `vsim` -- `vcs` -- `verilator` -- `synopsys` -- `riviera` -- `genus` -- `vivado` - -Individual commands may also set vendor-specific targets: - -- `xilinx` -- `synopsys` - -Individual commands may also set technology-specific targets: - -- `asic` -- `fpga` - -Additionally, we suggest to use the following targets to identify source code and netlists at different stages in the design process: - -- `test` for testbench code -- `rtl` for synthesizable RTL code -- `gate` for gate-level netlists - -Do not use `:` in your custom targets, as this is used to separate targets to apply to individual packages. - -Do not start the target name with `-`, as this is used to remove target application. - -[Relevant code](https://github.com/pulp-platform/bender/blob/master/src/target.rs) - - -### Override Files -If the `override_files` setting is applied to a source, then any files in that source will override other files that share the same basename. The overridden file will be removed from the output and replaced with the overriding file. For example, if `override_files` is applied to a source that has the file `src/core/pkg.sv`, then any other files that are also `pkg.sv` but in a different path will be removed and replaced with `src/core/pkg.sv`. If a file in an override files source does not override any other file, it will not be present in the output. - - -#### Example: -```yaml -sources: - - files: - - src/core/pkg.sv - - src/core/alu.sv - - src/core/top.sv - - target: custom_pkg - override_files: true - files: - - src/custom/pkg.sv - - src/custom/adder.sv -``` -If Bender is run with the `custom_pkg` target, the output files will be: - -``` -src/custom/pkg.sv -src/core/alu.sv -src/core/top.sv -``` - -### Vendor - -Section to list files and directories copied and patched within this repository from external repositories not supporting bender. -To update, see below `vendor` command. - - -## Configuration Format (`bender.yml`, `Bender.local`) - -Bender looks for a configuration file in the following places: - -- `/etc/bender.yml` -- `$HOME/.config/bender.yml` - -It will also look recursively upwards from the current working directory for the following: - -- `.bender.yml` -- `Bender.local` - -The contents of these files are merged as they are encountered, such that a configuration in `foo/.bender.yml` will overwrite a configuration in `foo/bar/.bender.yml`. - -The configuration file generally looks as follows: - -```yaml -# Location of the cloned and checked-out dependencies. Optional. -# Default: ".bender" in the current package's root directory. -database: some/directory - -# The command to use to invoke `git`. Optional. -# Default: "git" -git: git-wrapper.sh - - -# Overrides for dependencies. Optional. -# Forces a dependencies to use specific versions or local paths. Useful for -# locally resolving dependency conflicts in a package's own Bender.local file. -# Format is the same as `dependencies` in a package manifest. -overrides: - common_cells: { path: "/var/magic/common_cells" } - apb_uart: { git: "git@github.com:pulp-platform/apb_uart.git"} - -# Auxiliary plugin dependencies. Optional. -# Additional dependencies that will be loaded for every package in order to -# provide the `plugins` listed in their manifests. -# Format is the same as `dependencies` in a package manifest. -# DEPRECATED: This will be removed at some point. -plugins: - additional-tools: { path: "/usr/local/additional-tools" } - -# Number of parallel git tasks. Optional. -# Default: 4 -# The number of parallel git operations executed by bender can be adjusted to -# manage performance and load on git servers. Can be overriden as a command -# line argument. -git_throttle: 2 - -# Enable git lfs. Optional. -# Default: true -# Some git dependencies may use git-lfs for additional source files. As -# fetching these files may not always be desired or requried, it can be -# disabled. For multiple conflicting settings will use true. -git_lfs: false + common_cells: { git: "https://github.com/pulp-platform/common_cells.git", version: "1.21.0" } ``` -[Relevant code](https://github.com/pulp-platform/bender/blob/master/src/config.rs) - - -## Commands - -`bender` is the entry point to the dependency management system. Bender always operates within a package; starting at the current working directory, search upwards the file hierarchy until a `Bender.yml` is found, which marks the package. - - -### `path` --- Get the path of a checked-out package - -The `bender path ` prints the path of the checked-out version of package `PKG`. - -Useful in scripts: - - #!/bin/bash - cat `bender path mydep`/src/hello.txt - - -### `packages` --- Display the dependency graph - -- `bender packages`: List the package dependencies. The list is sorted and grouped according to a topological sorting of the dependencies. That is, leaf dependencies are compiled first, then dependent ones. -- `bender packages -f`: Produces the same list, but flattened. -- `bender packages -g`: Produces a graph description of the dependencies of the form `TAB`. - - -### `sources` --- List source files -[Code](https://github.com/pulp-platform/bender/blob/master/src/cmd/sources.rs) - -Produces a *sources manifest*, a JSON description of all files needed to build the project. - -The manifest is recursive by default; meaning that dependencies and groups are nested. Use the `-f`/`--flatten` switch to produce a simple flat listing. - -To enable specific targets, use the `-t`/`--target` option. Adding a package and colon `:` before a target will apply the target only to that specific package. Prefixing a target with `-` will remove that specific target, even for predefined targets (e.g., `-t-` or `-t :-`). - -To get the sources for a subset of packages, exclude specific packages and their dependencies, or exclude all dependencies, the following flags exist: - -- `-p`/`--package`: Specify package to show sources for. -- `-e`/`--exclude`: Specify package to exclude from sources. -- `-n`/`--no-deps`: Exclude all dependencies, i.e. only top level or specified package(s). - -For multiple packages (or excludes), multiple `-p` (or `-e`) arguments can be added to the command. - - -### `config` --- Emit the current configuration - -The `bender config` command prints the currently active configuration as JSON to standard output. - - -### `script` --- Generate tool-specific scripts - -The `bender script ` command can generate scripts to feed the source code of a package and its dependencies into a vendor tool. These scripts are rendered using internally stored templates with the [tera](https://keats.github.io/tera/docs/) crate, but custom templates can also be used. - -Supported formats: - -- `flist`: A flat whitespace-separated file list. -- `flist-plus`: A flat file list amenable to be directly inlined into the invocation command of a tool, e.g. `verilate $(bender script flist)`. -- `vsim`: A Tcl compilation script for Mentor ModelSim/QuestaSim. -- `vcs`: A Tcl compilation script for VCS. -- `verilator`: Command line arguments for Verilator. -- `synopsys`: A Tcl compilation script for Synopsys DC and DE. -- `formality`: A Tcl compilation script for Formality (as reference design). -- `riviera`: A Tcl compilation script for Aldec Riviera-PRO. -- `genus`: A Tcl compilation script for Cadence Genus. -- `vivado`: A Tcl file addition script for Xilinx Vivado. -- `vivado-sim`: Same as `vivado`, but specifically for simulation targets. -- `precision`: A Tcl compilation script for Mentor Precision. -- `template`: A custom [tera](https://keats.github.io/tera/docs/) template, provided using the `--template` flag. -- `template_json`: The json struct used to render the [tera](https://keats.github.io/tera/docs/) template. - -Furthermore, similar flags to the `sources` command exist. - -### `pickle` --- Parse and rewrite SystemVerilog sources with Slang - -The `bender pickle` command parses SystemVerilog sources with Slang and prints the resulting source again. It supports optional renaming and trimming of unreachable files for specified top modules. - -This command is only available when Bender is built with Slang support (for example via `cargo install bender --all-features`). - -Useful options: -- `--top `: Trim output to files reachable from one or more top modules. -- `--prefix ` / `--suffix `: Add a prefix and/or suffix to renamed symbols. -- `--exclude-rename `: Exclude specific symbols from renaming. -- `--ast-json`: Emit AST JSON instead of source code. -- `--expand-macros`, `--strip-comments`, `--squash-newlines`: Control output formatting. -- `-I `, `-D `: Add extra include directories and preprocessor defines. - -Furthermore, similar flags to the `sources` and `script` command exist: - -- `-t`/`--target`: Enable specific targets. -- `-p`/`--package`: Specify package to show sources for. -- `-e`/`--exclude`: Specify package to exclude from sources. -- `-n`/`--no-deps`: Exclude all dependencies, i.e. only top level or specified package(s). - -Examples: +Resolve dependencies and generate a compile script: ```sh -# Keep only files reachable from top module `top`. -bender pickle --top my_top - -# Rename symbols, but keep selected names unchanged. -bender pickle --top my_top --prefix p_ --suffix _s --exclude-rename my_top +bender update +bender script vsim > compile.tcl +vsim -do compile.tcl ``` +## How It Works -### `update` --- Re-resolve dependencies - -Whenever you update the list of dependencies, you likely have to run `bender update` to re-resolve the dependency versions, and recreate the `Bender.lock` file. - -Calling update with the `--fetch/-f` flag will force all git dependencies to be re-fetched from their corresponding urls. - -> Note: Actually this should be done automatically if you add a new dependency. But due to the lack of coding time, this has to be done manually as of now. - - -### `clone` --- Clone dependency to make modifications - -The `bender clone ` command checks out the package `PKG` into a directory (default `working_dir`, can be overridden with `-p / --path `). -To ensure the package is correctly linked in bender, the `Bender.local` file is modified to include a `path` dependency override, linking to the corresponding package. - -This can be used for development of dependent packages within the parent repository, allowing to test uncommitted and committed changes, without the worry that bender would update the dependency. - -To clean up once the changes are added, ensure the correct version is referenced by the calling packages and remove the path dependency in `Bender.local`, or have a look at `bender snapshot`. - -> Note: The location of the override may be updated in the future to prevent modifying the human-editable `Bender.local` file by adding a persistent section to `Bender.lock`. - -> Note: The newly created directory will be a git repo with a remote origin pointing to the `git` tag of the resolved dependency (usually evaluated from the manifest (`Bender.yml`)). You may need to adjust the git remote URL to properly work with your remote repository. - -### `snapshot` --- Relinks current checkout of cloned dependencies - -After working on a dependency cloned with `bender clone `, modifications are generally committed to the parent git repository. Once committed, this new hash can be quickly used by bender by calling `bender snapshot`. - -With `bender snapshot`, all dependencies previously cloned to a working directory are linked to the git repositories and commit hashes currently checked out. The `Bender.local` is modified correspondingly to ensure reproducibility. Once satisfied with the changes, it is encouraged to properly tag the dependency with a version, remove the override in the `Bender.local`, and update the required version in the `Bender.yml`. - -### `parents` --- Lists packages calling the specified package +Bender packages are built around three files: -The `bender parents ` command lists all packages calling the `PKG` package. - -### `checkout` --- Checkout all dependencies referenced in the Lock file - -This command will ensure all dependencies are downloaded from remote repositories. This is usually automatically executed by other commands, such as `sources` and `script`. - -### `fusesoc` --- Create FuseSoC `.core` files - -This command will generate FuseSoC `.core` files from the bender representation for open-source compatibility to the FuseSoC tool. It is intended to provide a basic manifest file in a compatible format, such that any project wanting to include a bender package can do so without much overhead. - -If the `--single` argument is provided, only to top-level `Bender.yml` file will be parsed and a `.core` file generated. - -If the `--single` argument is *not* provided, bender will walk through all the dependencies and generate a FuseSoC `.core` file where none is present. If a `.core` file is already present in the same directory as the `Bender.yml` for the corresponding dependency, this will be used to link dependencies (if multiple are available, the user will be prompted to select one). Previously generated `.core` files will be overwritten, based on the included `Created by bender from the available manifest file.` comment in the `.core` file. - -The `--license` argument will allow you to add multiple comment lines at the top of the generated `.core` files, e.g. a License header string. - -The `--fuse-vendor` argument will assign a vendor string to all generated `.core` dependencies for the VLNV name. - -The `--fuse-version` argument will assign a version to the top package being handled for the VLNV name. - -### `vendor` --- Copy files from dependencies that do not support bender - -Collection of commands to manage monorepos. Requires a subcommand. - -Please make sure you manage the includes and sources required for these files separately, as this command only fetches the files and patches them. -This is in part based on [lowRISC's `vendor.py` script](https://github.com/lowRISC/opentitan/blob/master/util/vendor.py). - -#### `vendor init` --- (Re-)initialize the vendorized dependencies - -This command will (re-)initialize the dependencies listed in the `vendor_package` section of the `Bender.yml` file, fetching the files from the remote repositories, applying the necessary patch files, and writing them to the respective `target_dir`. - -If the `-n/--no-patch` argument is passed, the dependency is initialized without applying any patches. - -#### `vendor diff` --- Print a diff of local, unpatched changes - -This command will print a diff to the remote repository with the patches in `patch_dir` applied. - -#### `vendor patch` --- Generate a patch file from local changes - -If there are local, *staged* changes in a vendored dependency, this command prompts for a commit message and generates a patch for that dependency. The patch is written into `patch_dir`. - -If the `--plain` argument is passed, this command will *not* prompt for a commit message and generate a patch of *all* (staged and unstaged) local changes of the vendored dependency. - -#### Example workflow - -Let's assume we would like to vendor a dependency `my_ip` into a project `monorepo`. -A simple configuration in a `Bender.yml` could look as follows (see the `Bender.yml` description above for more information on this): - -```yaml -vendor_package: - - name: my_ip - target_dir: deps/my_ip - upstream: { git: "", rev: "" } - patch_dir: "deps/patches/my_ip" -``` +- `Bender.yml`: Declares package metadata, dependencies, sources, targets, and workflow configuration. +- `Bender.lock`: Records the exact resolved dependency revisions for reproducible builds. +- `Bender.local`: Stores local overrides, for example when developing a dependency in-place. -Executing `bender vendor init` will now clone this dependency from `upstream` and place it in `target_dir`. +In a typical workflow, you edit `Bender.yml`, run `bender update` to resolve dependencies, use `bender checkout` to download the revisions pinned in `Bender.lock`, and generate tool inputs with `bender script`. -Next, let's assume that we edit two files within the dependency, `deps/my_ip/a` and `deps/my_ip/b`. -We can print these changes with the command `bender vendor diff`. +## Documentation -Now, we would like to generate a patch with the changes in `deps/my_ip/a` (but not those in `deps/my_ip/b`). -We stage the desired changes using `git add deps/my_ip/a` (of course, you can also just stage parts of a file using `git add --patch`). -The command `bender vendor patch` will now ask for a commit message that will be associated with this patch. -Then, it will place a patch that contains our changes in `deps/my_ip/a` into `deps/patches/my_ip/0001-commit-message.patch` (the number will increment if a numbered patch is already present). +The full documentation lives at **[pulp-platform.github.io/bender/](https://pulp-platform.github.io/bender/)**. -We can easily create a corresponding commit in the monorepo. -`deps/my_ip/a` is still staged from the previous step. -We only have to `git add deps/patches/my_ip/0001-commit-message.patch` and `git commit` for an atomic commit in the monorepo that contains both our changes to `deps/my_ip/a` and the corresponding patch. +- [Installation](https://pulp-platform.github.io/bender/installation.html) +- [Getting Started](https://pulp-platform.github.io/bender/getting_started.html) +- [Concepts](https://pulp-platform.github.io/bender/concepts.html) +- [Workflows](https://pulp-platform.github.io/bender/workflows.html) +- [Commands](https://pulp-platform.github.io/bender/commands.html) -Upstreaming patches to the dependency is easy as well. -We clone the dependencies' repository, check out `` and create a new branch. -Now, `git am /path/to/monorepo/deps/patches/my_ip/0001-commit-message.patch` will create a commit out of this patch -- including all metadata such as commit message, author(s), and timestamp. -This branch can then be rebased and a pull request can be opened from it as usual. +## License -Note: when using mappings in your `vendor_package`, the patches will be relative to the mapped directory. -Hence, for upstreaming, you might need to use `git am --directory=` instead of plain `git am`. +Bender is licensed under either of: -### `completion` --- Generate shell completion script +- Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) +- MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) -The `bender completion ` command prints a completion script for the given shell. +at your option. -Installation and usage of these scripts is shell-dependent. Please refer to your shell's documentation -for information on how to install and use the generated script -([bash](https://www.gnu.org/software/bash/manual/html_node/Programmable-Completion.html), -[zsh](https://zsh.sourceforge.io/Doc/Release/Completion-System.html), -[fish](https://fishshell.com/docs/current/completions.html)). +--- -Supported shells: -- `bash` -- `elvish` -- `fish` -- `powershell` -- `zsh` + +PULP Platform + -[aur-bender]: https://aur.archlinux.org/packages/bender -[releases]: https://github.com/pulp-platform/bender/releases -[rust-installation]: https://doc.rust-lang.org/book/ch01-01-installation.html +Bender is maintained by the [PULP Platform](https://pulp-platform.org/) at ETH Zurich and the University of Bologna. diff --git a/book/.gitignore b/book/.gitignore new file mode 100644 index 000000000..e9c072897 --- /dev/null +++ b/book/.gitignore @@ -0,0 +1 @@ +book \ No newline at end of file diff --git a/book/book.toml b/book/book.toml new file mode 100644 index 000000000..b280d272c --- /dev/null +++ b/book/book.toml @@ -0,0 +1,4 @@ +[book] +title = "bender" +authors = ["Michael Rogenmoser", "Tim Fischer", "Fabian Schuiki", "Andreas Kurth"] +language = "en" diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md new file mode 100644 index 000000000..990f9a577 --- /dev/null +++ b/book/src/SUMMARY.md @@ -0,0 +1,24 @@ +# Summary + +- [Overview](./index.md) +- [Installation](./installation.md) +- [Getting Started](./getting_started.md) +- [Concepts](./concepts.md) + - [Principles](./principles.md) + - [Manifest](./manifest.md) + - [Lockfile](./lockfile.md) + - [Local](./local.md) + - [Manifest vs. Lock vs. Local](./bender_files.md) + - [Dependencies](./dependencies.md) + - [Sources](./sources.md) + - [Targets](./targets.md) + - [Configuration](./configuration.md) +- [Workflows](./workflows.md) + - [Initialization](./workflow/init.md) + - [Dependencies](./workflow/dependencies.md) + - [Scripts](./workflow/scripts.md) + - [Package Development](./workflow/package_dev.md) + - [Vendor](./workflow/vendor.md) + - [Continuous Integration](./workflow/ci.md) + +- [Commands](./commands.md) diff --git a/book/src/bender_files.md b/book/src/bender_files.md new file mode 100644 index 000000000..047a347a1 --- /dev/null +++ b/book/src/bender_files.md @@ -0,0 +1,31 @@ +# Bender Files + +Bender relies on three core files to manage your hardware project. While they all use the YAML format, they serve very different roles in the development lifecycle. + +## Comparison Overview + +| Feature | `Bender.yml` | `Bender.lock` | `Bender.local` | +| :--- | :--- | :--- | :--- | +| **Role** | Manifest (Intent) | Lockfile (Reality) | Local Overrides | +| **Main Content** | Dependencies & Version Ranges | Exact Git Revisions | Local Paths & Tool Config | +| **Managed By** | User (Manual) | `bender update` (Auto) | User or `bender clone` | +| **Version Control** | **Commit** | **Commit** | **Ignore** (`.gitignore`) | +| **Shared?** | Yes, with everyone | Yes, for reproducibility | No, unique to your machine | + +## Summary of Roles + +### `Bender.yml` (The Manifest) +This is your **Intent**. It defines the requirements of your package. You use it to specify which other packages you need (e.g., "I need `axi` version `0.21.x`"). It is the only file required to define a Bender package. + +### `Bender.lock` (The Lockfile) +This is the **Reality**. It is a machine-generated file that captures the exact state of your dependency tree. It records the specific Git commit hash for every dependency. By committing this file, you ensure that every developer and CI machine works with the exact same source code. + +### `Bender.local` (Local Overrides) +This is your **Workspace**. It allows you to override the shared configuration for your local environment. Its most common use is to point a dependency to a local directory (via `bender clone`) so you can modify its code and see the effects immediately in your top-level project. + +## Interaction Flow + +1. **Define** your requirements in `Bender.yml`. +2. Run `bender update` to resolve those requirements into a fixed `Bender.lock`. +3. Run `bender checkout` to download the exact source code specified in the lockfile. +4. (Optional) Use `Bender.local` to temporarily swap a dependency for a local version during development. diff --git a/book/src/commands.md b/book/src/commands.md new file mode 100644 index 000000000..b4503c13f --- /dev/null +++ b/book/src/commands.md @@ -0,0 +1,193 @@ +# Commands + +`bender` is the entry point to the dependency management system. Bender always operates within a package; starting at the current working directory, search upwards the file hierarchy until a `Bender.yml` is found, which marks the package. + + +## `path` --- Get the path of a checked-out package + +The `bender path ` prints the path of the checked-out version of package `PKG`. + +Useful in scripts: + + #!/bin/bash + cat `bender path mydep`/src/hello.txt + + +## `packages` --- Display the dependency graph + +- `bender packages`: List the package dependencies. The list is sorted and grouped according to a topological sorting of the dependencies. That is, leaf dependencies are compiled first, then dependent ones. +- `bender packages -f`: Produces the same list, but flattened. +- `bender packages -g`: Produces a graph description of the dependencies of the form `TAB`. + + +## `sources` --- List source files +[Code](https://github.com/pulp-platform/bender/blob/master/src/cmd/sources.rs) + +Produces a *sources manifest*, a JSON description of all files needed to build the project. + +The manifest is recursive by default; meaning that dependencies and groups are nested. Use the `-f`/`--flatten` switch to produce a simple flat listing. + +To enable specific targets, use the `-t`/`--target` option. Adding a package and colon `:` before a target will apply the target only to that specific package. Prefixing a target with `-` will remove that specific target, even for predefined targets (e.g., `-t-` or `-t :-`). + +To get the sources for a subset of packages, exclude specific packages and their dependencies, or exclude all dependencies, the following flags exist: + +- `-p`/`--package`: Specify package to show sources for. +- `-e`/`--exclude`: Specify package to exclude from sources. +- `-n`/`--no-deps`: Exclude all dependencies, i.e. only top level or specified package(s). + +For multiple packages (or excludes), multiple `-p` (or `-e`) arguments can be added to the command. + + +## `config` --- Emit the current configuration + +The `bender config` command prints the currently active configuration as JSON to standard output. + + +## `script` --- Generate tool-specific scripts + +The `bender script ` command can generate scripts to feed the source code of a package and its dependencies into a vendor tool. These scripts are rendered using internally stored templates with the [tera](https://keats.github.io/tera/docs/) crate, but custom templates can also be used. + +Supported formats: + +- `flist`: A flat whitespace-separated file list. +- `flist-plus`: A flat file list amenable to be directly inlined into the invocation command of a tool, e.g. `verilate $(bender script flist)`. +- `vsim`: A Tcl compilation script for Mentor ModelSim/QuestaSim. +- `vcs`: A Tcl compilation script for VCS. +- `verilator`: Command line arguments for Verilator. +- `synopsys`: A Tcl compilation script for Synopsys DC and DE. +- `formality`: A Tcl compilation script for Formality (as reference design). +- `riviera`: A Tcl compilation script for Aldec Riviera-PRO. +- `genus`: A Tcl compilation script for Cadence Genus. +- `vivado`: A Tcl file addition script for Xilinx Vivado. +- `vivado-sim`: Same as `vivado`, but specifically for simulation targets. +- `precision`: A Tcl compilation script for Mentor Precision. +- `template`: A custom [tera](https://keats.github.io/tera/docs/) template, provided using the `--template` flag. +- `template_json`: The json struct used to render the [tera](https://keats.github.io/tera/docs/) template. + +Furthermore, similar flags to the `sources` command exist. + + +## `update` --- Re-resolve dependencies + +Whenever you update the list of dependencies, you likely have to run `bender update` to re-resolve the dependency versions, and recreate the `Bender.lock` file. + +Calling update with the `--fetch/-f` flag will force all git dependencies to be re-fetched from their corresponding urls. + +> Note: Actually this should be done automatically if you add a new dependency. But due to the lack of coding time, this has to be done manually as of now. + + +## `clone` --- Clone dependency to make modifications + +The `bender clone ` command checks out the package `PKG` into a directory (default `working_dir`, can be overridden with `-p / --path `). +To ensure the package is correctly linked in bender, the `Bender.local` file is modified to include a `path` dependency override, linking to the corresponding package. + +This can be used for development of dependent packages within the parent repository, allowing to test uncommitted and committed changes, without the worry that bender would update the dependency. + +To clean up once the changes are added, ensure the correct version is referenced by the calling packages and remove the path dependency in `Bender.local`, or have a look at `bender snapshot`. + +> Note: The location of the override may be updated in the future to prevent modifying the human-editable `Bender.local` file by adding a persistent section to `Bender.lock`. + +> Note: The newly created directory will be a git repo with a remote origin pointing to the `git` tag of the resolved dependency (usually evaluated from the manifest (`Bender.yml`)). You may need to adjust the git remote URL to properly work with your remote repository. + +## `snapshot` --- Relinks current checkout of cloned dependencies + +After working on a dependency cloned with `bender clone `, modifications are generally committed to the parent git repository. Once committed, this new hash can be quickly used by bender by calling `bender snapshot`. + +With `bender snapshot`, all dependencies previously cloned to a working directory are linked to the git repositories and commit hashes currently checked out. The `Bender.local` is modified correspondingly to ensure reproducibility. Once satisfied with the changes, it is encouraged to properly tag the dependency with a version, remove the override in the `Bender.local`, and update the required version in the `Bender.yml`. + +## `parents` --- Lists packages calling the specified package + +The `bender parents ` command lists all packages calling the `PKG` package. + +## `checkout` --- Checkout all dependencies referenced in the Lock file + +This command will ensure all dependencies are downloaded from remote repositories. This is usually automatically executed by other commands, such as `sources` and `script`. + +## `fusesoc` --- Create FuseSoC `.core` files + +This command will generate FuseSoC `.core` files from the bender representation for open-source compatibility to the FuseSoC tool. It is intended to provide a basic manifest file in a compatible format, such that any project wanting to include a bender package can do so without much overhead. + +If the `--single` argument is provided, only to top-level `Bender.yml` file will be parsed and a `.core` file generated. + +If the `--single` argument is *not* provided, bender will walk through all the dependencies and generate a FuseSoC `.core` file where none is present. If a `.core` file is already present in the same directory as the `Bender.yml` for the corresponding dependency, this will be used to link dependencies (if multiple are available, the user will be prompted to select one). Previously generated `.core` files will be overwritten, based on the included `Created by bender from the available manifest file.` comment in the `.core` file. + +The `--license` argument will allow you to add multiple comment lines at the top of the generated `.core` files, e.g. a License header string. + +The `--fuse-vendor` argument will assign a vendor string to all generated `.core` dependencies for the VLNV name. + +The `--fuse-version` argument will assign a version to the top package being handled for the VLNV name. + +## `vendor` --- Copy files from dependencies that do not support bender + +Collection of commands to manage monorepos. Requires a subcommand. + +Please make sure you manage the includes and sources required for these files separately, as this command only fetches the files and patches them. +This is in part based on [lowRISC's `vendor.py` script](https://github.com/lowRISC/opentitan/blob/master/util/vendor.py). + +### `vendor init` --- (Re-)initialize the vendorized dependencies + +This command will (re-)initialize the dependencies listed in the `vendor_package` section of the `Bender.yml` file, fetching the files from the remote repositories, applying the necessary patch files, and writing them to the respective `target_dir`. + +If the `-n/--no-patch` argument is passed, the dependency is initialized without applying any patches. + +### `vendor diff` --- Print a diff of local, unpatched changes + +This command will print a diff to the remote repository with the patches in `patch_dir` applied. + +### `vendor patch` --- Generate a patch file from local changes + +If there are local, *staged* changes in a vendored dependency, this command prompts for a commit message and generates a patch for that dependency. The patch is written into `patch_dir`. + +If the `--plain` argument is passed, this command will *not* prompt for a commit message and generate a patch of *all* (staged and unstaged) local changes of the vendored dependency. + +### Example workflow + +Let's assume we would like to vendor a dependency `my_ip` into a project `monorepo`. +A simple configuration in a `Bender.yml` could look as follows (see the `Bender.yml` description above for more information on this): + +```yaml +vendor_package: + - name: my_ip + target_dir: deps/my_ip + upstream: { git: "", rev: "" } + patch_dir: "deps/patches/my_ip" +``` + +Executing `bender vendor init` will now clone this dependency from `upstream` and place it in `target_dir`. + +Next, let's assume that we edit two files within the dependency, `deps/my_ip/a` and `deps/my_ip/b`. +We can print these changes with the command `bender vendor diff`. + +Now, we would like to generate a patch with the changes in `deps/my_ip/a` (but not those in `deps/my_ip/b`). +We stage the desired changes using `git add deps/my_ip/a` (of course, you can also just stage parts of a file using `git add --patch`). +The command `bender vendor patch` will now ask for a commit message that will be associated with this patch. +Then, it will place a patch that contains our changes in `deps/my_ip/a` into `deps/patches/my_ip/0001-commit-message.patch` (the number will increment if a numbered patch is already present). + +We can easily create a corresponding commit in the monorepo. +`deps/my_ip/a` is still staged from the previous step. +We only have to `git add deps/patches/my_ip/0001-commit-message.patch` and `git commit` for an atomic commit in the monorepo that contains both our changes to `deps/my_ip/a` and the corresponding patch. + +Upstreaming patches to the dependency is easy as well. +We clone the dependencies' repository, check out `` and create a new branch. +Now, `git am /path/to/monorepo/deps/patches/my_ip/0001-commit-message.patch` will create a commit out of this patch -- including all metadata such as commit message, author(s), and timestamp. +This branch can then be rebased and a pull request can be opened from it as usual. + +Note: when using mappings in your `vendor_package`, the patches will be relative to the mapped directory. +Hence, for upstreaming, you might need to use `git am --directory=` instead of plain `git am`. + +## `completion` --- Generate shell completion script + +The `bender completion ` command prints a completion script for the given shell. + +Installation and usage of these scripts is shell-dependent. Please refer to your shell's documentation +for information on how to install and use the generated script +([bash](https://www.gnu.org/software/bash/manual/html_node/Programmable-Completion.html), +[zsh](https://zsh.sourceforge.io/Doc/Release/Completion-System.html), +[fish](https://fishshell.com/docs/current/completions.html)). + +Supported shells: +- `bash` +- `elvish` +- `fish` +- `powershell` +- `zsh` diff --git a/book/src/concepts.md b/book/src/concepts.md new file mode 100644 index 000000000..15662f187 --- /dev/null +++ b/book/src/concepts.md @@ -0,0 +1,12 @@ +# Concepts + +This section explains the core ideas and files that make Bender work. Understanding these concepts will help you manage complex hardware projects more effectively. + +- **[Principles](./principles.md):** The high-level goals and design philosophy behind Bender. +- **[Manifest](./manifest.md):** How to define your package's metadata, dependencies, and sources. +- **[Lockfile](./lockfile.md):** How Bender ensures reproducible builds across different environments. +- **[Workspace](./local.md):** Overriding settings for your local development workspace. +- **[Comparing the Files](./bender_files.md):** A quick comparison of the three core files (`.yml`, `.lock`, `.local`). +- **[Dependencies](./dependencies.md):** How Bender handles hierarchical and transitive dependencies. +- **[Sources](./sources.md):** Managing HDL source files, include directories, and defines. +- **[Targets](./targets.md):** Using boolean expressions to conditionally include or exclude files and dependencies. diff --git a/book/src/configuration.md b/book/src/configuration.md new file mode 100644 index 000000000..765b49080 --- /dev/null +++ b/book/src/configuration.md @@ -0,0 +1,62 @@ +# Global and Local Configuration + +Bender uses a cascading configuration system that allows you to set global defaults while still providing the flexibility to override settings for specific projects or local environments. + +## Configuration Files + +Bender looks for configuration files in the following order, merging them as they are encountered: + +1. **System-wide:** `/etc/bender.yml` +2. **User-specific:** `$HOME/.config/bender.yml` (or the equivalent on your OS) +3. **Project-specific (checked in):** `.bender.yml` in your project root. +4. **Local Workspace (ignored):** `Bender.local` in your project root. + +The contents of these files are merged such that a configuration in a lower-level file (like `Bender.local`) will overwrite a configuration in a higher-level file (like `/etc/bender.yml`). + +## Configuration Fields + +The configuration files use the YAML format and support the following fields: + +### `database` +The directory where Bender stores cloned and checked-out dependencies. +- **Default:** `.bender` in the project root. +- **Example:** `database: /var/cache/bender_dependencies` + +### `git` +The command or path used to invoke Git. +- **Default:** `git` +- **Example:** `git: /usr/local/bin/git-wrapper.sh` + +### `git_throttle` +The maximum number of concurrent Git operations. +- **Default:** `4` +- **Example:** `git_throttle: 2` + +### `git_lfs` +Enable or disable Git Large File Storage (LFS) support. +- **Default:** `true` +- **Example:** `git_lfs: false` + +### `overrides` +Forces specific dependencies to use a particular version or local path. This is primarily used in `Bender.local` for development. +- **Example:** + ```yaml + overrides: + common_cells: { path: "../local_development/common_cells" } + ``` + +### `plugins` +Auxiliary plugin dependencies that are loaded for every package. These allow you to provide additional Bender subcommands across your entire environment. + +## Usage Example + +A typical `Bender.local` file used for local development might look like this: + +```yaml +# Speed up git operations locally +git_throttle: 8 + +# Work on a local copy of common_cells +overrides: + common_cells: { path: "../common_cells" } +``` diff --git a/book/src/dependencies.md b/book/src/dependencies.md new file mode 100644 index 000000000..91c91cb58 --- /dev/null +++ b/book/src/dependencies.md @@ -0,0 +1,88 @@ +# Dependencies + +Bender is designed to manage complex, hierarchical dependency trees for hardware projects. A dependency is any external package that provides its own `Bender.yml` manifest. + +## Dependency Types + +Dependencies are defined in the `dependencies` section of your `Bender.yml`. + +### Git Dependencies +Git is the primary way to distribute Bender packages. You can specify them in two ways: + +> **Important:** The shorthand notation (e.g., `common_cells: "1.21.0"`) is only available if you have defined at least one [remote](#remotes) in your manifest. If no remote is specified, you must use the full `git` URL (see [revision](#revision-based)) + +#### Version-based (Recommended) +Bender uses [Semantic Versioning (SemVer)](https://semver.org/) to find the best compatible version. You can use standard [SemVer operators](https://docs.rs/semver/latest/semver/enum.Op.html) to specify version ranges: + +```yaml +dependencies: + common_cells: "1.21.0" + axi: { version: ">=0.23.0, <0.26.0" } +``` +> **Note:** Bender only recognizes Git tags that follow the `vX.Y.Z` format (e.g., `v1.2.1`). + +#### Revision-based +Use this for specific commits, branches, or tags that don't follow SemVer. +```yaml +dependencies: + pulp_soc: { git: "https://github.com/pulp-platform/pulp_soc.git", rev: "develop" } +``` + +### Path Dependencies +Path dependencies point to a local directory. They are never versioned; Bender simply uses the code found at that location. +```yaml +dependencies: + my_local_ip: { path: "../local_ips/my_ip" } +``` + +## Remotes + +To avoid repeating full Git URLs, you can define `remotes` in your manifest. + +### Single Remote +If you only define a single remote, it is automatically treated as the default: + +```yaml +remotes: + pulp: "https://github.com/pulp-platform" + +dependencies: + common_cells: "1.21.0" # Automatically searched in the 'pulp' remote +``` + +### Multiple Remotes +When using multiple remotes, you must explicitly mark one as the `default` if you want to use shortened dependency syntax: + +```yaml +remotes: + pulp: + url: "https://github.com/pulp-platform" + default: true + openhw: "https://github.com/openhwgroup" + +dependencies: + common_cells: "1.21.0" # Uses the default 'pulp' remote + cva6: { version: "4.0.0", remote: openhw } # Explicitly uses 'openhw' +``` + +## Targets + +Dependencies can be conditionally included or configured using targets. For details on how to use target expressions or pass targets to dependencies, see the [Targets](./targets.md) documentation. + +## Git LFS Support + +Bender automatically detects if a dependency uses **Git Large File Storage (LFS)**. If `git-lfs` is installed on your system, Bender will automatically pull the required large files during the checkout process. + +## Submodules + +TODO + +## Version Resolution and the Lockfile + +When you run `bender update`, Bender performs the following: +1. **Resolution:** It scans the entire dependency tree and finds a set of versions that satisfy all constraints. +2. **Locking:** The exact versions and Git commit hashes are written to [`Bender.lock`](./lockfile.md). + +For details on updating dependencies, see [Adding and Updating Dependencies](./workflow/dependencies.md) + +**Reproducibility:** Once a `Bender.lock` exists, running `bender checkout` will always download the exact same code, even if newer compatible versions have been released. Always commit your `Bender.lock` to version control. diff --git a/book/src/getting_started.md b/book/src/getting_started.md new file mode 100644 index 000000000..3d2a602e6 --- /dev/null +++ b/book/src/getting_started.md @@ -0,0 +1,65 @@ +# Getting Started + +This guide will walk you through creating your first Bender project, adding a dependency, and generating a simulation script. + +## 1. Create a New Project + +Start by creating a directory for your project and initializing it with Bender: + +```sh +mkdir my_new_ip +cd my_new_ip +bender init +``` + +This creates a default `Bender.yml` file. Bender will automatically try to fill in your name and email from your Git configuration. + +## 2. Add a Dependency + +Open `Bender.yml` in your editor and add the `common_cells` library to the `dependencies` section: + +```yaml +package: + name: my_new_ip + authors: ["John Doe "] + +dependencies: + common_cells: { git: "https://github.com/pulp-platform/common_cells.git", version: "1.21.0" } + +sources: + - src/my_new_ip.sv +``` + +## 3. Resolve and Checkout + +Now, tell Bender to resolve the version of `common_cells` and download it: + +```sh +bender update +``` + +This command creates a `Bender.lock` file (the "exact" version chosen) and downloads the source code into a hidden `.bender` directory. + +## 4. Add Source Code + +Create a simple SystemVerilog file in `src/my_new_ip.sv`: + +```systemverilog +module my_new_ip ( + input logic clk_i, + output logic dummy_o +); + // We can use a module from common_cells here! + // ... +endmodule +``` + +## 5. Generate a Simulation Script + +Finally, generate a compilation script for your EDA tool (e.g., QuestaSim/ModelSim): + +```sh +bender script vsim > compile.tcl +``` + +You can now run `vsim -do compile.tcl` in your terminal to compile the entire project. diff --git a/book/src/index.md b/book/src/index.md new file mode 100644 index 000000000..f5c57e086 --- /dev/null +++ b/book/src/index.md @@ -0,0 +1,27 @@ +# Bender Documentation + +Welcome to the official documentation for **Bender**, a dependency management tool specifically designed for hardware design projects. + +Bender helps you manage complex hardware IP hierarchies, ensuring that every member of your team and every CI runner is working with the exact same source code. + +## Key Features + +- **Hierarchical Dependency Management:** Resolve and manage transitive dependencies across multiple Git repositories or local paths. +- **Reproducible Builds:** A precise lockfile mechanism (`Bender.lock`) ensures bit-identical design states across environments. +- **HDL-Aware Source Collection:** Automatically manages file ordering, include directories, and preprocessor defines for SystemVerilog and VHDL. +- **Target-Based Filtering:** Use powerful boolean expressions to include or exclude files based on your flow (simulation, synthesis, etc.). +- **Local Development Workflow:** Easily modify dependencies in-place using the `clone` and `snapshot` flow without breaking official manifests. +- **Tool Script Generation:** Generate compilation and simulation scripts for major EDA tools like QuestaSim, VCS, Vivado, Verilator, and more. + +## Getting Started + +If you are new to Bender, we recommend following these steps: + +1. **[Installation](./installation.md):** Get the `bender` binary onto your system. +2. **[Getting Started](./getting_started.md):** A quick tutorial to create your first Bender project. +3. **[Concepts](./concepts.md):** Dive deeper into how Bender works under the hood. +4. **[Workflows](./workflows.md):** Practical guides for daily development tasks. + +--- + +Bender is an open-source project maintained by the [PULP Platform](https://pulp-platform.org/) at ETH Zurich and the University of Bologna. diff --git a/book/src/installation.md b/book/src/installation.md new file mode 100644 index 000000000..e24a5f15c --- /dev/null +++ b/book/src/installation.md @@ -0,0 +1,64 @@ +# Installation + +Bender is a single standalone binary. You can either use our recommended shell installer, download a precompiled version, or build it from source. + +## Recommended: Shell Installer + +The fastest way to install Bender is using our shell script. It detects your operating system and architecture, downloads the latest release, and places it in your path (typically `~/.cargo/bin`). + +```sh +curl --proto '=https' --tlsv1.2 https://pulp-platform.github.io/bender/init -sSf | sh +``` + +### Installing a Specific Version +If you need a specific version of Bender (e.g., `0.31.0`), you can pass it as an argument: +```sh +curl --proto '=https' --tlsv1.2 https://pulp-platform.github.io/bender/init -sSf | sh -s -- 0.31.0 +``` + +## Alternative: Build from Source + +If you prefer building your own binary, you will need to [install Rust](https://rustup.rs/). + +### Using Cargo +You can install the latest official release directly from [crates.io](https://crates.io/crates/bender): + +```sh +cargo install bender +``` + +> **Note:** By default, Bender includes the `pickle` command which is backed by [Slang](https://github.com/MikePopoloski/slang). This requires a **C++20 compliant compiler** and increases build time significantly. To build without this feature, run: +> ```sh +> cargo install bender --no-default-features +> ``` + +### From Local Source +If you have cloned the repository, you can install the local version by running the following command from the project root: + +```sh +cargo install --path . +``` + +## Linux Distributions + +Some Linux distributions provide Bender through their package managers: + +- **ArchLinux:** [Bender on the AUR](https://aur.archlinux.org/packages/bender) + +## Verifying Installation + +After installation, verify that Bender is available in your terminal: + +```sh +bender --version +``` + +## Shell Completion + +Bender supports shell completion for Bash, Zsh, Fish, and PowerShell. To enable it, use the `completion` command and source the output in your shell's configuration file. + +For example, for **Zsh**: +```sh +bender completion zsh > ~/.bender_completion.zsh +echo "source ~/.bender_completion.zsh" >> ~/.zshrc +``` diff --git a/book/src/local.md b/book/src/local.md new file mode 100644 index 000000000..168c24575 --- /dev/null +++ b/book/src/local.md @@ -0,0 +1,44 @@ +# Local Configuration (`Bender.local`) + +`Bender.local` is an optional, user-specific configuration file used to override the project's default settings. Its primary purpose is to allow local development of dependencies without modifying the shared `Bender.yml` or `Bender.lock`. + +## Overriding Dependencies + +The most common use for `Bender.local` is the `overrides` section. This forces Bender to use a specific version or a local path for a dependency, regardless of what the manifest requires. + +```yaml +overrides: + # Force a local path for 'common_cells' + common_cells: { path: "../local_development/common_cells" } + + # Force a specific git URL/revision + axi: { git: "https://github.com/my_fork/axi.git", rev: "experimental_branch" } +``` + +When an override is present, Bender will prioritize it over any other version resolution. + +## Management + +`Bender.local` can be managed both manually and automatically. Several Bender commands manage it for you during the development process: + +- **[`bender clone`](./workflow/package_dev.md#cloning-dependencies):** Automates moving a dependency to a local working directory and adds a `path` override. +- **[`bender snapshot`](./workflow/package_dev.md#snapshotting):** Updates `Bender.local` with the current Git hashes of your local checkouts. + +For a detailed guide on using these commands for multi-package development, see the [Package Development Workflow](./workflow/package_dev.md). + +## Other Configurations + +`Bender.local` can also be used to configure tool-specific settings: + +```yaml +# Change the directory where dependencies are stored (default is .bender) +database: my_deps_cache + +# Use a custom git binary or wrapper +git: /usr/local/bin/git-wrapper.sh +``` + +## Best Practices + +- **Don't Commit It:** `Bender.local` should **rarely** be checked into version control. It contains paths and settings specific to your local machine. Always add it to your `.gitignore`. +- **Use for Development:** Think of it as your "scratchpad" for multi-package development. Once your changes to a dependency are stable and released (tagged), remember to remove the override from `Bender.local` and update your `Bender.yml` with the new version. diff --git a/book/src/lockfile.md b/book/src/lockfile.md new file mode 100644 index 000000000..5828308c5 --- /dev/null +++ b/book/src/lockfile.md @@ -0,0 +1,57 @@ +# Lockfile (`Bender.lock`) + +The lockfile, named `Bender.lock`, is a machine-generated file that records the exact version and Git revision (commit hash) of every dependency in your project's tree. + +## Why a Lockfile? + +While `Bender.yml` specifies your *intent* (e.g., "I need `common_cells` version 1.21.x"), the lockfile specifies the *reality* (e.g., "`common_cells` is version 1.21.5 at commit `a1b2c3d`"). + +The lockfile ensures: +- **Reproducible Builds:** Everyone on your team is working with the exact same code. +- **CI Stability:** Your CI pipeline won't suddenly fail because a dependency released a new (but incompatible) version. +- **Speed:** Bender doesn't need to re-resolve the entire dependency tree if the lockfile is already present. + +## How it Works + +The lockfile is managed by two primary commands: + +- **[`bender update`](./workflow/dependencies.md#updating-dependencies):** Scans manifests, resolves constraints, and **updates** the `Bender.lock`. +- **[`bender checkout`](./workflow/dependencies.md#checking-out-dependencies):** Reads the `Bender.lock` and ensures the local state matches the exact recorded revisions. + +## Structure of the Lockfile + +The lockfile is written in YAML. For each package, it stores: + +```yaml +packages: + common_cells: + revision: 290c010c26569ec18683510e1688536f98768000 + version: 1.21.0 + source: + git: "https://github.com/pulp-platform/common_cells.git" + dependencies: + - tech_cells_generic +``` + +- **revision:** The full 40-character Git commit hash. +- **version:** The SemVer version that was resolved. +- **source:** Where to download the package from. +- **dependencies:** A list of other packages that this specific package depends on, ensuring the entire tree is captured. + +## Best Practices + +- **Commit it:** Always check `Bender.lock` into your version control (Git). +- **Update with intention:** Only run `bender update` when you actually want to pull in newer versions of your dependencies. +- **Review changes:** When `Bender.lock` changes, review the diff to see exactly which packages were upgraded. + +## Frozen Manifests + +If you want to prevent accidental updates to your project's dependency tree, you can set `frozen: true` in your `Bender.yml`. + +```yaml +package: + name: my_chip + frozen: true # Prevents 'bender update' from running +``` + +When `frozen: true` is set, `bender update` will fail, ensuring that your `Bender.lock` remains unchanged until you explicitly unfreeze the manifest. This is mostly recommended for late-stage tapeouts. diff --git a/book/src/manifest.md b/book/src/manifest.md new file mode 100644 index 000000000..d32979410 --- /dev/null +++ b/book/src/manifest.md @@ -0,0 +1,112 @@ +# Manifest (`Bender.yml`) + +The package manifest describes the package, its metadata, its dependencies, and its source files. All paths in the manifest may be relative, in which case they are understood to be relative to the directory that contains the manifest. A manifest is required for each bender package. + +It is strongly recommended to start the Manifest file with a license header (open-source or proprietary) as a comment. This provides clarity on the project's usage. + +```yaml +# Copyright (c) 2026 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +``` + +The first section of the manifest should be the package description, outlining the package itself. The package name is required and must match with the name used to call this bender package as a dependency. The name is interpreted in a case-insensitive manner within bender, so additional care should be taken to avoid name conflicts. + +Additionally, authors and a description can be specified. Bender currently does not use these fields but supports their existence. + +```yaml +# Package metadata. Required. +package: + # The name of the package. Required. + name: magic-chip + + # The list of package authors and contributors. Optional. + authors: ["John Doe "] + + # A short description of the package. Optional. + description: "This is a magical chip" +``` + +The `remotes` section allows you to define shorthand names for Git repositories. This makes the `dependencies` section much cleaner by avoiding repeated long URLs. + +```yaml +# Specify git remotes for dependencies. Optional. +remotes: + pulp: "https://github.com/pulp-platform" + openhw: + url: "https://github.com/openhwgroup" + default: true # Used if no remote is specified in a dependency +``` + +The next section in the manifest is the dependencies. + Basic projects not requiring any modules from dependencies can omit this section. All packages this bender project depends on should be listed here for proper functionality, including the version requirements. More details on the specific format can be found [here](./dependencies.md). + +```yaml +# Other packages this package depends on. Optional. +dependencies: + common_cells: { git: "https://github.com/pulp-platform/common_cells.git", version: "1.39" } +``` + +The sources section lists the HDL source files belonging to this package. It is optional for packages that only provide headers or are otherwise used without their own source files. More details on the format can be found [here](./sources.md). + +```yaml +# List of source files in this package. Optional. +sources: + # Individual source files. + - src/pkg.sv + - src/top.sv +``` + +Include directories that should be passed to all packages depending on this one are listed under `export_include_dirs`. This is the standard mechanism for sharing header files. The include directories listed here also apply to all files in the current package. + +```yaml +# Include directories exported to dependent packages. Optional. +export_include_dirs: + - include +``` + +Setting `frozen` to `true` prevents Bender from updating dependencies beyond what is recorded in the lockfile. This is useful for chip packages in tapeout mode where dependency changes would require disastrous amounts of re-verification. + +```yaml +# Freeze dependency updates. Optional. Defaults to false. +frozen: true +``` + +The workspace section provides additional options for the local working environment. It is not relevant for library packages and is typically only used in the top-level chip package. Package links creates symlinks for the specified dependencies to known locations, while checkout_dir enforces the checkout of all dependencies to a specific location. + +```yaml +# Additional workspace configuration. Optional. +workspace: + # Create symlinks to checked-out dependencies. + package_links: + links/axi: axi + common: common_cells + + # Directory where dependencies will be checked out. Optional. + # Once set, Bender performs the initial checkout and then leaves the directory + # untouched. Useful for chip packages that commit all dependencies into their + # own version control. + checkout_dir: deps +``` + +Packages can expose shell scripts as bender subcommands using the `plugins` section. These commands are available to packages that depend on this one and can be invoked as `bender `. + +```yaml +# Package-provided commands callable as `bender `. Optional. +plugins: + hello: scripts/hello.sh +``` + +The `vendor_package` section lists files copied from external repositories that do not support Bender. Vendored files are managed with the `bender vendor` command. More details on the workflow can be found [here](./workflow/vendor.md). + +```yaml +# Vendorized files from external repositories not supporting Bender. Optional. +vendor_package: + - name: lowrisc_opentitan + target_dir: vendor/lowrisc_opentitan + # Only commit hashes are supported for the upstream revision. + upstream: { git: "https://github.com/lowRISC/opentitan.git", rev: "47a0f4798febd9e53dd131ef8c8c2b0255d8c139" } + # Custom file mappings from upstream paths to local paths. Optional. + mapping: + - { from: "hw/ip/prim/rtl/prim_subreg.sv", to: "src/prim_subreg.sv" } +``` diff --git a/book/src/principles.md b/book/src/principles.md new file mode 100644 index 000000000..91368fa35 --- /dev/null +++ b/book/src/principles.md @@ -0,0 +1,44 @@ +# Principles + +Bender was created to solve the challenges of managing large-scale hardware designs with complex, nested dependencies. It is built around four core principles that guide its development and usage. + +## 1. Modular and Opt-in +Bender is designed to be a "pre-build" tool. It does not replace your EDA tools (synthesis, simulation, formal); it orchestrates them. +- **Tool Agnostic:** Whether you use Vivado, Questa, VCS, or Verilator, Bender provides the necessary file lists and configurations. +- **Flexible Layout:** We do not enforce a strict directory structure. As long as a `Bender.yml` is present, Bender can manage it. + +## 2. Reproducibility as Ground Truth +In hardware design, knowing exactly what was taped out or simulated is critical. +- **Precise Locking:** The `Bender.lock` file tracks every dependency down to its specific Git commit hash. +- **Immutable States:** By committing the lockfile, you ensure that everyone on the team—and every CI runner—is using identical source code. + +## 3. Decentralized and Secure +Unlike many software package managers (like npm or cargo), Bender does not rely on a central, public registry. +- **Git-Centric:** Dependencies are resolved directly from Git repositories. +- **NDA Friendly:** This allows projects to use internal, private repositories or even local paths, ensuring sensitive IP remains protected and within your infrastructure. + +## 4. Local-First Development +Hardware development often requires modifying an IP and its dependencies simultaneously. +- **Zero-Friction Overrides:** The `Bender.local` mechanism allows you to temporarily swap a remote dependency for a local working copy without changing the project's official manifest. +- **Seamless Snapshots:** Captured states can be shared or moved to CI easily, bridging the gap between local development and official releases. + +--- + +## The Three Tiers of Bender + +Bender's functionality can be categorized into three distinct tiers, each building upon the other: + +### Tier 1: Source Collection +At its simplest level, Bender is a tool for collecting and organizing HDL source files. +- **Ordering:** Maintains the required order across source files (e.g., packages before modules). +- **Organization:** Allows files to be organized into recursive groups with specific targets, defines, and include directories. + +### Tier 2: Dependency Management +Bender resolves and manages transitive dependencies between different hardware IPs. +- **Version Resolution:** Enforces Semantic Versioning (SemVer) to ensure compatibility. +- **Lifecycle Management:** Automates the fetching, checking out, and updating of external packages. + +### Tier 3: Tool Script Generation +The final tier provides the ability to generate the actual scripts and file lists used by vendor tools. +- **Automation:** Eliminates the need to manually maintain tool-specific file lists (like `.f` files or TCL scripts). +- **Consistency:** Ensures that the exact same set of sources is used across all stages of the design flow. diff --git a/book/src/sources.md b/book/src/sources.md new file mode 100644 index 000000000..973705eda --- /dev/null +++ b/book/src/sources.md @@ -0,0 +1,103 @@ +# Sources + +The `sources` section in `Bender.yml` defines the source files, include directories, and preprocessor definitions that make up your package. + +## Basic File Listing + +The simplest way to include files is to list them as strings. All paths are relative to the location of the `Bender.yml` file. + +```yaml +sources: + - src/my_pkg.sv + - src/my_ip.sv + - src/my_vhdl.vhd +``` + +## Source Groups + +You can group files together to apply common settings like include directories, preprocessor defines, or [targets](./targets.md). + +```yaml +sources: + - include_dirs: + - include + - src/common/include + defines: + USE_FAST_ALU: ~ # Define without a value + BIT_WIDTH: 64 # Define with a value + target: synthesis + files: + - src/rtl/alu.sv + - src/rtl/top.sv +``` + +- **include_dirs**: Paths added to the `+incdir+` flag during compilation. +- **defines**: Preprocessor macros added via `+define+`. +- **target**: A [target expression](./targets.md) that determines if this entire group is included in the current flow. + +## Glob Patterns + +Bender supports glob patterns for automatically including multiple files without listing them individually: + +```yaml +sources: + - src/*.sv # All .sv files in the src directory + - src/submodules/**/*.sv # All .sv files in all subdirectories (recursive) +``` + +## Custom File Types + +While Bender automatically detects file types by standard extensions (`.sv`, `.v`, `.vhd`), you can explicitly specify the type for encrypted or non-standard files: + +```yaml +sources: + - sv: vendor/encrypted_src.svp + - v: vendor/legacy_code.vp + - vhd: vendor/encrypted_vhdl.e +``` + +## External File Lists (`external_flists`) + +If you have existing EDA tool file lists (often `.f` or `.flist`), you can include them directly. Bender will attempt to parse them for source files, include directories, and defines: + +```yaml +sources: + - external_flists: + - scripts/files.f + files: [] +``` + +## File Overrides + +The `override_files: true` flag allows a source group to replace files with the same basename from other parts of the dependency tree. This is a powerful mechanism for "patching" dependencies or swapping implementations at the top level. + +```yaml +sources: + - override_files: true + files: + - patches/axi_fifo.sv # Will replace any 'axi_fifo.sv' found in the dependency tree +``` + +## Exported Include Directories + +If your package provides headers that its *dependents* need (e.g., UVM macros or shared packages), use the `export_include_dirs` section at the top level of the manifest: + +```yaml +package: + name: my_utils + +export_include_dirs: + - include +``` + +Any package that depends on `my_utils` will automatically have the `include` directory added to its include search path. + +> **Best Practice:** To avoid naming collisions, place your headers in a sub-folder named after your package. +> +> **The Problem:** If two packages (`axi` and `apb`) both export a header named `typedefs.svh`, a file including `` `include "typedefs.svh" `` will be ambiguous, and the compiler will pick whichever directory it finds first in the include path. +> +> **The Solution:** Structure your files as `include/axi/typedefs.svh` and `include/apb/typedefs.svh`, then include them with the package prefix: +> ```systemverilog +> `include "axi/typedefs.svh" +> `include "apb/typedefs.svh" +> ``` diff --git a/book/src/targets.md b/book/src/targets.md new file mode 100644 index 000000000..b85d93ee6 --- /dev/null +++ b/book/src/targets.md @@ -0,0 +1,172 @@ +# Targets + +Targets are the primary mechanism in Bender for managing project configurations. They allow you to conditionally include source files, include directories, and dependencies based on the current context (e.g., simulation vs. synthesis, or FPGA vs. ASIC). + +## Target Expressions + +Bender uses a simple boolean expression language for targets: + +- `*`: Wildcard, matches all target sets. +- `name`: Matches if the target `name` is active (case-insensitive). +- `all(T1, T2, ...)`: Matches if **all** arguments match (boolean AND). +- `any(T1, T2, ...)`: Matches if **at least one** argument matches (boolean OR). +- `not(T)`: Matches if `T` does **not** match (boolean NOT). +- `(T)`: Parentheses for grouping. + +### Syntax Rules +Target names can contain alphanumeric characters, dots (`.`), underscores (`_`), and hyphens (`-`). + +**Restrictions:** +- They **cannot** contain colons (`:`), as colons are used for package-specific targets in the CLI. +- They **cannot** start with a hyphen (`-`), as leading hyphens are used to disable targets in the CLI. + +### Logical Examples +- `all(asic, synthesis)`: Matches when both 'asic' and 'synthesis' are set. +- `any(vsim, vcs, riviera)`: Matches if any of the listed simulation tools are active. +- `not(simulation)`: Matches only when 'simulation' is **not** set. +- `any(test, all(rtl, simulation))`: Matches for testbenches, or for RTL code during simulation. + +## Usage in Bender.yml + +### Source Groups +You can wrap a group of files in a `target` specification. This allows you to manage different implementations or verification components within the same package. + +#### Testbench Inclusion +Ensures that verification-only code is never included in the synthesis or production flow: + +```yaml +sources: + # RTL sources + - files: + - src/core.sv + - src/alu.sv + + # Testbench only + - target: test + files: + - tb/driver.sv + - tb/tb_top.sv +``` + +#### Simulation vs. Synthesis +Commonly used to swap between an actual hardware macro and a fast behavioral model for simulation: + +```yaml +sources: + # Behavioral model for faster simulation + - target: all(simulation, not(synthesis)) + files: + - src/behavioral/ip_model.sv +``` + +#### Technology Selection +Useful when choosing between different physical implementations, such as ASIC standard cells versus FPGA primitives: + +```yaml +sources: + - target: asic + files: + - target/asic/clock_gate.sv + - target: fpga + files: + - target/fpga/clock_gate.sv +``` + +#### Core Configuration +Targets can be used to select between different hardware architectures or feature sets: + +```yaml +sources: + # 32-bit architecture + - target: rv32 + files: + - src/core/alu_32.sv + - src/core/regfile_32.sv + # 64-bit architecture + - target: rv64 + files: + - src/core/alu_64.sv + - src/core/regfile_64.sv +``` + +### Dependencies +You can make a dependency conditional using target expressions. This is commonly used to include verification IP or platform-specific components only when needed: + +```yaml +dependencies: + # Included only during simulation + uvm: { version: "1.2.0", target: simulation } + # Included for either test or simulation + common_verification: { version: "0.2", target: any(test, simulation) } +``` + +## Built-in Targets + +Bender automatically activates certain targets based on the subcommand and output format. These "default targets" ensure that tool-specific workarounds or flow-specific files are included correctly. You can disable this behavior with the `--no-default-target` flag. + +### Script Format Targets + +The `bender script` command activates the following targets based on the chosen format: + +| Format | Default Targets | +| :--- | :--- | +| `flist`, `flist-plus` | `flist` | +| `vsim` | `vsim`, `simulation` | +| `vcs` | `vcs`, `simulation` | +| `verilator` | `verilator`, `synthesis` | +| `synopsys` | `synopsys`, `synthesis` | +| `formality` | `synopsys`, `synthesis`, `formality` | +| `riviera` | `riviera`, `simulation` | +| `genus` | `genus`, `synthesis` | +| `vivado` | `vivado`, `fpga`, `xilinx`, `synthesis` | +| `vivadosim` | `vivado`, `fpga`, `xilinx`, `simulation` | +| `precision` | `precision`, `fpga`, `synthesis` | + +### Special Targets + +- **RTL:** If you use the `--assume-rtl` flag, Bender will automatically assign the `rtl` target to any source group that does not have an explicit target specification. +- **ASIC:** While `asic` is a common convention, it is **not** set automatically by Bender. It should be manually activated via `-t asic` when needed. + +## Activating Targets via CLI + +Use the `-t` or `--target` flag with Bender commands: + +```bash +# Enable the 'test' target +bender script vsim -t test + +# Enable multiple targets +bender script vivado -t synthesis -t fpga +``` + +### Advanced CLI Syntax +- **Package-specific:** `-t my_pkg:my_target` activates `my_target` only for `my_pkg`. +- **Negative targets:** `-t -old_target` explicitly disables `old_target`. + +## Passing Targets Hierarchically + +Bender allows you to "configure" your dependencies by passing specific targets down to them using the `pass_targets` field. This is a powerful way to propagate global settings or select implementations in sub-modules. + +### Simple Passing +You can pass a target name as a string, which will then always be active for that specific dependency: + +```yaml +dependencies: + my_submodule: + version: "1.0.0" + pass_targets: ["enable_debug"] # 'enable_debug' is always active for my_submodule +``` + +### Conditional Passing +You can also pass targets conditionally, based on the targets active in the parent package: + +```yaml +dependencies: + ariane: + version: 5.3.0 + pass_targets: + - {target: rv64, pass: "cv64a6_imafdcv_sv39"} + - {target: rv32, pass: "cv32a6_imac_sv32"} +``` + +In this example, if the `rv64` target is active globally, Bender will ensure the `cv64a6_imafdcv_sv39` target is active specifically for the `ariane` dependency. diff --git a/book/src/workflow/ci.md b/book/src/workflow/ci.md new file mode 100644 index 000000000..76d85991c --- /dev/null +++ b/book/src/workflow/ci.md @@ -0,0 +1,83 @@ +# Continuous Integration + +Integrating Bender into your CI/CD pipeline ensures that your hardware project is consistently built, linted, and simulated using the exact same dependency versions as your local environment. + +## Reproducibility + +In a CI environment, you should never run `bender update`. Instead, your pipeline should rely on the `Bender.lock` file to fetch the exact revisions of your dependencies. + +1. **Commit** `Bender.lock`: Ensure the lockfile is checked into your repository. +2. **Use** `bender checkout`: This command reads the lockfile and reconstructs the dependency tree without changing any versions. + +## GitHub Actions + +If you are using GitHub Actions, the [`pulp-platform/pulp-actions`](https://github.com/pulp-platform/pulp-actions) repository provides a dedicated action to simplify the setup. + +### `bender-install` + +The `bender-install` action handles downloading the Bender binary and adding it to your `PATH`. + +```yaml +steps: + - uses: actions/checkout@v4 + + - name: Install Bender + uses: pulp-platform/pulp-actions/bender-install@v2 + with: + version: 0.31.0 # Optional: specify a version + + - name: Checkout dependencies + run: bender checkout +``` + +## GitLab CI + +For GitLab CI, you can use the standard [installation script](../installation.md) to fetch the Bender binary at the start of your job. + +### Example Workflow + +```yaml +variables: + BENDER_VERSION: "0.31.0" + +before_script: + # Install Bender using the init script + - curl --proto '=https' --tlsv1.2 https://pulp-platform.github.io/bender/init -sSf | sh -s -- $BENDER_VERSION + - export PATH=$PATH:$(pwd) + +sim_job: + stage: test + script: + - bender checkout + - bender script vsim > compile.tcl + - # Run your simulation tool here... +``` + +## Caching + +Since `Bender.lock` uniquely identifies the state of all dependencies, it is theoretically possible to cache the `.bender` directory to speed up pipelines. However, for most projects, the overhead of managing the cache (uploading/downloading) might outweigh the time saved by `bender checkout`, especially with fast network connections to Git remotes. + +Caching is only recommended for projects with exceptionally large dependency trees or slow network access. + +### Caching Examples + +#### GitHub Actions +```yaml + - name: Cache Bender dependencies + uses: actions/cache@v4 + with: + path: .bender + key: bender-${{ hashFiles('**/Bender.lock') }} +``` + +#### GitLab CI +```yaml +cache: + key: + files: + - Bender.lock + paths: + - .bender/ +``` + +> **Note on Cache Keys:** We use the hash of `Bender.lock` as the cache key. This ensures the cache is only reused when the dependencies haven't changed. diff --git a/book/src/workflow/dependencies.md b/book/src/workflow/dependencies.md new file mode 100644 index 000000000..a987de651 --- /dev/null +++ b/book/src/workflow/dependencies.md @@ -0,0 +1,71 @@ +# Adding and Updating Dependencies + +As working with dependencies is one of bender's main features, there are a few commands to ensure functionality and assist with understanding the dependency structure. + +## New dependencies +Once new dependencies are added to the manifest, bender needs to first be made aware of their existence. Otherwise, some commands will not work correctly and return an error. To update dependencies, run the following command: + +```sh +bender update +``` + +In case other dependencies already exist and you do not want to re-resolve these, you can add the `--new-only` flag to the update command. + +## Updating dependencies +Similar to when adding new dependencies, updating existing dependencies to more recent versions is also done with the `update` command. + +### Single Dependency Update +You don't always have to update your entire dependency tree. To update only a specific package, provide its name: + +```sh +bender update +``` + +### Recursive Updates +By default, updating a single dependency will not update its own child dependencies. If you want to update a package and all of its dependencies recursively, use the `-r/--recursive` flag: + +```sh +bender update --recursive +``` + +## Checking out Dependencies + +While `bender update` resolves versions and updates the lockfile, it does not necessarily download all the source code. To ensure all dependencies are locally available, use: + +```sh +bender checkout +``` + +Bender will download the exact revisions specified in `Bender.lock`. This command is safe to run multiple times; it will only download missing packages or update those that have changed in the lockfile. + +> **Note:** Many other commands (like `sources` or `script`) will automatically trigger a checkout if they detect missing dependencies. + +## Inspecting the Dependency Tree + +To see the full list of dependencies and how they relate to each other, use the `packages` command: + +```sh +bender packages +``` + +This prints a tree structure showing every package in your project, its resolved version, and its source. Use the `-f/--flat` flag if you just want a simple list of names and versions. + +## Finding Package Paths + +If you need to know where a specific dependency is stored (e.g., to point an external tool to it), use: + +```sh +bender path +``` + +This will output the absolute path to the package's root directory. + +## Checking Usage (`parents`) + +If you are wondering why a specific dependency was included or why a certain version was forced, you can use: + +```sh +bender parents +``` + +This will show you all packages in your tree that depend on `` and what version constraints they have placed on it. diff --git a/book/src/workflow/init.md b/book/src/workflow/init.md new file mode 100644 index 000000000..7e84d802c --- /dev/null +++ b/book/src/workflow/init.md @@ -0,0 +1,33 @@ +# Initialization + +To start a new Bender project, use the `init` command to set up a basic manifest: + +```sh +bender init +``` + +## How it Works + +Bender attempts to intelligently pre-fill your `Bender.yml` with the following information: + +- **Package Name:** Set to the name of the current working directory. +- **Authors:** Pulled automatically from your global Git configuration (`git config user.name` and `git config user.email`). + +## The Generated Manifest + +The command creates a `Bender.yml` file with a structure similar to this: + +```yaml +package: + name: my_new_ip + authors: + - "John Doe " + +dependencies: + +sources: + # Source files grouped in levels. + # Level 0 +``` + +Once the manifest is created, you can begin adding your [dependencies](../dependencies.md) and [source files](../sources.md) to the respective sections. diff --git a/book/src/workflow/package_dev.md b/book/src/workflow/package_dev.md new file mode 100644 index 000000000..11144aec6 --- /dev/null +++ b/book/src/workflow/package_dev.md @@ -0,0 +1,46 @@ +# Package Development + +Bender makes it easy to develop multiple packages simultaneously. If you find yourself needing to modify a dependency, you don't have to manually manage local paths and Git remotes. Instead, you can use the `clone` and `snapshot` workflow. + +## The Development Workflow + +### 1. Clone the Dependency +Use the `clone` command to move a dependency from Bender's internal cache into a local directory where you can modify it: + +```sh +bender clone +``` + +By default, the package is checked out into a `working_dir` folder (you can change this with `-p/--path`). Bender automatically: +1. Performs a `git clone` of the dependency into that folder. +2. Adds a `path` override to your **`Bender.local`** file. + +Now, any changes you make in that folder are immediately reflected in your top-level project when you run Bender commands. + +### 2. Modify and Commit +You can now work on the cloned package as if it were a normal Git repository. You can add files, fix bugs, and commit your changes within that directory. + +### 3. Snapshot the State +Once you have committed changes in your cloned dependency and want to record that specific state (for sharing with others or for CI), use the `snapshot` command: + +```sh +bender snapshot +``` + +Bender will: +1. Detect all dependencies that are currently overridden by a local path. +2. Check the current Git commit hash of those local repositories. +3. Update **`Bender.local`** to use a `git` override with that specific `rev` (commit hash). +4. Automatically update **`Bender.lock`** to include these exact revisions. + +**Why use Snapshot?** +The main benefit of a snapshot is portability. Because the lockfile is updated with the specific commit hashes, you can commit `Bender.lock` and share it with colleagues or run it in CI. The other environments will download the exact revisions you were working on from the Git remotes, without needing access to your local development paths. + +## Finalizing Changes + +Once your changes are stable and you are ready to "release" them: + +1. **Tag the Dependency:** Push your changes to the remote repository and create a new version tag (e.g., `v1.2.2`). +2. **Update Manifest:** Update the version requirement in your top-level `Bender.yml` to include the new version. +3. **Clean Up:** Remove the local overrides from your `Bender.local` file. +4. **Resolve:** Run `bender update` to re-resolve the dependency tree and update `Bender.lock` to point to the new official version. diff --git a/book/src/workflow/scripts.md b/book/src/workflow/scripts.md new file mode 100644 index 000000000..a0fde8026 --- /dev/null +++ b/book/src/workflow/scripts.md @@ -0,0 +1,86 @@ +# Generating Tool Scripts + +Bender's `script` command is the bridge between dependency management and your EDA tools. It generates TCL or shell scripts that include all necessary source files, include directories, and preprocessor defines in the correct order. + +## Basic Usage + +The `script` command requires a format (the target EDA tool or output style): + +```sh +bender script +``` + +For example, to generate a script for ModelSim/QuestaSim: + +```sh +bender script vsim > compile.tcl +``` + +## Supported Formats + +Bender supports a wide range of EDA tools and generic formats: + +| Format | Tool / Use Case | Output Type | +| :--- | :--- | :--- | +| `vsim` | ModelSim / QuestaSim | TCL | +| `vcs` | Synopsys VCS | Shell | +| `verilator` | Verilator | Shell | +| `vivado` | Xilinx Vivado (Synthesis) | TCL | +| `vivadosim` | Xilinx Vivado (Simulation) | TCL | +| `synopsys` | Synopsys Design Compiler | TCL | +| `genus` | Cadence Genus | TCL | +| `flist` / `flist-plus` | Generic file lists | Text | + +For a full list of formats and their specific options, see the [Command Reference](../commands.md#bender-script). + +## Targets and Filtering + +When you generate a script, Bender automatically activates certain [built-in targets](../targets.md#built-in-targets). For example, `bender script vsim` automatically enables the `simulation` and `vsim` targets. + +You can manually enable additional targets using the `-t/--target` flag: + +```sh +# Generate a simulation script with the 'test' and 'gate' targets enabled +bender script vsim -t test -t gate > compile.tcl +``` + +## Useful Flags + +The `script` command provides several flags to fine-tune the generated output: + +### Package Filtering +- **`-p/--package `**: Only include source files from the specified package (and its dependencies). +- **`-n/--no-deps`**: Exclude all dependencies. This generates a script containing only the files from the current package or the packages explicitly listed with `-p`. +- **`-e/--exclude `**: Exclude a specific package from the generated script. + +### RTL Assumption +- **`--assume-rtl`**: Automatically adds the `rtl` target to any source group that does not have an explicit target specification. This is a common shorthand for generating synthesis scripts without having to tag every RTL file. + +### Compilation Control +- **`--compilation-mode `**: + - `separate` (default): Compiles each source group as a separate unit. + - `common`: Attempts to compile all source groups together in a single compilation unit. +- **`--no-abort-on-error`**: Tells the EDA tool to continue analysis/compilation even if errors are encountered in individual files. + +### Metadata and Debugging +- **`--source-annotations`**: Includes comments in the generated script that indicate which Bender package and source group each file belongs to. This is very helpful for debugging ordering or missing-file issues. +- **`--no-default-target`**: Disables the automatic activation of built-in targets (like `simulation` for `vsim`). Use this if you want absolute control over which targets are active. + +## Advanced Options + +### Adding Defines +You can pass additional preprocessor macros to all files in the script using the `-D` flag: + +```sh +bender script vsim -D USE_DDR4 -D CLK_FREQ=100 +``` + +### Relative Paths +For generic file lists (`flist` or `flist-plus`), you can force Bender to use paths relative to the current directory using the `--relative-path` flag. This is useful for sharing file lists between different machines or environments. + +### Custom Templates +Bender uses the [Tera](https://keats.github.io/tera/) templating engine for its scripts. If the built-in formats don't meet your needs, you can provide your own template file: + +```sh +bender script template --template my_custom_format.tera > output.txt +``` diff --git a/book/src/workflow/vendor.md b/book/src/workflow/vendor.md new file mode 100644 index 000000000..a11513e05 --- /dev/null +++ b/book/src/workflow/vendor.md @@ -0,0 +1,91 @@ +# Vendorizing External Code + +Bender's `vendor` command allows you to manage external dependencies that aren't natively packaged for Bender. It works by copying a subset of files from an upstream Git repository into a local directory within your project, allowing you to track changes and maintain local patches. + +This flow is heavily inspired by the `vendor.py` script used in the [OpenTitan](https://opentitan.org/) project. + +## Configuration + +Vendorized packages are defined in the `vendor_package` section of your `Bender.yml`: + +```yaml +vendor_package: + - name: my_ip + target_dir: "deps/my_ip" + upstream: { git: "https://github.com/external/my_ip.git", rev: "abcd123" } + include_from_upstream: + - "src/*.sv" + - "include/*.svh" + exclude_from_upstream: + - "src/deprecated/*" + patch_dir: "deps/patches/my_ip" + mapping: + - {from: 'src/old_name.sv', to: 'src/new_name.sv' } +``` + +### Key Fields +- **`name`**: A unique identifier for the vendorized package. +- **`target_dir`**: Where the files should be copied to in your repository. +- **`upstream`**: The Git repository and specific revision (commit/tag/branch) to pull from. +- **`include_from_upstream`**: (Optional) Glob patterns of files to copy. +- **`exclude_from_upstream`**: (Optional) Glob patterns of files to ignore. +- **`patch_dir`**: (Optional) A directory containing `.patch` files to apply after copying. +- **`mapping`**: (Optional) A list of specific file renames or movements during the copy process. + +## The Vendor Workflow + +Using the configuration above, here is how you manage a vendorized IP: + +### 1. Initialize (`init`) +To download the upstream code and copy it into your `target_dir`: + +```sh +bender vendor init +``` + +Bender clones the upstream repository, filters the files based on your rules, and copies them to your project. If a `patch_dir` is specified, any existing patches are applied automatically. + +### 2. Make Local Changes and Diff +Assume you need to fix a bug in `deps/my_ip/src/top.sv`. You edit the file directly in your workspace. You can see how your local code differs from the upstream source (plus existing patches) by running: + +```sh +bender vendor diff +``` + +### 3. Create a Patch (`patch`) +To make your fix permanent and shareable, stage the change and generate a patch: + +```sh +# Stage the change in your main repository +git add deps/my_ip/src/top.sv + +# Generate the patch file +bender vendor patch +``` + +Bender will prompt for a commit message and create a numbered patch file in your `patch_dir` (e.g., `0001-fix-bug.patch`). + +### 4. Commit Everything +Now you can create an atomic commit in your repository that contains both the modified source and the new patch file: + +```sh +git add deps/patches/my_ip/0001-fix-bug.patch +git commit -m "Update my_ip with local bugfix" +``` + +The next time you (or a teammate) run `bender vendor init`, Bender will pull the fresh upstream code and automatically apply your patch. + +## Upstreaming Patches + +If you want to contribute your fix back to the upstream repository: +1. Clone the upstream repository separately. +2. Check out the same revision (`abcd123`). +3. Apply your patch: `git am /path/to/your_repo/deps/patches/my_ip/0001-fix-bug.patch`. +4. The fix is now a proper Git commit in the upstream repo, with all metadata (author, timestamp) preserved. + +## When to use Vendor? + +Use the vendor flow when: +- The external IP does not have its own `Bender.yml`. +- You need to include only a small subset of a massive repository. +- You must maintain local modifications (bug fixes or tool workarounds) that haven't been merged upstream yet. diff --git a/book/src/workflows.md b/book/src/workflows.md new file mode 100644 index 000000000..8ec5971e7 --- /dev/null +++ b/book/src/workflows.md @@ -0,0 +1,10 @@ +# Workflows + +This section provides practical guides on how to use Bender for common hardware development tasks throughout the project lifecycle. + +- **[Initialization](./workflow/init.md):** Setting up a new Bender package from scratch. +- **[Adding and Updating Dependencies](./workflow/dependencies.md):** Managing external packages and ensuring reproducible builds. +- **[Inspecting Source Files](./workflow/sources.md):** Exploring and filtering the source files in your project. +- **[Generating Tool Scripts](./workflow/scripts.md):** Creating compilation and simulation scripts for various EDA tools. +- **[Package Development](./workflow/package_dev.md):** Developing multiple packages simultaneously using the `clone` and `snapshot` flow. +- **[Vendorizing External Code](./workflow/vendor.md):** Managing external repositories that don't natively support Bender. diff --git a/tests/cli_regression/Bender.yml b/tests/cli_regression/Bender.yml index a4200b04d..5dd69d9bb 100644 --- a/tests/cli_regression/Bender.yml +++ b/tests/cli_regression/Bender.yml @@ -1,3 +1,7 @@ +# Copyright (c) 2026 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + package: name: regression_repo diff --git a/website/init b/website/init new file mode 100755 index 000000000..37c7ce048 --- /dev/null +++ b/website/init @@ -0,0 +1,465 @@ +#!/usr/bin/env bash + +# Copyright (c) 2019 ETH Zurich +# SPDX-License-Identifier: Apache-2.0 OR MIT +# Author: Andreas Kurth + +# This is just a little script that can be downloaded from the internet to +# install Bender. It does platform detection, downloads the specified or the +# newest release package, and extracts it. +# Code and concept derived from the rustup installer. + +set -u + +readonly GITHUB_API_BENDER_ROOT='https://api.github.com/repos/pulp-platform/bender' +readonly GITHUB_API_HEADER='Accept: application/vnd.github.v3+json' + +usage() { + cat 1>&2 </dev/null + mv "$_tmpdir/bender" . || err "Failed to move executable to target directory!" + # Remove temporary directory. + rm -rf "$_tmpdir" || echo "Warning: Failed to clean up temporary directory." + + echo "Successfully installed $(./bender -V) in '$(pwd)'." +} + +get_bitness() { + need_cmd head + # Architecture detection without dependencies beyond coreutils. + # ELF files start out "\x7fELF", and the following byte is + # 0x01 for 32-bit and + # 0x02 for 64-bit. + # The printf builtin on some shells like dash only supports octal + # escape sequences, so we use those. + local _current_exe_head + _current_exe_head=$(head -c 5 /proc/self/exe ) + if [ "$_current_exe_head" = "$(printf '\177ELF\001')" ]; then + echo 32 + elif [ "$_current_exe_head" = "$(printf '\177ELF\002')" ]; then + echo 64 + else + err "unknown platform bitness" + fi +} + +get_endianness() { + local cputype=$1 + local suffix_eb=$2 + local suffix_el=$3 + + # detect endianness without od/hexdump, like get_bitness() does. + need_cmd head + need_cmd tail + + local _current_exe_endianness + _current_exe_endianness="$(head -c 6 /proc/self/exe | tail -c 1)" + if [ "$_current_exe_endianness" = "$(printf '\001')" ]; then + echo "${cputype}${suffix_el}" + elif [ "$_current_exe_endianness" = "$(printf '\002')" ]; then + echo "${cputype}${suffix_eb}" + else + err "unknown platform endianness" + fi +} + +get_architecture() { + local _ostype _cputype _bitness _arch + _ostype="$(uname -s)" + _cputype="$(uname -m)" + + if [ "$_ostype" = Linux ]; then + if [ "$(uname -o)" = Android ]; then + _ostype=Android + fi + fi + + if [ "$_ostype" = Darwin ] && [ "$_cputype" = i386 ]; then + # Darwin `uname -m` lies + if sysctl hw.optional.x86_64 | grep -q ': 1'; then + _cputype=x86_64 + fi + fi + + case "$_ostype" in + + Android) + _ostype=linux-android + ;; + + Linux) + _ostype=linux-gnu + _bitness=$(get_bitness) + ;; + + FreeBSD) + _ostype=freebsd + ;; + + NetBSD) + _ostype=netbsd + ;; + + DragonFly) + _ostype=dragonfly + ;; + + Darwin) + _ostype=apple-darwin + ;; + + MINGW* | MSYS* | CYGWIN*) + _ostype=pc-windows-gnu + ;; + + *) + err "unrecognized OS type: $_ostype" + ;; + + esac + + case "$_cputype" in + + i386 | i486 | i686 | i786 | x86) + _cputype=i686 + ;; + + xscale | arm) + _cputype=arm + if [ "$_ostype" = "linux-android" ]; then + _ostype=linux-androideabi + fi + ;; + + armv6l) + _cputype=arm + if [ "$_ostype" = "linux-android" ]; then + _ostype=linux-androideabi + else + _ostype="${_ostype}eabihf" + fi + ;; + + armv7l | armv8l) + _cputype=armv7 + if [ "$_ostype" = "linux-android" ]; then + _ostype=linux-androideabi + else + _ostype="${_ostype}eabihf" + fi + ;; + + aarch64) + _cputype=aarch64 + ;; + + arm64) + _cputype=aarch64 + ;; + + x86_64 | x86-64 | x64 | amd64) + _cputype=x86_64 + ;; + + mips) + _cputype=$(get_endianness mips '' el) + ;; + + mips64) + if [ "$_bitness" -eq 64 ]; then + # only n64 ABI is supported for now + _ostype="${_ostype}abi64" + _cputype=$(get_endianness mips64 '' el) + fi + ;; + + ppc) + _cputype=powerpc + ;; + + ppc64) + _cputype=powerpc64 + ;; + + ppc64le) + _cputype=powerpc64le + ;; + + s390x) + _cputype=s390x + ;; + + *) + err "unknown CPU type: $_cputype" + + esac + + # Detect 64-bit linux with 32-bit userland + if [ "${_ostype}" = linux-gnu ] && [ "${_bitness}" -eq 32 ]; then + case $_cputype in + x86_64) + _cputype=i686 + ;; + mips64) + _cputype=$(get_endianness mips '' el) + ;; + powerpc64) + _cputype=powerpc + ;; + esac + fi + + # Detect armv7 but without the CPU features Rust needs in that build, + # and fall back to arm. + # See https://github.com/rust-lang/rustup.rs/issues/587. + if [ "$_ostype" = "linux-gnueabihf" ] && [ "$_cputype" = armv7 ]; then + if ensure grep '^Features' /proc/cpuinfo | grep -q -v neon; then + # At least one processor does not have NEON. + _cputype=arm + fi + fi + + _arch="${_cputype}-${_ostype}" + + RETVAL="$_arch" +} + +get_distribution() { + local _vendor="" + local _release="" + if check_cmd lsb_release; then + _vendor=$(lsb_release -i | cut -f 2 | tr '[:upper:]' '[:lower:]') + _release=$(lsb_release -r | cut -f 2) + else + _vendor=$(cat /etc/os-release | sed -n -e "s/^ID=\(.*\)$/\1/p" | sed "s/\"//g") + _release=$(cat /etc/os-release | sed -n -e "s/^VERSION_ID=\(.*\)$/\1/p" | sed "s/\"//g") + fi + if [ "$_vendor" = "centos" ] && [ "$(echo "$_release" | cut -d. -f3)" -lt 1708 ]; then + say "Warning: CentOS older than 7.4.1708 detected, falling back to the latter." + _release="7.4.1708" + fi + if [ "$_vendor" = "redhatenterprise" ]; then + _vendor="rhel" + fi + RETVAL="$_vendor$_release" +} + +get_download_url() { + local _suburl="$1" + local _platform="$2" + RETVAL="$(downloader "$GITHUB_API_BENDER_ROOT/releases/$_suburl" | \ + grep -Eo " \"browser_download_url\": \".*?bender-.*-$_platform\.tar\.gz\"" | \ + grep -Eo "http.*?bender-.*-$_platform\.tar\.gz")" +} + +get_asset_url() { + local _version="$1" + local _platform="$2" + if [ $_version = "latest" ]; then + local _url="latest" + else + local _url="tags/v$_version" + fi + get_download_url "$_url" "$_platform" + if [ -z "$RETVAL" ]; then + say "Warning: No release for platform '$_platform' version '$_version' found, using latest." + get_download_url "latest" "$_platform" + fi + if [ -z "$RETVAL" ]; then + case "$_platform" in + x86_64-linux-gnu*) + say "Warning: Latest release not available for platform '$_platform', falling back to 'x86_64-linux-gnu'." + _fallback="x86_64-linux-gnu" + ;; + arm64-linux-gnu*) + say "Warning: Latest release not available for platform '$_platform', falling back to 'arm64-linux-gnu'." + _fallback="arm64-linux-gnu" + ;; + *) + err "Error: Latest release not available for platform '$_platform'!" + ;; + esac + get_download_url "latest" "$_fallback" + fi + assert_nz "$RETVAL" assets +} + +say() { + printf 'bender-init: %s\n' "$1" +} + +err() { + say "$1" >&2 + exit 1 +} + +need_cmd() { + if ! check_cmd "$1"; then + err "need '$1' (command not found)" + fi +} + +check_cmd() { + command -v "$1" > /dev/null 2>&1 +} + +assert_nz() { + if [ -z "$1" ]; then err "assert_nz $2"; fi +} + +# Run a command that should never fail. If the command fails execution +# will immediately terminate with an error showing the failing +# command. +ensure() { + if ! "$@"; then err "command failed: $*"; fi +} + +# This is just for indicating that commands' results are being +# intentionally ignored. Usually, because it's being executed +# as part of error handling. +ignore() { + "$@" +} + +# This wraps curl or wget. Try curl first, if not installed, +# use wget instead. +downloader() { + local _dld + if check_cmd curl; then + _dld=curl + elif check_cmd wget; then + _dld=wget + else + _dld='curl or wget' # to be used in error message of need_cmd + fi + + if [ "$1" = --check ]; then + need_cmd "$_dld" + elif [ "$_dld" = curl ]; then + #local _curl="curl -H '$GITHUB_API_HEADER'" + local _curl="curl" + if ! check_help_for curl --proto --tlsv1.2; then + echo "Warning: Not forcing TLS v1.2, this is potentially less secure" + else + _curl="$_curl --proto =https --tlsv1.2" + fi + _curl="$_curl --silent --show-error --fail --location $1" + if [ "$#" -ge 2 ]; then + _curl="$_curl --output $2" + fi + $_curl 2>/dev/null + elif [ "$_dld" = wget ]; then + local _wget="wget -H '$GITHUB_API_HEADER'" + if ! check_help_for wget --https-only --secure-protocol; then + echo "Warning: Not forcing TLS v1.2, this is potentially less secure" + else + _wget="$_wget --https-only --secure-protocol=TLSv1_2" + fi + _wget="$_wget $1" + if [ "$#" -ge 2 ]; then + _wget="$_wget -O $2" + fi + $_wget 2>/dev/null + else + err "Unknown downloader" # should not reach here + fi + return $? +} + +check_help_for() { + local _cmd + local _arg + local _ok + _cmd="$1" + _ok="y" + shift + + if [ "$_cmd" = "curl" ]; then + _cmd="$_cmd --help all" + else + _cmd="$_cmd --help" + fi + + # If we're running on OS-X, older than 10.13, then we always + # fail to find these options to force fallback + if check_cmd sw_vers; then + if [ "$(sw_vers -productVersion | cut -d. -f2)" -lt 13 ]; then + # Older than 10.13 + echo "Warning: Detected OS X platform older than 10.13" + _ok="n" + fi + fi + + for _arg in "$@"; do + if ! $_cmd | grep -q -- "$_arg"; then + _ok="n" + fi + done + + test "$_ok" = "y" +} + +main "$@" || exit 1 diff --git a/website/pulp_logo_icon.svg b/website/pulp_logo_icon.svg new file mode 100644 index 000000000..1437e65df --- /dev/null +++ b/website/pulp_logo_icon.svg @@ -0,0 +1,194 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +