From e0424d8ac0952f3389363422e78cbe501c245c2d Mon Sep 17 00:00:00 2001 From: Arvind Iyengar Date: Mon, 17 May 2021 15:28:30 -0700 Subject: [PATCH] Regenerate template based on v0.2.0 release ```bash make template ``` --- .github/workflows/pull-request.yaml | 25 ++++ Makefile | 5 +- README.md | 108 ++++++++++++---- assets/README.md | 3 + charts/README.md | 3 + packages/README.md | 115 +++++++++++++++++ scripts/package-ci | 42 +++++++ scripts/regenerate-assets | 3 + scripts/regenerate-packages | 183 ++++++++++++++++++++++++++++ 9 files changed, 459 insertions(+), 28 deletions(-) create mode 100755 .github/workflows/pull-request.yaml create mode 100755 assets/README.md create mode 100755 charts/README.md create mode 100755 packages/README.md create mode 100755 scripts/package-ci create mode 100755 scripts/regenerate-packages diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml new file mode 100755 index 000000000..98e290773 --- /dev/null +++ b/.github/workflows/pull-request.yaml @@ -0,0 +1,25 @@ +name: CI-pullrequest + +on: + pull_request: + branches: + - dev-v2.5 + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Checkout into branch + run: git checkout -b staging-pr-workflow + + - name: Pull scripts + run: sudo make pull-scripts + + - name: Pull in all relevant branches + run: git fetch origin release-v2.5 + + - name: Validate + run: sudo make validate diff --git a/Makefile b/Makefile index 1ae552810..6a456f494 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,7 @@ pull-scripts: ./scripts/pull-scripts -stage-release: - ./scripts/stage-release - -TARGETS := prepare patch charts clean sync validate rebase docs +TARGETS := prepare patch charts clean validate template $(TARGETS): @ls ./bin/charts-build-scripts 1>/dev/null 2>/dev/null || ./scripts/pull-scripts diff --git a/README.md b/README.md index b1bf43f1b..2fdb7be45 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,14 @@ ## Staging Branch -This branch contains generated assets that have not been officially released yet. +This branch contains Packages and generated assets that have not been officially released yet. + +See the README.md under `packages/`, `assets/`, and `charts/` for more information. The following directory structure is expected: ```text +package/ + / + ... assets/ / -.tgz @@ -20,47 +25,102 @@ charts/ This repository branch contains a `configuration.yaml` file that is used to specify how it interacts with other repository branches. -#### Sync - -This branch syncs with the generated assets from the following branches: -- dev-v2.6-source at https://github.com/rancher/charts.git - -To release a new version of a chart, please open the relevant PRs to one of these branches. - -Merging should trigger a sync workflow on pushing to these branches. - #### Validate This branch validates against the generated assets of the following branches to make sure it isn't overriding already released charts. -- release-v2.6 at https://github.com/rancher/charts.git (only latest assets) +- release-v2.6 at https://github.com/rancher/charts.git Before submitting any PRs, a Github Workflow will check to see if your package doesn't break any already released packages in these repository branches. -### Cutting a Release +### Making Changes -In the Staging branch, cutting a release involves moving the contents of the `assets/` directory into `released/assets/`, deleting the contents of the `charts/` directory, and updating the `index.yaml` to point to the new location for those assets. +As a developer making changes to a particular package, you will usually follow the following steps: -This process is entirely automated via the `make release` command. +#### If this is the first time you are adding a package: -Once you successfully run the `make release` command, ensure the following is true: -- The `assets/` and `charts/` directories each only have a single file contained within them: `README.md` -- The `released/assets/` directory has a .tgz file for each releaseCandidateVersion of a Chart that was created during this release. -- The `index.yaml` and `released/assets/index.yaml` both are identical and the `index.yaml`'s diff shows only two types of changes: a timestamp update or a modification of an existing URL from `assets/*` to `released/assets/*`. +```shell +PACKAGE= +mkdir -p packages/${PACKAGE} +touch packages/${PACKAGE}/package.yaml +``` + +See `packages/README.md` to configure the `packages/${PACKAGE}/package.yaml` file based on the Package that you are planning to add. + +To make changes, see the steps listed below. + +#### If the package already exists + +If you are working with a single Package, set `export PACKAGE=` to inform the scripts that you only want to make changes to a particular package. + +This will prevent the scripts from running commands on every package in this repository. + +You'll also want to update the `packageVersion` located in `packages/${PACKAGE}/package.yaml`. + +See the section below for how to update this field. + +Once you have made those changes, the Workflow will be: +```shell +make prepare # Instantiates the chart in the workingDir specified in the package.yaml +# Make your changes here to the workingDir directly here +make patch # Saves changes to generated-changes/ +make clean # Cleans up your workingDir, leaving behind only the generated-changes/ +``` + +Once your directory is clean, you are ready to submit a PR. + +#### Versioning Packages + +If this `major.minor.patch` (e.g. `0.0.1`) version of the Chart has never been released, reset the `packageVersion` to `01`. + +If this `major.minor.patch` (e.g. `0.0.1`) version of the Chart has been released before, increment the `packageVersion`. + +### Porting over Charts / Assets from another Branch + +In the Staging branch, porting over charts from another branch (e.g. `dev-v2.x+1`) requires you to copy the contents of that branch into your Staging branch, which can be done with the following simple Bash script. However, you will need to manually regenerate the Helm index since you only want the index.yaml on the Staging branch to be updated to include the new chart. + +```bash +# Assuming that your upstream remote (e.g. https://github.com/rancher/charts.git) is named `upstream` +# Replace the following environment variables +OTHER_BRANCH=dev-v2.x+1 +STAGING_BRANCH=dev-v2.x +FORKED_BRANCH=dev-v2.x-with-port +NEW_CHART_DIR=charts/rancher-monitoring/rancher-monitoring/X.Y.Z +NEW_ASSET_TGZ=assets/rancher-monitoring/rancher-monitoring-X.Y.Z.tgz +git fetch upstream +git checkout upstream/${STAGING_BRANCH} -b ${FORKED_BRANCH} +git branch -u origin/${FORKED_BRANCH} +git checkout upstream/${OTHER_BRANCH} -- ${NEW_CHART_DIR} ${NEW_ASSET_TGZ} +helm repo index --merge ./index.yaml --url assets assets; # FYI: This will generate new 'created' timestamps across *all charts*. +mv assets/index.yaml index.yaml +git add ${NEW_CHART_DIR} ${NEW_ASSET_TGZ} index.yaml +git commit -m "Porting a chart from ${OTHER_BRANCH}" +git push --set-upstream origin ${FORKED_BRANCH} +# Create your pull request! +``` + +Once complete, you should see the following: +- The new chart should exist in `assets` and `charts`. Existing charts should not be modified. +- The `index.yaml`'s diff should show an additional entry for your new chart. +- The `index.yaml`'s diff should show modified `created` timestamps across all charts (due to the behavior of `helm repo index`). No other changes are expected. -Note: these steps should be taken only after following the steps to cut a release to your Live Branch. - ### Makefile #### Basic Commands `make pull-scripts`: Pulls in the version of the `charts-build-scripts` indicated in scripts. -`make sync`: Syncs the assets in your current repository with the merged contents of all of the repository branches indicated in your configuration.yaml +`make prepare`: Pulls in your charts from upstream and creates a basic `generated-changes/` directory with your dependencies from upstream + +`make patch`: Updates your `generated-changes/` to reflect the difference between upstream and the current working directory of your branch (note: this command should only be run after `make prepare`). + +`make clean`: Cleans up all the working directories of charts to get your repository ready for a PR + +`make charts`: Runs `make prepare` and then exports your charts to `assets/` and `charts/` and generates or updates your `index.yaml`. + +#### Advanced Commands `make validate`: Validates your current repository branch against all the repository branches indicated in your configuration.yaml -`make docs`: Pulls in the latest docs, scripts, etc. from the charts-build-scripts repository - -`make release`: moves the contents of the `assets/` directory into `released/assets/`, deletes the contents of the `charts/` directory, and updates the `index.yaml` to point to the new location for those assets. +`make template`: Updates the current directory by applying the configuration.yaml on [upstream Go templates](https://github.com/rancher/charts-build-scripts/tree/master/templates/template) to pull in the most up-to-date docs, scripts, etc. from [rancher/charts-build-scripts](https://github.com/rancher/charts-build-scripts) diff --git a/assets/README.md b/assets/README.md new file mode 100755 index 000000000..fbb1c0cec --- /dev/null +++ b/assets/README.md @@ -0,0 +1,3 @@ +## Assets + +This folder contains Helm chart archives that may or may not have been released yet. diff --git a/charts/README.md b/charts/README.md new file mode 100755 index 000000000..48d46dff7 --- /dev/null +++ b/charts/README.md @@ -0,0 +1,3 @@ +## Charts + +This folder contains unarchived Helm charts that may or may not have been released yet. diff --git a/packages/README.md b/packages/README.md new file mode 100755 index 000000000..4d4f89fed --- /dev/null +++ b/packages/README.md @@ -0,0 +1,115 @@ +## Packages + +### What is a Package? + +A Package represents a grouping of one or more Helm Charts. It is declared within `packages//package.yaml` with the following spec: + +```text +packageVersion: 0 +workingDir: # The directory within your package that will contain your working copy of the chart (e.g. charts) +url: # A URL pointing to an UpstreamConfiguration +subdirectory: # Optional field for a specific subdirectory for all upstreams +commit: # Optional field for a specific commit if your URL point to a Github Repository +additionalCharts: +# These contain other charts that you would like to package alongside this chart +- workingDir: # same as above + upstreamOptions: + # Mutually exclusive with crdOptions + url: # same as above + subdirectory: # optional, same as above + commit: # optional, same as above + crdOptions: + # Mutually exclusive with upstreamOptions + templateDirectory: # A directory within packages//template that will contain a template for your CRD chart + crdDirectory: # Where to place your CRDs within a CRD chart (e.g. crds for default charts) + addCRDValidationToMainChart: # Whether to add additional validation to your main chart to check that the CRD chart is installed. +``` + +As seen in the spec above, every Package must have exactly one Chart designated as a main Chart (multiple main Charts are not supported at this time) and all other Charts will be considered AdditionalCharts. + +#### UpstreamOptions + +Charts or AdditionalCharts can provide UpstreamOptions with the following possible configurations: +- Chart Archive: provide the `url` and optionally `subdirectory` +- Github Repository: provide the `url` (e.g. `https://github.com/rancher/charts-build-scripts.git`) and optionally a `subdirectory` and a `commit` +- Package: provide a `url: packages/` and the main Chart from that package can be pulled. You should ensure that a loop is not introduced. +- Local: provide `url: local` and the package will assume the contents of `workingDir` are exactly the chart you want to use. + +#### [AdditionalCharts] CRDOptions + +AdditionalCharts can provide CRDOptions instead of UpstreamOptions. These CRDOptions allow the scripts to automatically construct a CRD chart from your main Chart's contents based on the template provided. + +A CRD Chart is a Helm Chart whose sole purpose is to install CRDs onto a cluster before the main Chart is installed. + +You should not need a CRD chart if your main chart has the following qualities: +1) Your main chart does not install any CRDs. +2) Even if your main chart installs CRDs, it never installs resources of that kind as part of the release. In this case, CRDs can just remain in your `templates/` directory to be managed by Helm. +3) Neither option from above applies to you, but you do not need to facilitate automatically upgrading CRDs or providing a way for a user to cleanly delete CRDs via a second Helm release. In this case, the current Helm feature of having your CRDs placed in the `crds/` directory should work for you. + +### Directory Structure + +```text +packages/ + / + package.yaml # A file that represents your package's overall configuration + generated-changes/ + additional-charts/ + # Contains one directory per additional chart, keeping track of its dependencies and patches + / + generated-changes/ + # Same as above, but no more additionalCharts + dependencies/ + # Contains one directory per dependency. + + dependency.yaml # The UpstreamConfiguration of a particular dependency + exclude/ + # Files that were excluded from upstream verbatim. Follows the same directory structure as the chart + overlay/ + # Files that were overlaid onto upstream verbatim. Follows the same directory structure as the chart + patch/ + # Files that were patches from upstream. Follows the same directory structure as the chart and contains Unified Unix Diffs + templates/ + # Contains any templates. Currently only used by CRDOptions +``` + +### Developer Workflow + +Developers will use the following commands to work with packages: + +- `make prepare`: Pulls in your charts from upstream and creates a basic `generated-changes/` directory with your dependencies from upstream + +- `make patch`: Updates your `generated-changes/` to reflect the difference between upstream and the current working directory of your branch. Requires prepare + +- `make charts`: Runs prepare and then exports your charts to `assets/` and `charts/` and generates or updates your `index.yaml`. This is expected to be run before submitting a PR. Once this has been run and pushed to a branch, a Rancher Helm Repository that points to a branch that has these directories with the `index.yaml` should be able to find and deploy working copies of your chart. + +- `make clean`: Cleans up all the working directories of charts to get your repository ready for a PR + +To update your working copy of the charts-build-scripts after rebasing against upstream, run: + +- `make pull-scripts`: Pulls in the version of the `charts-build-scripts` indicated in scripts + +To check whether the new packages you are introducing will cause any issues with upstream that contains already released charts, run: + +- `make validate`: Validates your current repository branch against all the repository branches indicated in your configuration.yaml + +#### Common Workflow + +- Make or update the `packages//package.yaml` to point to your upstream and set any other chart options +- Run `PACKAGE= make prepare` to pull in your upstream repositories + - Note: On a prepare, the charts-build-scripts will automatically replace your charts existing dependencies with dependencies that will show up in `generated-changes/dependencies//dependency.yaml`. Since the spec of this file follows the `UpstreamConfiguration` described above, you can modify this to point your dependencies to a local chart (rooted at `generated-changes/dependencies//`), another package, a Chart archive (e.g. point it to a newer version of the dependency), or a Github Repository (at a subdirectory / commit). + - Note: As part of replacing your dependencies, the Chart.yaml and requirements.yaml are considered "Managed Files". These files are prone to having conflicts and may need to be deleted / recreated on a prepare if there are conflicts. Please open up an issue if you encounter frequent bugs with these files so that we can take a look. +- Make changes to the relevant working directories +- Run `PACKAGE= make patch` to save your changes +- Run `make clean`. Make a commit with your `packages/` changes +- Run `make charts`. Make a commit with your `charts/` changes +- Test your changes by pushing it to a branch. Once it is available at such a branch, it can be picked up by any tools that can point to Github based Helm Repositories (e.g. Rancher Cluster Explorer Apps & Marketplace). + - Note: If you encounter bugs on testing and need to modify your commits, it is recommended that you continue to keep your `packages/` changes and `charts/` changes separate. Ideally, you should have 1+ commits that modify the `packages/` directory for reviewers to view and 1 commit that modifies the `charts/` directory as a generated changes commit. +- Once you are ready to make a PR, ensure the last commit contains only the generated `make charts` changes and submit a PR. The PR workflow will automatically run `make validate` to ensure that your current repository wouldn't introduce any conflicts during a release. +- Open up a PR with your changes. + +### Troubleshooting + +Open up an issue on [https://github.com/rancher/charts-build-scripts](https://github.com/rancher/charts-build-scripts) + +### Maintainers +- aiyengar2 (arvind.iyengar@rancher.com) diff --git a/scripts/package-ci b/scripts/package-ci new file mode 100755 index 000000000..84b393cff --- /dev/null +++ b/scripts/package-ci @@ -0,0 +1,42 @@ +#!/bin/bash + +if [ -n "$(git status --porcelain)" ]; then + echo "Git needs to be clean to run this script" + exit 1 +fi + +ORIG_PACKAGE=${PACKAGE} + +FAILED_PACKAGES=() +for package in $(find packages -type d -mindepth 1 -maxdepth 1 | sort); do + export PACKAGE="${package#packages\/}" + + # Do developer workflow + if ! make prepare > /dev/null 2>&1; then + FAILED_PACKAGES+=("- ${PACKAGE} failed at prepare") + elif ! make patch > /dev/null 2>&1; then + FAILED_PACKAGES+=("- ${PACKAGE} failed at patch") + elif ! make clean > /dev/null 2>&1; then + FAILED_PACKAGES+=("- ${PACKAGE} failed at clean") + elif [ -n "$(git status --porcelain)" ]; then + FAILED_PACKAGES+=("- ${PACKAGE} generated additional changes: $(git status --porcelain | xargs)") + elif ! make charts > /dev/null 2>&1; then + FAILED_PACKAGES+=("- ${PACKAGE} failed at charts") + else + echo "${PACKAGE} passed ci" + fi + + git clean -df > /dev/null 2>&1 + git checkout -- . > /dev/null 2>&1 +done + +echo "" +if [ ${#FAILED_PACKAGES[@]} -ne 0 ]; then + echo "Failed to pass ci for the following packages:" + printf '%s\n' "${FAILED_PACKAGES[@]}" +else + echo "All packages pass ci!" +fi + +export PACKAGE=${ORIG_PACKAGE} +unset ORIG_PACKAGE \ No newline at end of file diff --git a/scripts/regenerate-assets b/scripts/regenerate-assets index cfcb16de4..153eac9b3 100755 --- a/scripts/regenerate-assets +++ b/scripts/regenerate-assets @@ -1,6 +1,9 @@ #!/bin/bash set -e +# Note: These scripts are only intended to migrate from the original build scripts to charts-build-scripts v0.1.x +# A separate migration process is required for v0.2.x + cd $(dirname $0) if [[ -z ${BRANCH} ]]; then diff --git a/scripts/regenerate-packages b/scripts/regenerate-packages new file mode 100755 index 000000000..150b2ee49 --- /dev/null +++ b/scripts/regenerate-packages @@ -0,0 +1,183 @@ +#!/bin/bash +set -e + +# Note: These scripts are only intended to migrate from the original build scripts to charts-build-scripts v0.1.x +# A separate migration process is required for v0.2.x + +cd $(dirname $0) + +if [[ -z ${BRANCH} ]]; then + branch=$(git rev-parse --abbrev-ref HEAD) +else + branch=${BRANCH} +fi + +echo "Using branch ${branch}" + +if [[ -z ${REPOSITORY} ]]; then + echo "Need to provide REPOSITORY as environment variable" + exit 1 +fi + +cd .. + +# Setup +rm -rf ./repository +mkdir -p ./repository +cd repository + +# Pull in branch +echo "Pulling in ${REPOSITORY}@${branch}" +git clone --depth 1 --branch ${branch} ${REPOSITORY} . > /dev/null 2>&1 + +if ! (test -d packages); then + echo "There are no packages in this repository" + cd .. + rm -rf ./repository + exit 1 +fi + +# Run make prepare to get all the packages passed through the scripts +make prepare + +# Copy the working directories of the packages after running prepare on them +for package in packages/*; do + cp -R ${package} ../packages/ +done + +# Go back +cd .. +rm -rf ./repository + +# Initialize package +for package in packages/*; do + if ! (test -d ${package}); then + continue + fi + # Add any overlay files and delete + if test -d ${package}/overlay; then + cp -R ${package}/overlay/* ${package}/charts # Usually happens on make charts + rm -rf ${package}/overlay + fi + # Delete the patch file + if test -f ${package}/*.patch; then + rm ${package}/*.patch + fi + # Remove comments from the Chart.yaml since this causes weird problems with Helm loading + if test -f ${package}/charts/Chart.yaml; then + sed -i '' '/^#/d' ${package}/charts/Chart.yaml + fi + # Local charts + if ! (test -f ${package}/package.yaml); then + # Local chart has no package.yaml + yq n 'url' 'local' > ${package}/package.yaml + elif [[ -z $(yq r ${package}/package.yaml 'url') ]]; then + # Local chart's package.yaml does not have url + yq w -i ${package}/package.yaml 'url' 'local' + fi + # Remove deprecated fields and add additional fields + if [[ -n $(yq r ${package}/package.yaml 'type') ]]; then + yq d -i ${package}/package.yaml 'type' + fi + if [[ -z $(yq r ${package}/package.yaml 'packageVersion') ]]; then + yq w -i ${package}/package.yaml 'packageVersion' '00' + fi +done + +# Update the dependencies +for package in packages/*; do + if ! (test -d ${package} && test -d ${package}/charts/charts); then + continue + fi + # Untar any dependencies + for dependency in ${package}/charts/charts/*; do + if test -f ${dependency} && [[ ${dependency} == *.tgz ]]; then + # Untar the dependency + tar -xvzf ${dependency} -C ${package}/charts/charts > /dev/null 2>&1 + rm ${dependency} + fi + done + # Move dependency into generated-changes + for dependency in ${package}/charts/charts/*; do + if test -d ${dependency}; then + dependency_name=$(basename -- ${dependency}) + dependency_gc_dir=${package}/generated-changes/dependencies/${dependency_name} + mkdir -p ${dependency_gc_dir} + if test -d packages/$(basename -- ${dependency}); then + # It tracks another package in the repository + yq n 'url' "packages/${dependency_name}" > ${dependency_gc_dir}/dependency.yaml + elif test -f ${package}/charts/requirements.yaml; then + # Get the URL from the dependencies + repository=$(yq r ${package}/charts/requirements.yaml "dependencies[name == ${dependency_name}].repository") + version=$(yq r ${package}/charts/requirements.yaml "dependencies[name == ${dependency_name}].version") + url=$(curl -sLf ${repository}/index.yaml | cat | yq r - "entries.${dependency_name}.(version == ${version}).urls[0]") + if [[ -z ${url} ]]; then + echo "Count not find download URL for ${dependency}" + exit 1 + fi + yq n 'url' ${url} > ${dependency_gc_dir}/dependency.yaml + unset url + else + # Has to be a local dependency + mkdir -p ${dependency_gc_dir}/charts + cp -R ${dependency}/* ${dependency_gc_dir}/charts + yq n 'url' 'local' > ${dependency_gc_dir}/dependency.yaml + yq w -i ${dependency_gc_dir}/dependency.yaml 'workingDir' 'charts' + fi + rmdir ${package}/charts/charts 2> /dev/null || echo "No directory to delete" > /dev/null + fi + done +done + +# Update the CRD charts +for package in packages/*; do + if ! (test -d ${package} && [[ -n $(yq r ${package}/package.yaml 'generateCRDChart') ]]); then + continue + fi + if [[ $(yq r ${package}/package.yaml 'generateCRDChart.enabled') != "true" ]]; then + rm -rf ${package}/charts-crd + continue + fi + unset assume_ownership_of_crds + if [[ $(yq r ${package}/package.yaml 'generateCRDChart.assumeOwnershipOfCRDs') == "true" ]]; then + assume_ownership_of_crds=1 + fi + # Copy template out to templates/crd-template + mkdir -p ${package}/templates/crd-template + mkdir -p ${package}/charts/crds + cp -R ${package}/charts-crd/* ${package}/templates/crd-template + if [[ -z ${assume_ownership_of_crds} ]]; then + mv ${package}/templates/crd-template/templates/* ${package}/charts/crds + else + mv ${package}/templates/crd-template/crd-manifest/* ${package}/charts/crds + fi + # Remove validate-install-crd.yaml file from original chart + rm ${package}/charts/templates/validate-install-crd.yaml + # Remove generateCRDChart from package.yaml + yq d -i ${package}/package.yaml 'generateCRDChart' +done + +# Do a make patch with the current scripts +PACKAGE="" make patch + +# Update the additional charts with the CRD charts +for package in packages/*; do + if ! (test -d ${package}); then + continue + fi + # Add back in to package.yaml as additional chart + if test -d ${package}/charts-crd; then + yq w -i ${package}/package.yaml 'additionalCharts[0].workingDir' 'charts-crd' + yq w -i ${package}/package.yaml 'additionalCharts[0].crdOptions.templateDirectory' 'crd-template' + if [[ -z ${assume_ownership_of_crds} ]]; then + yq w -i ${package}/package.yaml 'additionalCharts[0].crdOptions.crdDirectory' 'templates' + else + yq w -i ${package}/package.yaml 'additionalCharts[0].crdOptions.crdDirectory' 'crd-manifest' + fi + yq w -i ${package}/package.yaml 'additionalCharts[0].crdOptions.addCRDValidationToMainChart' 'true' + rm -rf ${package}/charts-crd + fi +done + +# Clean up the current directory +PACKAGE="" make clean \ No newline at end of file