Merge pull request #1 from StrongMonkey/master

Add framework for partner-charts

.github/workflows/pull-request.yaml

@@ -0,0 +1,15 @@
+name: CI-pullrequest
+
+on:
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Validate
+        run: sudo make validate

.github/workflows/push.yml

@@ -0,0 +1,19 @@
+name: CI-push
+
+on:
+  push:
+    branches: [ master ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Run CI scripts
+      run: |-
+        sudo make ci
+
+    - uses: stefanzweifel/git-auto-commit-action@v4.1.1
+      with:
+        commit_message: Generated changes

.gitignore

@@ -0,0 +1,2 @@
+/packages/**/charts-original
+/.idea

Makefile

@@ -0,0 +1,26 @@
+ci: bootstrap
+	./scripts/ci
+
+prepare: bootstrap
+	./scripts/prepare
+
+bootstrap:
+	./scripts/bootstrap
+
+charts: bootstrap prepare
+	./scripts/generate-charts
+
+patch: bootstrap
+	./scripts/generate-patch
+
+validate: bootstrap
+	./scripts/validate
+
+mirror: bootstrap
+	./scripts/image-mirror
+
+clean:
+	./scripts/clean
+
+.DEFAULT_GOAL := ci
+

README.md

@@ -1 +1,126 @@
-# partner-charts
+# partner-charts
+
+This is an experimental project to automate system charts and adding overlays files on top of it.
+
+Charts will have two categories: 
+
+1. Charts that Rancher created and maintained (**Rancher original**).
+
+2. Charts that Rancher modified from upstream (**Rancher modified**). 
+
+**Rancher original** chart is created and maintained by Rancher Team, such as rancher-cis-benchmark, rancher-k3s-upgrader. 
+
+**Rancher modified** chart is modified from upstream chart, while there are customizations added into the upstream chart from rancher side.
+
+For **Rancher original** charts, it should have the following tree structure
+
+```text
+packages/${CHART_NAME}/
+  charts/                   # regular helm chart directory
+    templates/
+    Chart.yaml
+    values.yaml
+```
+
+For **Rancher modified** charts, it should have the following tree structure
+
+```text
+packages/${CHART_NAME}/
+  package.yaml              # metadata manifest containing upstream chart location, package version
+  ${CHART_NAME}.patch       # patch file containing the diff between modified chart and upstream
+  overlay/*                 # overlay files that needs to added on top of upstream, for example, questions.yaml
+```
+
+A regular `package.yaml` will have the following content:
+
+```yaml
+url: https://charts.bitnami.com/bitnami/external-dns-2.20.10.tgz # url to fetch upstream chart
+packageVersion: 00 # packageVersion of modified charts, producing a $version-$packageVersion chart. For example, if istio 1.4.7 is modified with changes, rancher produces a 1.4.700 chart version that includes the modification rancher made on top of upstream charts.
+```
+
+Here is an example of upstream chart based on git repository
+
+```yaml
+url: https://github.com/open-policy-agent/gatekeeper.git  # Url to fetch upstream chart from git
+subdirectory: chart/gatekeeper-operator # Sub directory for helm charts in git repo
+type: git # optinal, indicate that upstream chart is from git
+commit: v3.1.0-beta.8 # the revision of git repo
+packageVersion: 00 # package version
+``` 
+
+### Example
+
+Todo: Add example
+
+### Workflow
+
+Modifying **Rancher original** charts is the same workflow as modifying helm charts. First make changes into `charts/` and commit changes. CI will automatically upload artifacts if file contents have been changed.
+
+Modifying **Rancher modified** takes extra steps, as it requires modifications to be saved into patch files so that later it can retrieve the chart based on upstream chart and patch files.
+
+The step includes:
+
+1. Run `make CHART={CHART_NAME} prepare`
+   
+   This prepares `charts` with the current upstream chart and current patch. 
+   
+2. Change the version in `package.yaml`. If upstream chart needs to be updated, update url to point the latest chart. `packageVersion` also needs to updated.
+
+3. Make modification to your charts. 
+
+4. Run `make CHART={CHART_NAME} patch`
+ 
+   This will compare your current chart with upstream chart and generate the correct patch. 
+   
+5. Run `make CHART={CHART_NAME} clean`
+   
+   This will clean up the `charts` directory so that it won't committed.
+
+This repo provides a [workflow](./.github/workflows) that automatically uploads patch files and tarball of charts. Commit will only need to update `package/${chart-name}/charts` and make sure patches are 
+up-to-date with the latest chart. It also automatically build github pages to serve `index.yaml` and artifacts of charts.
+
+### Override existing Chart
+
+By defauly CI script doesn't allow changes to be made against existing chart. In order to make changes you have to bump chart version. There is a backdoor method to make changes to your existing chart without having to bump version. You can delete the tar.gz file you want to override and commit the change. Here is an example of [commit](https://github.com/rancher/dev-charts/commit/8be888076487e23a24121a532d25b9bf9ea936f3).
+
+### Helm repo index
+
+To add this repo as a helm repo, run
+
+```text
+helm repo add ${repo_name} https://partner-charts.rancher.io
+```
+
+To use a forked version of this chart repo, you can try either of these:
+
+1. If you just need to test chart tar.gz file, you can run `make CHART=${name} charts` to generate tar.gz files. It will be generated under `docs/${chart_name}`.
+
+2. You can also setup github page to serve your tar.gz files on your forked repo. Github pages usually requires you to have this set up on [specific branches](https://help.github.com/en/github/working-with-github-pages/configuring-a-publishing-source-for-your-github-pages-site#choosing-a-publishing-source). 
+
+3. You can directly add `https://github.com/rancher/partner-charts` into rancher catalog. In order to show all the charts you have to run `make CHART=${chart_name} prepare` and make sure there is `chart-original` folder on each chart folder if your chart relies on a upstream chart.
+
+### Makefile
+
+`make bootstrap`: 
+
+Download binaries that are needed for ci scripts.
+
+`make prepare`: 
+
+Prepare the chart for modification. This will apply the upstream chart with the current patch. Use `CHART=${NAME}` for specific chart.
+
+`make charts`: 
+
+Generate tarball for each charts. Use `CHART=${NAME}` for specific chart.
+
+`make patch`: 
+
+Compare the current chart with upstream and generate patch file. Use `CHART=${NAME}` for specific chart. 
+
+`make validate`:
+
+Validate if patch file can be applied.
+
+`make mirror`: 
+
+Run image mirroring scripts.(Experimental)

scripts/bootstrap

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -e
+
+if [[ $(uname -s) == 'Darwin' ]]; then 
+	echo "Please make sure you have helm and yq installed"
+	exit 0
+fi
+
+# Download helm
+if [[ -z $(command -v helm) ]] || [[ $(helm version --short) != "v3"* ]]; then
+	curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash && chmod +x /usr/local/bin/helm
+fi
+
+# Download yq
+if [[ -z $(command -v yq) ]]; then
+	curl -sLf https://github.com/mikefarah/yq/releases/download/3.2.1/yq_linux_amd64 > /usr/local/bin/yq && chmod +x /usr/local/bin/yq
+fi

scripts/ci

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -e
+
+./scripts/prepare
+./scripts/generate-charts
+./scripts/clean

scripts/clean

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+for f in packages/*; do
+  if [[ -f ${f}/package.yaml ]]; then
+  	if [[ -z $CHART || $CHART == $(basename -- ${f}) ]]; then
+  		rm -rf ${f}/charts
+  	fi
+  fi
+done

scripts/generate-charts

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+set -e
+
+add=0
+for f in packages/*; do
+	if [[ -z $CHART || $CHART == $(basename -- ${f}) ]]; then
+		mkdir -p docs/$(basename -- ${f})
+		if [[ -d ${f}/overlay ]]; then
+			cp -R ${f}/overlay/* ${f}/charts
+		fi
+		if [[ -f ${f}/package.yaml ]]; then
+			version=$(yq r ${f}/charts/Chart.yaml version)
+			packageVersion=$(yq r ${f}/package.yaml packageVersion)
+			yq w -i ${f}/charts/Chart.yaml 'version' "${version}${packageVersion}"
+		fi
+		helm package ${f}/charts --destination docs/$(basename -- ${f})
+		git checkout docs/$(basename -- ${f})/"$(basename -- ${f})-$(yq r ${f}/charts/Chart.yaml version).tgz" || true
+	fi
+done
+
+if [[ -n $(git ls-files --other | grep "tgz") ]]; then
+	helm repo index --merge ./docs/index.yaml --url https://partner-charts.rancher.io docs
+fi

scripts/generate-patch

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+set -e
+set -x
+
+for f in packages/*; do
+  if [[ -f ${f}/package.yaml ]]; then
+  	if [[ -z $CHART || $CHART == $(basename -- ${f}) ]]; then
+			mkdir -p ${f}/charts-original
+			url=$(cat ${f}/package.yaml | yq r - url)
+			subdirectory=$(cat ${f}/package.yaml | yq r - subdirectory)
+			type=$(cat ${f}/package.yaml | yq r - type)
+			commit=$(cat ${f}/package.yaml | yq r - commit)
+			fields=$(echo ${subdirectory} | awk -F'/' '{print NF}')
+			if [[ $fields -eq '0' ]]; then
+				fields='1'
+			fi
+			if [[ $type == 'git' ]]; then
+				mkdir -p /tmp/tmp-charts
+				git clone $url /tmp/tmp-charts
+				pwd=$(pwd)
+				cd /tmp/tmp-charts/${subdirectory}
+				git checkout $commit
+				cd $pwd
+				cp -r /tmp/tmp-charts/${subdirectory}/* ${f}/charts-original
+				rm -rf /tmp/tmp-charts
+			else
+				curl -sLf ${url} | tar xvzf - -C ${f}/charts-original --strip ${fields} ${subdirectory} > /dev/null 2>&1
+			fi
+			if [[ -d ${f}/charts ]]; then
+				diff -uNr ${f}/charts-original ${f}/charts > ${f}/$(basename -- ${f}).patch || true
+			fi
+			rm -rf ${f}/charts-original
+		fi
+  fi
+done

scripts/image-mirror

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+set -e
+
+for f in packages/*; do
+	if [[ -f ${f}/package.yaml ]]; then
+		if [[ -n $(cat ${f}/package.yaml | yq r - imageMirror) ]]; then
+			for (( i=0; i<$(cat ${f}/package.yaml | yq r - imageMirror --length); i++ )); do
+				name=$(cat ${f}/package.yaml | yq r - imageMirror.[$i].name)
+				for (( j=0; j<$(cat ${f}/package.yaml | yq r - imageMirror.[$i].images --length); j++ )); do
+					arch=$(cat ${f}/package.yaml | yq r - imageMirror.[$i].images.[$j].arch)
+					os=$(cat ${f}/package.yaml | yq r - imageMirror.[$i].images.[$j].os)
+					original=$(cat ${f}/package.yaml | yq r - imageMirror.[$i].images.[$j].original)
+					tag=$(cat ${f}/package.yaml | yq r - imageMirror.[$i].images.[$j].tag)
+					docker pull $original:$tag
+					docker tag $original:$tag $name-$arch:$tag
+					docker push $name-$arch:$tag
+					docker manifest create --amend $name:$tag $name-$arch:$tag
+					docker manifest annotate $name:$tag $name-$arch:$tag --arch $arch --os $os
+				done
+				docker manifest push -p $name:$tag
+			done
+		fi
+  fi
+done

scripts/prepare

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+set -e
+
+# rebase the patch with upsteam charts defined in package.yaml. Make sure patches still apply
+# for the latest upsteam charts. If there is conflits commiter should solve it.
+
+for f in packages/*; do
+  if [[ -f ${f}/package.yaml ]]; then
+  	if [[ -z $CHART || $CHART == $(basename -- ${f}) ]]; then
+			url=$(cat ${f}/package.yaml | yq r - url)
+			subdirectory=$(cat ${f}/package.yaml | yq r - subdirectory)
+			type=$(cat ${f}/package.yaml | yq r - type)
+			fields=$(echo ${subdirectory} | awk -F'/' '{print NF}')
+			commit=$(cat ${f}/package.yaml | yq r - commit)
+			if [[ $fields -eq '0' ]]; then
+				fields='1'
+			fi
+			rm -rf ${f}/charts
+			mkdir -p ${f}/charts
+			if [[ $type == 'git' ]]; then
+				mkdir -p /tmp/tmp-charts
+				git clone $url /tmp/tmp-charts
+				pwd=$(pwd)
+				cd /tmp/tmp-charts/${subdirectory}
+				git checkout $commit
+				cd $pwd
+				cp -r /tmp/tmp-charts/${subdirectory}/* ${f}/charts
+				rm -rf /tmp/tmp-charts
+			else
+				curl -sLf ${url} | tar xvzf - -C ${f}/charts --strip ${fields} ${subdirectory} > /dev/null 2>&1
+			fi
+			for file in $(find ./${f} -type f -name "*.patch"); do
+				basename=$(basename -- ${file})
+				patch -p3 -d ${f}/charts < ${f}/$basename
+			done
+		fi
+  fi
+done

scripts/validate

@@ -0,0 +1,14 @@
+#!/bin/bash
+set -e
+
+cd $(dirname $0)/..
+
+./scripts/prepare
+
+source ./scripts/version
+
+if [ -n "$DIRTY" ]; then
+    echo Git is dirty
+    git status
+    exit 1
+fi

scripts/version

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+if [ -n "$(git status --porcelain --untracked-files=no)" ]; then
+    DIRTY="-dirty"
+fi
+
+COMMIT=$(git rev-parse --short HEAD)
+GIT_TAG=${DRONE_TAG:-$(git tag -l --contains HEAD | head -n 1)}
+
+if [[ -z "$DIRTY" && -n "$GIT_TAG" ]]; then
+    VERSION=$GIT_TAG
+else
+    VERSION="${COMMIT}${DIRTY}"
+fi
+
+if [ -z "$ARCH" ]; then
+    ARCH=$(go env GOHOSTARCH)
+fi
+
+SUFFIX="-${ARCH}"
+TAG=${TAG:-${VERSION}}
+REPO=${REPO:-rancher}
+
+if echo $TAG | grep -q dirty; then
+    TAG=dev
+fi