2020-07-21 20:22:47 +00:00
|
|
|
#!/usr/bin/env bash
|
|
|
|
|
set -e
|
|
|
|
|
set -x
|
|
|
|
|
|
|
|
|
|
function remove_timestamp_from_diff {
|
|
|
|
|
prefix="[-\+]\{3\}"
|
|
|
|
|
filename_format="packages\/$(basename -- ${f})\/[^[:space:]]*"
|
|
|
|
|
# https://www.gnu.org/software/diffutils/manual/html_node/Example-Unified.html#Example-Unified
|
|
|
|
|
# Example timestamp: 2002-02-21 23:30:39.942229878 -0800
|
|
|
|
|
date="[[:digit:]]\{4\}-[[:digit:]]\{2\}-[[:digit:]]\{2\}"
|
|
|
|
|
time="[[:digit:]]\{2\}:[[:digit:]]\{2\}:[[:digit:]]\{2\}.[[:digit:]]\{9\}"
|
|
|
|
|
timezone="[-\+][[:digit:]]\{4\}"
|
|
|
|
|
timestamp_format="${date}[[:space:]]${time}[[:space:]]${timezone}"
|
2020-08-18 18:02:39 +00:00
|
|
|
sed -i.bak "s/\(${prefix} ${filename_format}\)[[:space:]]${timestamp_format}/\1/g" ${f}/$(basename -- ${f}).patch
|
|
|
|
|
rm ${f}/$(basename -- ${f}).patch.bak
|
2020-07-21 20:22:47 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for f in packages/*; do
|
2020-09-09 02:13:42 +00:00
|
|
|
if [[ -f ${f}/package.yaml ]]; then
|
|
|
|
|
if [[ -z $CHART || $CHART == $(basename -- ${f}) ]]; then
|
2020-07-21 20:22:47 +00:00
|
|
|
url=$(cat ${f}/package.yaml | yq r - url)
|
2020-09-02 00:05:17 +00:00
|
|
|
if [[ -z ${url} ]]; then
|
|
|
|
|
continue
|
|
|
|
|
fi
|
2020-07-21 20:22:47 +00:00
|
|
|
subdirectory=$(cat ${f}/package.yaml | yq r - subdirectory)
|
|
|
|
|
type=$(cat ${f}/package.yaml | yq r - type)
|
|
|
|
|
commit=$(cat ${f}/package.yaml | yq r - commit)
|
|
|
|
|
fields=$(echo ${subdirectory} | awk -F'/' '{print NF}')
|
|
|
|
|
if [[ $fields -eq '0' ]]; then
|
|
|
|
|
fields='1'
|
|
|
|
|
fi
|
|
|
|
|
if [[ $type == 'git' ]]; then
|
|
|
|
|
mkdir -p /tmp/tmp-charts
|
|
|
|
|
git clone --depth=1 --no-tags $url /tmp/tmp-charts
|
|
|
|
|
pwd=$(pwd)
|
|
|
|
|
cd /tmp/tmp-charts
|
2020-08-03 16:54:00 +00:00
|
|
|
git fetch origin $commit
|
2020-07-21 20:22:47 +00:00
|
|
|
git checkout $commit
|
|
|
|
|
cd $pwd
|
|
|
|
|
cp -r /tmp/tmp-charts/${subdirectory} ${f}/charts-original
|
|
|
|
|
rm -rf /tmp/tmp-charts
|
|
|
|
|
else
|
|
|
|
|
mkdir -p ${f}/charts-original
|
|
|
|
|
curl -sLf ${url} | tar xvzf - -C ${f}/charts-original --strip ${fields} ${subdirectory} > /dev/null 2>&1
|
|
|
|
|
fi
|
Allow patches on remote deps with fixed versions
This commit adds the ability to add patches on subcharts in `rancher/charts` that meet the following constraints:
- Must be a remote chart (i.e. the repo url cannot point to a `file://<path>`)
- Must have a fixed version number (since we need a consistent base to apply the patch on)
The following changes have been made to the scripts to faciliate this feature:
**prepare**
Only run the patch after preparing the subchart
**prepare-subcharts:**
Inital checks:
1. We only prepare the subcharts if the chart has a `requirements.yaml`.
2. Extract the patches that would be applied to the requirements.yaml and only apply that
Note: in the future, we need to add support for dependencies outlined in the Chart.yaml directory, which is currently the recommended approach for Helm 3 charts that use `apiVersion: v2`, but since none of our charts currently use that it is assumed that they will encode their dependencies in a `requirements.yaml` at the moment.
Generate special dependencies (overlay, package.yaml):
1. For dependencies that are mirrored (i.e. rancher-kiali-server, where the dependency itself is stored as a `package.yaml`) that have not generated a `charts/` directory yet, we recursively run the `prepare` script for them in order to generate the directories so that `helm dependency update` can pull them in. The assumption here is that there will be no circular dependencies, although there is no explicit check for this.
2. For dependencies that are overlaid, we perform the overlay so that `helm dependency update` operates as expected.
Pull in latest subcharts and update files:
1. Perform `helm dependency update` to update the `requirements.yaml` and `requirements.lock` with the latest subchart versions
Cleanup (overlay, package.yaml, patched requirements)
1. Remove overlaid dependencies since they won't be tracked in the patch
2. Run the `clean` script on the mirrored dependencies
3. Revert the patch on the requirements.yaml
Apply logic to enable / disable patching:
1. If a subchart is a local chart, leave it as a tgz so it will be ignored on patch
2. If a subchart does not have a fixed version number, leave it as a tgz so it will be ignored on patch
3. If neither of these are true, unarchive the tgz for the subchart and delete the tgz so it will be tracked by the patch
NOTE: since the dependency still exists as either a tgz or a directory, `generate-charts` will not break from this change.
**generate-patch:**
Loop through the local `charts/requirements.yaml` to update the `charts-original/charts` directory:
1. If a subchart is not a local chart, remove it as we don't need to check for patches against it
2. If a subchart does not have a fixed version number, remove it as we don't need to check for patches against it
3. If neither of these are true, pull in the version number specified from the remote helm repo to track patches against it.
**Why does the version number need to be fixed?**:
In general, Helm chart owners can use wildcards within a `requirements.yaml` to specify the version of a subchart when a specific patch version does not necessarily need to be fixed. This is usually a great feature since that means that parent charts do not need to be updated in order for a `helm dependency update` to automatically pull in the latest versions of a dependency that do not break the parent chart; however, this causes an issue with patching subcharts since the base that the patch is applied on is not fixed.
Therefore, this commit only allows users to apply patches on subcharts that have a fixed subchart version specified in their `requirements.yaml`.
**Why can't we track local charts?**
If the chart is local to this repo, the assumption is that you should directly be making the changes to the chart.
2020-08-24 20:28:24 +00:00
|
|
|
if [[ -f ${f}/charts/requirements.yaml ]]; then
|
|
|
|
|
i=0
|
|
|
|
|
while ! [[ -z $(yq r ${f}/charts/requirements.yaml "dependencies[${i}]") ]]; do
|
|
|
|
|
subchart=$(yq r ${f}/charts/requirements.yaml "dependencies[${i}].name" | head -n 1)
|
|
|
|
|
version=$(yq r ${f}/charts/requirements.yaml "dependencies[${i}].version" | head -n 1)
|
|
|
|
|
helm_repo=$(yq r ${f}/charts/requirements.yaml "dependencies[${i}].repository" | head -n 1)
|
2020-09-18 01:32:04 +00:00
|
|
|
# Remove trailing slash
|
|
|
|
|
[[ ${helm_repo} == */ ]] && helm_repo=${helm_repo%/}
|
Allow patches on remote deps with fixed versions
This commit adds the ability to add patches on subcharts in `rancher/charts` that meet the following constraints:
- Must be a remote chart (i.e. the repo url cannot point to a `file://<path>`)
- Must have a fixed version number (since we need a consistent base to apply the patch on)
The following changes have been made to the scripts to faciliate this feature:
**prepare**
Only run the patch after preparing the subchart
**prepare-subcharts:**
Inital checks:
1. We only prepare the subcharts if the chart has a `requirements.yaml`.
2. Extract the patches that would be applied to the requirements.yaml and only apply that
Note: in the future, we need to add support for dependencies outlined in the Chart.yaml directory, which is currently the recommended approach for Helm 3 charts that use `apiVersion: v2`, but since none of our charts currently use that it is assumed that they will encode their dependencies in a `requirements.yaml` at the moment.
Generate special dependencies (overlay, package.yaml):
1. For dependencies that are mirrored (i.e. rancher-kiali-server, where the dependency itself is stored as a `package.yaml`) that have not generated a `charts/` directory yet, we recursively run the `prepare` script for them in order to generate the directories so that `helm dependency update` can pull them in. The assumption here is that there will be no circular dependencies, although there is no explicit check for this.
2. For dependencies that are overlaid, we perform the overlay so that `helm dependency update` operates as expected.
Pull in latest subcharts and update files:
1. Perform `helm dependency update` to update the `requirements.yaml` and `requirements.lock` with the latest subchart versions
Cleanup (overlay, package.yaml, patched requirements)
1. Remove overlaid dependencies since they won't be tracked in the patch
2. Run the `clean` script on the mirrored dependencies
3. Revert the patch on the requirements.yaml
Apply logic to enable / disable patching:
1. If a subchart is a local chart, leave it as a tgz so it will be ignored on patch
2. If a subchart does not have a fixed version number, leave it as a tgz so it will be ignored on patch
3. If neither of these are true, unarchive the tgz for the subchart and delete the tgz so it will be tracked by the patch
NOTE: since the dependency still exists as either a tgz or a directory, `generate-charts` will not break from this change.
**generate-patch:**
Loop through the local `charts/requirements.yaml` to update the `charts-original/charts` directory:
1. If a subchart is not a local chart, remove it as we don't need to check for patches against it
2. If a subchart does not have a fixed version number, remove it as we don't need to check for patches against it
3. If neither of these are true, pull in the version number specified from the remote helm repo to track patches against it.
**Why does the version number need to be fixed?**:
In general, Helm chart owners can use wildcards within a `requirements.yaml` to specify the version of a subchart when a specific patch version does not necessarily need to be fixed. This is usually a great feature since that means that parent charts do not need to be updated in order for a `helm dependency update` to automatically pull in the latest versions of a dependency that do not break the parent chart; however, this causes an issue with patching subcharts since the base that the patch is applied on is not fixed.
Therefore, this commit only allows users to apply patches on subcharts that have a fixed subchart version specified in their `requirements.yaml`.
**Why can't we track local charts?**
If the chart is local to this repo, the assumption is that you should directly be making the changes to the chart.
2020-08-24 20:28:24 +00:00
|
|
|
if [[ ${helm_repo} == file://* ]] || [[ ${version} == *'*'* ]]; then
|
|
|
|
|
# Subchart is tracked locally; patches will not be tracked
|
|
|
|
|
rm -rf ${f}/charts-original/charts/${subchart}
|
|
|
|
|
((i=i+1))
|
|
|
|
|
continue
|
|
|
|
|
fi
|
|
|
|
|
# Subchart is tracked in remote Helm repo with a fixed version number; we grab the download URL and get the local chart archive
|
2020-09-18 01:32:04 +00:00
|
|
|
download_url=$(curl -sLf ${helm_repo}/index.yaml | cat | yq r - "entries.${subchart}.(version == ${version}).urls[0]")
|
|
|
|
|
if [[ -z ${download_url} ]]; then
|
|
|
|
|
echo "Failed to get download URL for ${subchart} at version ${version} from index.yaml at ${helm_repo}"
|
|
|
|
|
exit 1
|
|
|
|
|
fi
|
|
|
|
|
echo "Comparing current changes to ${subchart}-${version} against upstream found at ${download_url}"
|
Allow patches on remote deps with fixed versions
This commit adds the ability to add patches on subcharts in `rancher/charts` that meet the following constraints:
- Must be a remote chart (i.e. the repo url cannot point to a `file://<path>`)
- Must have a fixed version number (since we need a consistent base to apply the patch on)
The following changes have been made to the scripts to faciliate this feature:
**prepare**
Only run the patch after preparing the subchart
**prepare-subcharts:**
Inital checks:
1. We only prepare the subcharts if the chart has a `requirements.yaml`.
2. Extract the patches that would be applied to the requirements.yaml and only apply that
Note: in the future, we need to add support for dependencies outlined in the Chart.yaml directory, which is currently the recommended approach for Helm 3 charts that use `apiVersion: v2`, but since none of our charts currently use that it is assumed that they will encode their dependencies in a `requirements.yaml` at the moment.
Generate special dependencies (overlay, package.yaml):
1. For dependencies that are mirrored (i.e. rancher-kiali-server, where the dependency itself is stored as a `package.yaml`) that have not generated a `charts/` directory yet, we recursively run the `prepare` script for them in order to generate the directories so that `helm dependency update` can pull them in. The assumption here is that there will be no circular dependencies, although there is no explicit check for this.
2. For dependencies that are overlaid, we perform the overlay so that `helm dependency update` operates as expected.
Pull in latest subcharts and update files:
1. Perform `helm dependency update` to update the `requirements.yaml` and `requirements.lock` with the latest subchart versions
Cleanup (overlay, package.yaml, patched requirements)
1. Remove overlaid dependencies since they won't be tracked in the patch
2. Run the `clean` script on the mirrored dependencies
3. Revert the patch on the requirements.yaml
Apply logic to enable / disable patching:
1. If a subchart is a local chart, leave it as a tgz so it will be ignored on patch
2. If a subchart does not have a fixed version number, leave it as a tgz so it will be ignored on patch
3. If neither of these are true, unarchive the tgz for the subchart and delete the tgz so it will be tracked by the patch
NOTE: since the dependency still exists as either a tgz or a directory, `generate-charts` will not break from this change.
**generate-patch:**
Loop through the local `charts/requirements.yaml` to update the `charts-original/charts` directory:
1. If a subchart is not a local chart, remove it as we don't need to check for patches against it
2. If a subchart does not have a fixed version number, remove it as we don't need to check for patches against it
3. If neither of these are true, pull in the version number specified from the remote helm repo to track patches against it.
**Why does the version number need to be fixed?**:
In general, Helm chart owners can use wildcards within a `requirements.yaml` to specify the version of a subchart when a specific patch version does not necessarily need to be fixed. This is usually a great feature since that means that parent charts do not need to be updated in order for a `helm dependency update` to automatically pull in the latest versions of a dependency that do not break the parent chart; however, this causes an issue with patching subcharts since the base that the patch is applied on is not fixed.
Therefore, this commit only allows users to apply patches on subcharts that have a fixed subchart version specified in their `requirements.yaml`.
**Why can't we track local charts?**
If the chart is local to this repo, the assumption is that you should directly be making the changes to the chart.
2020-08-24 20:28:24 +00:00
|
|
|
curl -sLf ${download_url} | tar xvzf - -C ${f}/charts-original/charts > /dev/null 2>&1
|
|
|
|
|
((i=i+1))
|
|
|
|
|
done
|
|
|
|
|
fi
|
2020-07-21 20:22:47 +00:00
|
|
|
if [[ -d ${f}/charts ]]; then
|
2020-08-03 19:13:19 +00:00
|
|
|
split_crds=$(cat ${f}/package.yaml | yq r - generateCRDChart.enabled)
|
|
|
|
|
if [[ "${split_crds}" == "true" ]]; then
|
2020-09-08 18:58:49 +00:00
|
|
|
./scripts/clean-crds ${f}
|
|
|
|
|
fi
|
2020-07-21 20:22:47 +00:00
|
|
|
diff -x *.tgz -x *.lock -uNr ${f}/charts-original ${f}/charts > ${f}/$(basename -- ${f}).patch || true
|
Allow patches on remote deps with fixed versions
This commit adds the ability to add patches on subcharts in `rancher/charts` that meet the following constraints:
- Must be a remote chart (i.e. the repo url cannot point to a `file://<path>`)
- Must have a fixed version number (since we need a consistent base to apply the patch on)
The following changes have been made to the scripts to faciliate this feature:
**prepare**
Only run the patch after preparing the subchart
**prepare-subcharts:**
Inital checks:
1. We only prepare the subcharts if the chart has a `requirements.yaml`.
2. Extract the patches that would be applied to the requirements.yaml and only apply that
Note: in the future, we need to add support for dependencies outlined in the Chart.yaml directory, which is currently the recommended approach for Helm 3 charts that use `apiVersion: v2`, but since none of our charts currently use that it is assumed that they will encode their dependencies in a `requirements.yaml` at the moment.
Generate special dependencies (overlay, package.yaml):
1. For dependencies that are mirrored (i.e. rancher-kiali-server, where the dependency itself is stored as a `package.yaml`) that have not generated a `charts/` directory yet, we recursively run the `prepare` script for them in order to generate the directories so that `helm dependency update` can pull them in. The assumption here is that there will be no circular dependencies, although there is no explicit check for this.
2. For dependencies that are overlaid, we perform the overlay so that `helm dependency update` operates as expected.
Pull in latest subcharts and update files:
1. Perform `helm dependency update` to update the `requirements.yaml` and `requirements.lock` with the latest subchart versions
Cleanup (overlay, package.yaml, patched requirements)
1. Remove overlaid dependencies since they won't be tracked in the patch
2. Run the `clean` script on the mirrored dependencies
3. Revert the patch on the requirements.yaml
Apply logic to enable / disable patching:
1. If a subchart is a local chart, leave it as a tgz so it will be ignored on patch
2. If a subchart does not have a fixed version number, leave it as a tgz so it will be ignored on patch
3. If neither of these are true, unarchive the tgz for the subchart and delete the tgz so it will be tracked by the patch
NOTE: since the dependency still exists as either a tgz or a directory, `generate-charts` will not break from this change.
**generate-patch:**
Loop through the local `charts/requirements.yaml` to update the `charts-original/charts` directory:
1. If a subchart is not a local chart, remove it as we don't need to check for patches against it
2. If a subchart does not have a fixed version number, remove it as we don't need to check for patches against it
3. If neither of these are true, pull in the version number specified from the remote helm repo to track patches against it.
**Why does the version number need to be fixed?**:
In general, Helm chart owners can use wildcards within a `requirements.yaml` to specify the version of a subchart when a specific patch version does not necessarily need to be fixed. This is usually a great feature since that means that parent charts do not need to be updated in order for a `helm dependency update` to automatically pull in the latest versions of a dependency that do not break the parent chart; however, this causes an issue with patching subcharts since the base that the patch is applied on is not fixed.
Therefore, this commit only allows users to apply patches on subcharts that have a fixed subchart version specified in their `requirements.yaml`.
**Why can't we track local charts?**
If the chart is local to this repo, the assumption is that you should directly be making the changes to the chart.
2020-08-24 20:28:24 +00:00
|
|
|
if ! [[ -s ${f}/$(basename -- ${f}).patch ]]; then
|
|
|
|
|
# If no changes exist in patch then remove it
|
|
|
|
|
rm ${f}/$(basename -- ${f}).patch
|
|
|
|
|
fi
|
|
|
|
|
if [[ -f ${f}/$(basename -- ${f}).patch ]]; then
|
|
|
|
|
remove_timestamp_from_diff
|
|
|
|
|
fi
|
2020-08-03 19:13:19 +00:00
|
|
|
if [[ "${split_crds}" == "true" ]]; then
|
|
|
|
|
./scripts/prepare-crds ${f}
|
|
|
|
|
fi
|
2020-07-21 20:22:47 +00:00
|
|
|
fi
|
|
|
|
|
rm -rf ${f}/charts-original
|
|
|
|
|
fi
|
|
|
|
|
fi
|
|
|
|
|
done
|
