(dev-v2.6-archive) Mimic Helm CRD install process by introducing wait

As part of the Helm CRD installation process shown [here](43853ea772/pkg/action/install.go (L160-L162)), charts that both attempt to install CRDs and then install resources (e.g. `len(totalItems) > 0`) invalidate the cache and attempt to get the CRDs again to `Give time for the CRD to be recognized.` before continuing the Helm installation.

Because Monitoring V2 currently does not observe this invalidation and wait, flakey race conditions documented in issues like https://github.com/rancher/rancher/issues/32025 and https://github.com/rancher/rancher/issues/29171 can be caused. This commit introduces that invalidation process.

(partially cherry picked from commit 5f3bbe4a36)

packages/rancher-monitoring/templates/crd-template/templates/jobs.yaml

@@ -30,11 +30,24 @@ spec:
             {{- range $path, $_ :=  (.Files.Glob  "crd-manifest/**.yaml") }}
             {{- $crd := get (get ($.Files.Get $path | fromYaml) "metadata") "name" }}
             if [[ -n "$(kubectl get crd {{ $crd }} -o jsonpath='{.spec.preserveUnknownFields}')" ]]; then
-                patch="{\"spec\": {\"preserveUnknownFields\": false}}"
+                patch="{\"spec\": {\"preserveUnknownFields\": false}}";
                 if [[ -z "$(kubectl get crd {{ $crd }} -o jsonpath='{.spec.versions[0].schema}' 2>&1)" ]]; then
-                  patch="{\"spec\": {\"preserveUnknownFields\": false, \"versions\": [{\"name\": \"v1\", \"served\": false, \"storage\": true}]}}"
+                  patch="{\"spec\": {\"preserveUnknownFields\": false, \"versions\": [{\"name\": \"v1\", \"served\": false, \"storage\": true}]}}";
                 fi
-                kubectl patch crd {{ $crd }} -p "${patch}" --type="merge";
+                if kubectl patch crd {{ $crd }} -p "${patch}" --type="merge"; then
+                  echo "beginning wait for {{ $crd }} to be established...";
+                  num_tries=1;
+                  until kubectl get crd {{ $crd }} -o=jsonpath='{range .status.conditions[*]}{.type}={.status} {end}' | grep -qE 'Established=True'; do
+                    if (( num_tries == 30 )); then
+                      echo "timed out waiting for {{ $crd }}";
+                      exit 1;
+                    fi;
+                    num_tries=$(( num_tries + 1 ));
+                    echo "{{ $crd }} is not established. Sleeping for 2 seconds and trying again...";
+                    sleep 2;
+                  done;
+                  echo "successfully established {{ $crd }}";
+                fi;
             fi;
             {{- end }}
       containers:
@@ -45,7 +58,31 @@ spec:
           - /bin/sh
           - -c
           - >
-            kubectl apply -f /etc/config/crd-manifest.yaml
+            echo "Applying CRDs...";
+            kubectl apply -f /etc/config/crd-manifest.yaml;
+            
+            echo "Waiting for CRDs to be recognized before finishing installation...";
+
+            {{- range $path, $_ :=  (.Files.Glob  "crd-manifest/**.yaml") }}
+            {{- $apiGroup := get (get ($.Files.Get $path | fromYaml) "spec") "group" }}
+            rm -rf $HOME/.kube/cache/discovery/*/{{ $apiGroup }};
+            {{- end }}
+
+            {{- range $path, $_ :=  (.Files.Glob  "crd-manifest/**.yaml") }}
+            {{- $crd := get (get ($.Files.Get $path | fromYaml) "metadata") "name" }}
+            echo "beginning wait for {{ $crd }} to be established...";
+            num_tries=1;
+            until kubectl get crd {{ $crd }} -o=jsonpath='{range .status.conditions[*]}{.type}={.status} {end}' | grep -qE 'Established=True'; do
+              if (( num_tries == 30 )); then
+                echo "timed out waiting for {{ $crd }}";
+                exit 1;
+              fi;
+              num_tries=$(( num_tries + 1 ));
+              echo "{{ $crd }} is not established. Sleeping for 2 seconds and trying again...";
+              sleep 2;
+            done;
+            echo "successfully established {{ $crd }}";
+            {{- end }}
           volumeMounts:
             - name: crd-manifest
               readOnly: true
@@ -89,7 +126,20 @@ spec:
           - >
             {{- range $path, $_ :=  (.Files.Glob  "crd-manifest/**.yaml") }}
             {{- $crd := get (get ($.Files.Get $path | fromYaml) "metadata") "name" }}
-            kubectl patch crd {{ $crd }} -p '{"metadata": {"finalizers": []}}' || true;
+            if kubectl patch crd {{ $crd }} -p '{"metadata": {"finalizers": []}}'; then
+              echo "beginning wait for {{ $crd }} to be established...";
+              num_tries=1;
+              until kubectl get crd {{ $crd }} -o=jsonpath='{range .status.conditions[*]}{.type}={.status} {end}' | grep -qE 'Established=True'; do
+                if (( num_tries == 30 )); then
+                  echo "timed out waiting for {{ $crd }}";
+                  exit 1;
+                fi;
+                num_tries=$(( num_tries + 1 ));
+                echo "{{ $crd }} is not established. Sleeping for 2 seconds and trying again...";
+                sleep 2;
+              done;
+              echo "successfully established {{ $crd }}";
+            fi;
             {{- end }}
           volumeMounts:
             - name: crd-manifest