Add ability to configure calico failsafe ports

Made FailsafeOutboundHostPorts and FailsafeInboundHostPorts configurable
to allow users to have custom network policy for example SSH (22)

Previously failsafe rules would always have opened 22 even if user had
custom GlobalNetworkPolicy applied on the node.

Fixes https://github.com/rancher/rke2/issues/921
pull/473/head
actions 2021-04-29 00:16:16 +00:00
parent 467b7f5447
commit dbd66a2662
11 changed files with 819 additions and 0 deletions

Binary file not shown.

View File

@ -0,0 +1,13 @@
apiVersion: v1
appVersion: v3.13.3
description: Install Canal Network Plugin.
home: https://www.projectcalico.org/
keywords:
- canal
maintainers:
- email: charts@rancher.com
name: Rancher Labs
name: rke2-canal
sources:
- https://github.com/rancher/rke2-charts
version: v3.13.300-build2021022303

View File

@ -0,0 +1,3 @@
Canal network plugin has been installed.
NOTE: It may take few minutes until Canal image install CNI files and node become in ready state.

View File

@ -0,0 +1,7 @@
{{- define "system_default_registry" -}}
{{- if .Values.global.systemDefaultRegistry -}}
{{- printf "%s/" .Values.global.systemDefaultRegistry -}}
{{- else -}}
{{- "" -}}
{{- end -}}
{{- end -}}

View File

@ -0,0 +1,67 @@
---
# Source: calico/templates/calico-config.yaml
# This ConfigMap is used to configure a self-hosted Canal installation.
kind: ConfigMap
apiVersion: v1
metadata:
name: {{ .Release.Name }}-config
namespace: kube-system
data:
# Typha is disabled.
typha_service_name: {{ .Values.calico.typhaServiceName | quote }}
# The interface used by canal for host <-> host communication.
# If left blank, then the interface is chosen using the node's
# default route.
canal_iface: {{ .Values.flannel.iface | quote }}
# Whether or not to masquerade traffic to destinations not within
# the pod network.
masquerade: {{ .Values.calico.masquerade | quote }}
# Configure the MTU to use
veth_mtu: {{ .Values.calico.vethuMTU | quote }}
# The CNI network configuration to install on each node. The special
# values in this config will be automatically populated.
cni_network_config: |-
{
"name": "k8s-pod-network",
"cniVersion": "0.3.1",
"plugins": [
{
"type": "calico",
"log_level": "info",
"datastore_type": "kubernetes",
"nodename": "__KUBERNETES_NODE_NAME__",
"mtu": __CNI_MTU__,
"ipam": {
"type": "host-local",
"subnet": "usePodCidr"
},
"policy": {
"type": "k8s"
},
"kubernetes": {
"kubeconfig": "__KUBECONFIG_FILEPATH__"
}
},
{
"type": "portmap",
"snat": true,
"capabilities": {"portMappings": true}
},
{
"type": "bandwidth",
"capabilities": {"bandwidth": true}
}
]
}
# Flannel network configuration. Mounted into the flannel container.
net-conf.json: |
{
"Network": {{ .Values.podCidr | quote }},
"Backend": {
"Type": {{ .Values.flannel.backend | quote }}
}
}

View File

@ -0,0 +1,197 @@
---
# Source: calico/templates/kdd-crds.yaml
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: bgpconfigurations.crd.projectcalico.org
spec:
scope: Cluster
group: crd.projectcalico.org
version: v1
names:
kind: BGPConfiguration
plural: bgpconfigurations
singular: bgpconfiguration
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: bgppeers.crd.projectcalico.org
spec:
scope: Cluster
group: crd.projectcalico.org
version: v1
names:
kind: BGPPeer
plural: bgppeers
singular: bgppeer
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: blockaffinities.crd.projectcalico.org
spec:
scope: Cluster
group: crd.projectcalico.org
version: v1
names:
kind: BlockAffinity
plural: blockaffinities
singular: blockaffinity
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: clusterinformations.crd.projectcalico.org
spec:
scope: Cluster
group: crd.projectcalico.org
version: v1
names:
kind: ClusterInformation
plural: clusterinformations
singular: clusterinformation
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: felixconfigurations.crd.projectcalico.org
spec:
scope: Cluster
group: crd.projectcalico.org
version: v1
names:
kind: FelixConfiguration
plural: felixconfigurations
singular: felixconfiguration
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: globalnetworkpolicies.crd.projectcalico.org
spec:
scope: Cluster
group: crd.projectcalico.org
version: v1
names:
kind: GlobalNetworkPolicy
plural: globalnetworkpolicies
singular: globalnetworkpolicy
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: globalnetworksets.crd.projectcalico.org
spec:
scope: Cluster
group: crd.projectcalico.org
version: v1
names:
kind: GlobalNetworkSet
plural: globalnetworksets
singular: globalnetworkset
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: hostendpoints.crd.projectcalico.org
spec:
scope: Cluster
group: crd.projectcalico.org
version: v1
names:
kind: HostEndpoint
plural: hostendpoints
singular: hostendpoint
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: ipamblocks.crd.projectcalico.org
spec:
scope: Cluster
group: crd.projectcalico.org
version: v1
names:
kind: IPAMBlock
plural: ipamblocks
singular: ipamblock
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: ipamconfigs.crd.projectcalico.org
spec:
scope: Cluster
group: crd.projectcalico.org
version: v1
names:
kind: IPAMConfig
plural: ipamconfigs
singular: ipamconfig
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: ipamhandles.crd.projectcalico.org
spec:
scope: Cluster
group: crd.projectcalico.org
version: v1
names:
kind: IPAMHandle
plural: ipamhandles
singular: ipamhandle
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: ippools.crd.projectcalico.org
spec:
scope: Cluster
group: crd.projectcalico.org
version: v1
names:
kind: IPPool
plural: ippools
singular: ippool
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: networkpolicies.crd.projectcalico.org
spec:
scope: Namespaced
group: crd.projectcalico.org
version: v1
names:
kind: NetworkPolicy
plural: networkpolicies
singular: networkpolicy
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: networksets.crd.projectcalico.org
spec:
scope: Namespaced
group: crd.projectcalico.org
version: v1
names:
kind: NetworkSet
plural: networksets
singular: networkset

View File

@ -0,0 +1,266 @@
---
# Source: calico/templates/calico-node.yaml
# This manifest installs the canal container, as well
# as the CNI plugins and network config on
# each master and worker node in a Kubernetes cluster.
kind: DaemonSet
apiVersion: apps/v1
metadata:
name: {{ .Release.Name | quote }}
namespace: kube-system
labels:
k8s-app: canal
spec:
selector:
matchLabels:
k8s-app: canal
updateStrategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 1
template:
metadata:
labels:
k8s-app: canal
annotations:
# This, along with the CriticalAddonsOnly toleration below,
# marks the pod as a critical add-on, ensuring it gets
# priority scheduling and that its resources are reserved
# if it ever gets evicted.
scheduler.alpha.kubernetes.io/critical-pod: ''
spec:
nodeSelector:
kubernetes.io/os: linux
hostNetwork: true
tolerations:
# Make sure canal gets scheduled on all nodes.
- effect: NoSchedule
operator: Exists
# Mark the pod as a critical add-on for rescheduling.
- key: CriticalAddonsOnly
operator: Exists
- effect: NoExecute
operator: Exists
serviceAccountName: canal
# Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force
# deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods.
terminationGracePeriodSeconds: 0
priorityClassName: system-node-critical
initContainers:
# This container installs the CNI binaries
# and CNI network config file on each node.
- name: install-cni
image: {{ template "system_default_registry" . }}{{ .Values.calico.cniImage.repository }}:{{ .Values.calico.cniImage.tag }}
command: ["/install-cni.sh"]
env:
# Name of the CNI config file to create.
- name: CNI_CONF_NAME
value: "10-canal.conflist"
# The CNI network config to install on each node.
- name: CNI_NETWORK_CONFIG
valueFrom:
configMapKeyRef:
name: {{ .Release.Name }}-config
key: cni_network_config
# Set the hostname based on the k8s node name.
- name: KUBERNETES_NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
# CNI MTU Config variable
- name: CNI_MTU
valueFrom:
configMapKeyRef:
name: {{ .Release.Name }}-config
key: veth_mtu
# Prevents the container from sleeping forever.
- name: SLEEP
value: "false"
volumeMounts:
- mountPath: /host/opt/cni/bin
name: cni-bin-dir
- mountPath: /host/etc/cni/net.d
name: cni-net-dir
securityContext:
privileged: true
# Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes
# to communicate with Felix over the Policy Sync API.
- name: flexvol-driver
image: {{ template "system_default_registry" . }}{{ .Values.calico.flexvolImage.repository }}:{{ .Values.calico.flexvolImage.tag }}
command: ['/usr/local/bin/flexvol.sh', '-s', '/usr/local/bin/flexvol', '-i', 'flexvoldriver']
volumeMounts:
- name: flexvol-driver-host
mountPath: /host/driver
securityContext:
privileged: true
containers:
# Runs canal container on each Kubernetes node. This
# container programs network policy and routes on each
# host.
- name: calico-node
command:
- "start_runit"
image: {{ template "system_default_registry" . }}{{ .Values.calico.nodeImage.repository }}:{{ .Values.calico.nodeImage.tag }}
env:
# Use Kubernetes API as the backing datastore.
- name: DATASTORE_TYPE
value: {{ .Values.calico.datastoreType | quote }}
# Configure route aggregation based on pod CIDR.
- name: USE_POD_CIDR
value: {{ .Values.calico.usePodCIDR | quote }}
# Wait for the datastore.
- name: WAIT_FOR_DATASTORE
value: {{ .Values.calico.waitForDatastore | quote }}
# Set based on the k8s node name.
- name: NODENAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
# Don't enable BGP.
- name: CALICO_NETWORKING_BACKEND
value: {{ .Values.calico.networkingBackend | quote }}
# Cluster type to identify the deployment type
- name: CLUSTER_TYPE
value: {{ .Values.calico.clusterType | quote}}
# Period, in seconds, at which felix re-applies all iptables state
- name: FELIX_IPTABLESREFRESHINTERVAL
value: {{ .Values.calico.felixIptablesRefreshInterval | quote}}
- name: FELIX_IPTABLESBACKEND
value: {{ .Values.calico.felixIptablesBackend | quote}}
# No IP address needed.
- name: IP
value: ""
# The default IPv4 pool to create on startup if none exists. Pod IPs will be
# chosen from this range. Changing this value after installation will have
# no effect. This should fall within `--cluster-cidr`.
# - name: CALICO_IPV4POOL_CIDR
# value: "192.168.0.0/16"
# Disable file logging so `kubectl logs` works.
- name: CALICO_DISABLE_FILE_LOGGING
value: "true"
# Set Felix endpoint to host default action to ACCEPT.
- name: FELIX_DEFAULTENDPOINTTOHOSTACTION
value: {{ .Values.calico.felixDefaultEndpointToHostAction | quote }}
# Disable IPv6 on Kubernetes.
- name: FELIX_IPV6SUPPORT
value: {{ .Values.calico.felixIpv6Support | quote }}
# Set Felix logging to "info"
- name: FELIX_LOGSEVERITYSCREEN
value: {{ .Values.calico.felixLogSeverityScreen | quote }}
- name: FELIX_HEALTHENABLED
value: {{ .Values.calico.felixHealthEnabled | quote }}
# enable promentheus metrics
- name: FELIX_PROMETHEUSMETRICSENABLED
value: {{ .Values.calico.felixPrometheusMetricsEnabled | quote }}
- name: FELIX_XDPENABLED
value: {{ .Values.calico.felixXDPEnabled | quote }}
- name: FELIX_FAILSAFEINBOUNDHOSTPORTS
value: {{ .Values.calico.felixFailsafeInboundHostPorts | quote }}
- name: FELIX_FAILSAFEOUTBOUNDHOSTPORTS
value: {{ .Values.calico.felixFailsafeOutboundHostPorts | quote }}
securityContext:
privileged: true
resources:
requests:
cpu: 250m
livenessProbe:
exec:
command:
- /bin/calico-node
- -felix-live
periodSeconds: 10
initialDelaySeconds: 10
failureThreshold: 6
readinessProbe:
httpGet:
path: /readiness
port: 9099
host: localhost
periodSeconds: 10
volumeMounts:
- mountPath: /lib/modules
name: lib-modules
readOnly: true
- mountPath: /run/xtables.lock
name: xtables-lock
readOnly: false
- mountPath: /var/run/calico
name: var-run-calico
readOnly: false
- mountPath: /var/lib/calico
name: var-lib-calico
readOnly: false
- name: policysync
mountPath: /var/run/nodeagent
# This container runs flannel using the kube-subnet-mgr backend
# for allocating subnets.
- name: kube-flannel
image: {{ template "system_default_registry" . }}{{ .Values.flannel.image.repository }}:{{ .Values.flannel.image.tag }}
command:
- "/opt/bin/flanneld"
{{- range .Values.flannel.args }}
- {{ . | quote }}
{{- end }}
securityContext:
privileged: true
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: FLANNELD_IFACE
valueFrom:
configMapKeyRef:
name: {{ .Release.Name }}-config
key: canal_iface
- name: FLANNELD_IP_MASQ
valueFrom:
configMapKeyRef:
name: {{ .Release.Name }}-config
key: masquerade
volumeMounts:
- mountPath: /run/xtables.lock
name: xtables-lock
readOnly: false
- name: flannel-cfg
mountPath: /etc/kube-flannel/
volumes:
# Used by canal.
- name: lib-modules
hostPath:
path: /lib/modules
- name: var-run-calico
hostPath:
path: /var/run/calico
- name: var-lib-calico
hostPath:
path: /var/lib/calico
- name: xtables-lock
hostPath:
path: /run/xtables.lock
type: FileOrCreate
# Used by flannel.
- name: flannel-cfg
configMap:
name: {{ .Release.Name }}-config
# Used to install CNI.
- name: cni-bin-dir
hostPath:
path: /opt/cni/bin
- name: cni-net-dir
hostPath:
path: /etc/cni/net.d
# Used to create per-pod Unix Domain Sockets
- name: policysync
hostPath:
type: DirectoryOrCreate
path: /var/run/nodeagent
# Used to install Flex Volume Driver
- name: flexvol-driver-host
hostPath:
type: DirectoryOrCreate
path: {{ .Values.calico.flexVolumePluginDir }}/nodeagent~uds

View File

@ -0,0 +1,163 @@
---
# Source: calico/templates/rbac.yaml
# Include a clusterrole for the calico-node DaemonSet,
# and bind it to the calico-node serviceaccount.
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: calico-node
rules:
# The CNI plugin needs to get pods, nodes, and namespaces.
- apiGroups: [""]
resources:
- pods
- nodes
- namespaces
verbs:
- get
- apiGroups: [""]
resources:
- endpoints
- services
verbs:
# Used to discover service IPs for advertisement.
- watch
- list
# Used to discover Typhas.
- get
# Pod CIDR auto-detection on kubeadm needs access to config maps.
- apiGroups: [""]
resources:
- configmaps
verbs:
- get
- apiGroups: [""]
resources:
- nodes/status
verbs:
# Needed for clearing NodeNetworkUnavailable flag.
- patch
# Calico stores some configuration information in node annotations.
- update
# Watch for changes to Kubernetes NetworkPolicies.
- apiGroups: ["networking.k8s.io"]
resources:
- networkpolicies
verbs:
- watch
- list
# Used by Calico for policy information.
- apiGroups: [""]
resources:
- pods
- namespaces
- serviceaccounts
verbs:
- list
- watch
# The CNI plugin patches pods/status.
- apiGroups: [""]
resources:
- pods/status
verbs:
- patch
# Calico monitors various CRDs for config.
- apiGroups: ["crd.projectcalico.org"]
resources:
- globalfelixconfigs
- felixconfigurations
- bgppeers
- globalbgpconfigs
- bgpconfigurations
- ippools
- ipamblocks
- globalnetworkpolicies
- globalnetworksets
- networkpolicies
- networksets
- clusterinformations
- hostendpoints
- blockaffinities
verbs:
- get
- list
- watch
# Calico must create and update some CRDs on startup.
- apiGroups: ["crd.projectcalico.org"]
resources:
- ippools
- felixconfigurations
- clusterinformations
verbs:
- create
- update
# Calico stores some configuration information on the node.
- apiGroups: [""]
resources:
- nodes
verbs:
- get
- list
- watch
# These permissions are only requried for upgrade from v2.6, and can
# be removed after upgrade or on fresh installations.
- apiGroups: ["crd.projectcalico.org"]
resources:
- bgpconfigurations
- bgppeers
verbs:
- create
- update
---
# Flannel ClusterRole
# Pulled from https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel-rbac.yml
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: flannel
rules:
- apiGroups: [""]
resources:
- pods
verbs:
- get
- apiGroups: [""]
resources:
- nodes
verbs:
- list
- watch
- apiGroups: [""]
resources:
- nodes/status
verbs:
- patch
---
# Bind the flannel ClusterRole to the canal ServiceAccount.
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: canal-flannel
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: flannel
subjects:
- kind: ServiceAccount
name: canal
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: canal-calico
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: calico-node
subjects:
- kind: ServiceAccount
name: canal
namespace: kube-system

View File

@ -0,0 +1,6 @@
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: canal
namespace: kube-system

View File

@ -0,0 +1,80 @@
---
# The IPv4 cidr pool to create on startup if none exists. Pod IPs will be
# chosen from this range.
podCidr: "10.42.0.0/16"
flannel:
# kube-flannel image
image:
repository: rancher/hardened-flannel
tag: v0.13.0-rancher1-build20210223
# The interface used by canal for host <-> host communication.
# If left blank, then the interface is chosen using the node's
# default route.
iface: ""
# kube-flannel command arguments
args:
- "--ip-masq"
- "--kube-subnet-mgr"
# Backend for kube-flannel. Backend should not be changed
# at runtime.
backend: "vxlan"
calico:
# CNI installation image.
cniImage:
repository: rancher/hardened-calico
tag: v3.13.3-build20210223
# Canal node image.
nodeImage:
repository: rancher/hardened-calico
tag: v3.13.3-build20210223
# Flexvol Image.
flexvolImage:
repository: rancher/hardened-calico
tag: v3.13.3-build20210223
# Datastore type for canal. It can be either kuberentes or etcd.
datastoreType: kubernetes
# Wait for datastore to initialize.
waitForDatastore: true
# Configure route aggregation based on pod CIDR.
usePodCIDR: true
# Disable BGP routing.
networkingBackend: none
# Cluster type to identify the deployment type.
clusterType: "k8s,canal"
# Disable file logging so `kubectl logs` works.
disableFileLogging: true
# Disable IPv6 on Kubernetes.
felixIpv6Support: false
# Period, in seconds, at which felix re-applies all iptables state
felixIptablesRefreshInterval: 60
# iptables backend to use for felix, defaults to auto but can also be set to nft or legacy
felixIptablesBackend: auto
# Set Felix logging to "info".
felixLogSeverityScreen: info
# Enable felix healthcheck.
felixHealthEnabled: true
# Enable prometheus metrics
felixPrometheusMetricsEnabled: true
# Disable XDP Acceleration as we do not support it with our ubi7 base image
felixXDPEnabled: false
# Whether or not to masquerade traffic to destinations not within
# the pod network.
masquerade: true
# Set Felix endpoint to host default action to ACCEPT.
felixDefaultEndpointToHostAction: ACCEPT
# Configure the MTU to use.
vethuMTU: 1450
# Typha is disabled.
typhaServiceName: none
# Kubelet flex-volume-plugin-dir
flexVolumePluginDir: /var/lib/kubelet/volumeplugins
# calico inbound failsafe ports. Empty string means defaults. Use 'none' to disable failsafe if you have your own rules.
felixFailsafeInboundHostPorts: ""
# calico outbound failsafe ports. Empty string means defaults. Use 'none' to disable failsafe if you have your own rules.
felixFailsafeOutboundHostPorts: ""
global:
systemDefaultRegistry: ""

View File

@ -1,6 +1,23 @@
apiVersion: v1
entries:
rke2-canal:
- apiVersion: v1
appVersion: v3.13.3
created: "2021-04-29T00:16:16.864175502Z"
description: Install Canal Network Plugin.
digest: 6e444eda17a9470676e7b16d6e43c07d6c260de37baea9c704ea9a0d9f2f745f
home: https://www.projectcalico.org/
keywords:
- canal
maintainers:
- email: charts@rancher.com
name: Rancher Labs
name: rke2-canal
sources:
- https://github.com/rancher/rke2-charts
urls:
- assets/rke2-canal/rke2-canal-v3.13.300-build2021022303.tgz
version: v3.13.300-build2021022303
- apiVersion: v1
appVersion: v3.13.3
created: "2021-03-10T21:22:09.901795809Z"