--- # Source: calico/templates/calico-kube-controllers.yaml # This manifest creates a Pod Disruption Budget for Controller to allow K8s Cluster Autoscaler to evict apiVersion: policy/v1 kind: PodDisruptionBudget metadata: name: calico-kube-controllers namespace: kube-system labels: k8s-app: calico-kube-controllers spec: maxUnavailable: 1 selector: matchLabels: k8s-app: calico-kube-controllers --- # Source: calico/templates/calico-kube-controllers.yaml apiVersion: v1 kind: ServiceAccount metadata: name: calico-kube-controllers namespace: kube-system --- # Source: calico/templates/calico-node.yaml apiVersion: v1 kind: ServiceAccount metadata: name: canal-node namespace: kube-system --- # Source: calico/templates/calico-etcd-secrets.yaml # The following contains k8s Secrets for use with a TLS enabled etcd cluster. # For information on populating Secrets, see http://kubernetes.io/docs/user-guide/secrets/ apiVersion: v1 kind: Secret type: Opaque metadata: name: calico-etcd-secrets namespace: kube-system data: # Populate the following with etcd TLS configuration if desired, but leave blank if # not using TLS for etcd. # The keys below should be uncommented and the values populated with the base64 # encoded contents of each file that would be associated with the TLS data. # Example command for encoding a file contents: cat | base64 -w 0 # etcd-key: null # etcd-cert: null # etcd-ca: null --- # Source: calico/templates/calico-config.yaml # This ConfigMap is used to configure a self-hosted Canal installation. kind: ConfigMap apiVersion: v1 metadata: name: canal-config namespace: kube-system data: # Configure this with the location of your etcd cluster. etcd_endpoints: "http://:" # If you're using TLS enabled etcd uncomment the following. # You must also populate the Secret below with these files. etcd_ca: "" # "/calico-secrets/etcd-ca" etcd_cert: "" # "/calico-secrets/etcd-cert" etcd_key: "" # "/calico-secrets/etcd-key" # Typha is disabled. typha_service_name: "none" # The interface used by canal for host <-> host communication. # If left blank, then the interface is chosen using the node's # default route. canal_iface: "" # Whether or not to masquerade traffic to destinations not within # the pod network. masquerade: "true" # Configure the MTU to use for workload interfaces and tunnels. # By default, MTU is auto-detected, and explicitly setting this field should not be required. # You can override auto-detection by providing a non-zero value. veth_mtu: "0" # The CNI network configuration to install on each node. The special # values in this config will be automatically populated. cni_network_config: |- { "name": "canal", "cniVersion": "0.3.1", "plugins": [ { "type": "flannel", "delegate": { "type": "calico", "include_default_routes": true, "etcd_endpoints": "__ETCD_ENDPOINTS__", "etcd_key_file": "__ETCD_KEY_FILE__", "etcd_cert_file": "__ETCD_CERT_FILE__", "etcd_ca_cert_file": "__ETCD_CA_CERT_FILE__", "log_level": "info", "log_file_path": "/var/log/calico/cni/cni.log", "policy": { "type": "k8s", "k8s_api_root": "https://__KUBERNETES_SERVICE_HOST__:__KUBERNETES_SERVICE_PORT__", "k8s_auth_token": "__SERVICEACCOUNT_TOKEN__" }, "kubernetes": { "kubeconfig": "__KUBECONFIG_FILEPATH__" } } }, { "type": "portmap", "capabilities": {"portMappings": true}, "snat": true } ] } --- # Source: calico/templates/calico-kube-controllers-rbac.yaml # Include a clusterrole for the kube-controllers component, # and bind it to the calico-kube-controllers serviceaccount. kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: calico-kube-controllers rules: # Pods are monitored for changing labels. # The node controller monitors Kubernetes nodes. # Namespace and serviceaccount labels are used for policy. - apiGroups: [""] resources: - pods - nodes - namespaces - serviceaccounts verbs: - watch - list - get # Watch for changes to Kubernetes NetworkPolicies. - apiGroups: ["networking.k8s.io"] resources: - networkpolicies verbs: - watch - list --- # Source: calico/templates/calico-node-rbac.yaml # Include a clusterrole for the calico-node DaemonSet, # and bind it to the calico-node serviceaccount. kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: calico-node rules: # Used for creating service account tokens to be used by the CNI plugin - apiGroups: [""] resources: - serviceaccounts/token resourceNames: - canal verbs: - create # The CNI plugin needs to get pods, nodes, and namespaces. - apiGroups: [""] resources: - pods - nodes - namespaces verbs: - get # EndpointSlices are used for Service-based network policy rule # enforcement. - apiGroups: ["discovery.k8s.io"] resources: - endpointslices verbs: - watch - list - apiGroups: [""] resources: - endpoints - services verbs: # Used to discover service IPs for advertisement. - watch - list # Pod CIDR auto-detection on kubeadm needs access to config maps. - apiGroups: [""] resources: - configmaps verbs: - get - apiGroups: [""] resources: - nodes/status verbs: # Needed for clearing NodeNetworkUnavailable flag. - patch --- # Source: calico/templates/calico-node-rbac.yaml # Flannel ClusterRole # Pulled from https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel-rbac.yml kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: flannel rules: - apiGroups: [""] resources: - pods verbs: - get - apiGroups: [""] resources: - nodes verbs: - list - watch - apiGroups: [""] resources: - nodes/status verbs: - patch --- # Source: calico/templates/calico-node.yaml kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: canal rules: - apiGroups: [""] resources: - pods - nodes - namespaces verbs: - get # Pod CIDR auto-detection on kubeadm needs access to config maps. - apiGroups: [""] resources: - configmaps verbs: - get --- # Source: calico/templates/calico-kube-controllers-rbac.yaml kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: calico-kube-controllers roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: calico-kube-controllers subjects: - kind: ServiceAccount name: calico-kube-controllers namespace: kube-system --- # Source: calico/templates/calico-node-rbac.yaml # Bind the flannel ClusterRole to the canal ServiceAccount. kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: canal-flannel roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: flannel subjects: - kind: ServiceAccount name: canal namespace: kube-system --- # Source: calico/templates/calico-node-rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: canal-calico roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: calico-node subjects: - kind: ServiceAccount name: canal namespace: kube-system --- # Source: calico/templates/calico-node.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: canal roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: canal subjects: - kind: ServiceAccount name: canal-node namespace: kube-system --- # Source: calico/templates/calico-node.yaml # This manifest installs the canal-node container, as well # as the CNI plugins and network config on # each master and worker node in a Kubernetes cluster. kind: DaemonSet apiVersion: apps/v1 metadata: name: canal-node namespace: kube-system labels: k8s-app: canal-node spec: selector: matchLabels: k8s-app: canal-node updateStrategy: type: RollingUpdate rollingUpdate: maxUnavailable: 1 template: metadata: labels: k8s-app: canal-node spec: nodeSelector: kubernetes.io/os: linux hostNetwork: true tolerations: # Make sure canal-node gets scheduled on all nodes. - effect: NoSchedule operator: Exists # Mark the pod as a critical add-on for rescheduling. - key: CriticalAddonsOnly operator: Exists - effect: NoExecute operator: Exists serviceAccountName: canal-node # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. terminationGracePeriodSeconds: 0 priorityClassName: system-node-critical initContainers: # This container installs the CNI binaries # and CNI network config file on each node. - name: install-cni image: docker.io/calico/cni:master imagePullPolicy: IfNotPresent command: ["/opt/cni/bin/install"] envFrom: - configMapRef: # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. name: kubernetes-services-endpoint optional: true env: # Set the serviceaccount name to use for the Calico CNI plugin. # We use canal-node instead of calico-node when using flannel networking. - name: CALICO_CNI_SERVICE_ACCOUNT valueFrom: fieldRef: fieldPath: spec.serviceAccountName # Name of the CNI config file to create. - name: CNI_CONF_NAME value: "10-canal.conflist" # The CNI network config to install on each node. - name: CNI_NETWORK_CONFIG valueFrom: configMapKeyRef: name: canal-config key: cni_network_config # The location of the etcd cluster. - name: ETCD_ENDPOINTS valueFrom: configMapKeyRef: name: canal-config key: etcd_endpoints # Location of the CA certificate for etcd. - name: ETCD_CA_CERT_FILE valueFrom: configMapKeyRef: name: canal-config key: etcd_ca # Location of the client key for etcd. - name: ETCD_KEY_FILE valueFrom: configMapKeyRef: name: canal-config key: etcd_key # Location of the client certificate for etcd. - name: ETCD_CERT_FILE valueFrom: configMapKeyRef: name: canal-config key: etcd_cert # CNI MTU Config variable - name: CNI_MTU valueFrom: configMapKeyRef: name: canal-config key: veth_mtu # Prevents the container from sleeping forever. - name: SLEEP value: "false" volumeMounts: - mountPath: /host/opt/cni/bin name: cni-bin-dir - mountPath: /host/etc/cni/net.d name: cni-net-dir - mountPath: /calico-secrets name: etcd-certs securityContext: privileged: true # This init container mounts the necessary filesystems needed by the BPF data plane # i.e. bpf at /sys/fs/bpf and cgroup2 at /run/calico/cgroup. Calico-node initialisation is executed # in best effort fashion, i.e. no failure for errors, to not disrupt pod creation in iptable mode. - name: "mount-bpffs" image: docker.io/calico/node:master imagePullPolicy: IfNotPresent command: ["calico-node", "-init", "-best-effort"] volumeMounts: - mountPath: /sys/fs name: sys-fs # Bidirectional is required to ensure that the new mount we make at /sys/fs/bpf propagates to the host # so that it outlives the init container. mountPropagation: Bidirectional - mountPath: /var/run/calico name: var-run-calico # Bidirectional is required to ensure that the new mount we make at /run/calico/cgroup propagates to the host # so that it outlives the init container. mountPropagation: Bidirectional # Mount /proc/ from host which usually is an init program at /nodeproc. It's needed by mountns binary, # executed by calico-node, to mount root cgroup2 fs at /run/calico/cgroup to attach CTLB programs correctly. - mountPath: /nodeproc name: nodeproc readOnly: true securityContext: privileged: true containers: # Runs canal-node container on each Kubernetes node. This # container programs network policy and routes on each # host. - name: calico-node image: docker.io/calico/node:master imagePullPolicy: IfNotPresent envFrom: - configMapRef: # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. name: kubernetes-services-endpoint optional: true env: # The location of the etcd cluster. - name: ETCD_ENDPOINTS valueFrom: configMapKeyRef: name: canal-config key: etcd_endpoints # Location of the CA certificate for etcd. - name: ETCD_CA_CERT_FILE valueFrom: configMapKeyRef: name: canal-config key: etcd_ca # Location of the client key for etcd. - name: ETCD_KEY_FILE valueFrom: configMapKeyRef: name: canal-config key: etcd_key # Location of the client certificate for etcd. - name: ETCD_CERT_FILE valueFrom: configMapKeyRef: name: canal-config key: etcd_cert # Set noderef for node controller. - name: CALICO_K8S_NODE_REF valueFrom: fieldRef: fieldPath: spec.nodeName # Set the serviceaccount name to use for the Calico CNI plugin. # We use canal-node instead of calico-node when using flannel networking. - name: CALICO_CNI_SERVICE_ACCOUNT valueFrom: fieldRef: fieldPath: spec.serviceAccountName # Don't enable BGP. - name: CALICO_NETWORKING_BACKEND value: "none" # Cluster type to identify the deployment type - name: CLUSTER_TYPE value: "k8s,canal" # Period, in seconds, at which felix re-applies all iptables state - name: FELIX_IPTABLESREFRESHINTERVAL value: "60" # No IP address needed. - name: IP value: "" # The default IPv4 pool to create on startup if none exists. Pod IPs will be # chosen from this range. Changing this value after installation will have # no effect. This should fall within `--cluster-cidr`. # - name: CALICO_IPV4POOL_CIDR # value: "192.168.0.0/16" # Disable file logging so `kubectl logs` works. - name: CALICO_DISABLE_FILE_LOGGING value: "true" # Set Felix endpoint to host default action to ACCEPT. - name: FELIX_DEFAULTENDPOINTTOHOSTACTION value: "ACCEPT" # Disable IPv6 on Kubernetes. - name: FELIX_IPV6SUPPORT value: "false" - name: FELIX_HEALTHENABLED value: "true" securityContext: privileged: true resources: requests: cpu: 250m lifecycle: preStop: exec: command: - /bin/calico-node - -shutdown livenessProbe: exec: command: - /bin/calico-node - -felix-live periodSeconds: 10 initialDelaySeconds: 10 failureThreshold: 6 timeoutSeconds: 10 readinessProbe: httpGet: path: /readiness port: 9099 host: localhost periodSeconds: 10 timeoutSeconds: 10 volumeMounts: # For maintaining CNI plugin API credentials. - mountPath: /host/etc/cni/net.d name: cni-net-dir readOnly: false - mountPath: /lib/modules name: lib-modules readOnly: true - mountPath: /run/xtables.lock name: xtables-lock readOnly: false - mountPath: /var/run/calico name: var-run-calico readOnly: false - mountPath: /var/lib/calico name: var-lib-calico readOnly: false - mountPath: /calico-secrets name: etcd-certs - name: policysync mountPath: /var/run/nodeagent # For eBPF mode, we need to be able to mount the BPF filesystem at /sys/fs/bpf so we mount in the # parent directory. - name: bpffs mountPath: /sys/fs/bpf - name: cni-log-dir mountPath: /var/log/calico/cni readOnly: true # Runs the flannel daemon to enable vxlan networking between # container hosts. - name: flannel image: docker.io/flannelcni/flannel:v0.16.3 imagePullPolicy: IfNotPresent env: # The location of the etcd cluster. - name: FLANNELD_ETCD_ENDPOINTS valueFrom: configMapKeyRef: name: canal-config key: etcd_endpoints # Location of the CA certificate for etcd. - name: ETCD_CA_CERT_FILE valueFrom: configMapKeyRef: name: canal-config key: etcd_ca # Location of the client key for etcd. - name: ETCD_KEY_FILE valueFrom: configMapKeyRef: name: canal-config key: etcd_key # Location of the client certificate for etcd. - name: ETCD_CERT_FILE valueFrom: configMapKeyRef: name: canal-config key: etcd_cert # Location of the CA certificate for etcd. - name: FLANNELD_ETCD_CAFILE valueFrom: configMapKeyRef: name: canal-config key: etcd_ca # Location of the client key for etcd. - name: FLANNELD_ETCD_KEYFILE valueFrom: configMapKeyRef: name: canal-config key: etcd_key # Location of the client certificate for etcd. - name: FLANNELD_ETCD_CERTFILE valueFrom: configMapKeyRef: name: canal-config key: etcd_cert # The interface flannel should run on. - name: FLANNELD_IFACE valueFrom: configMapKeyRef: name: canal-config key: canal_iface # Perform masquerade on traffic leaving the pod cidr. - name: FLANNELD_IP_MASQ valueFrom: configMapKeyRef: name: canal-config key: masquerade # Write the subnet.env file to the mounted directory. - name: FLANNELD_SUBNET_FILE value: "/run/flannel/subnet.env" securityContext: privileged: true volumeMounts: - mountPath: /etc/resolv.conf name: resolv - mountPath: /run/flannel name: run-flannel - mountPath: /calico-secrets name: etcd-certs volumes: # Used by canal-node. - name: lib-modules hostPath: path: /lib/modules - name: var-run-calico hostPath: path: /var/run/calico - name: var-lib-calico hostPath: path: /var/lib/calico - name: xtables-lock hostPath: path: /run/xtables.lock type: FileOrCreate - name: sys-fs hostPath: path: /sys/fs/ type: DirectoryOrCreate - name: bpffs hostPath: path: /sys/fs/bpf type: Directory # mount /proc at /nodeproc to be used by mount-bpffs initContainer to mount root cgroup2 fs. - name: nodeproc hostPath: path: /proc # Used by flannel. - name: run-flannel hostPath: path: /run/flannel - name: resolv hostPath: path: /etc/resolv.conf # Used to install CNI. - name: cni-bin-dir hostPath: path: /opt/cni/bin - name: cni-net-dir hostPath: path: /etc/cni/net.d # Used to access CNI logs. - name: cni-log-dir hostPath: path: /var/log/calico/cni # Mount in the etcd TLS secrets with mode 400. # See https://kubernetes.io/docs/concepts/configuration/secret/ - name: etcd-certs secret: secretName: calico-etcd-secrets defaultMode: 0400 # Used to create per-pod Unix Domain Sockets - name: policysync hostPath: type: DirectoryOrCreate path: /var/run/nodeagent --- # Source: calico/templates/calico-kube-controllers.yaml # See https://github.com/projectcalico/kube-controllers apiVersion: apps/v1 kind: Deployment metadata: name: calico-kube-controllers namespace: kube-system labels: k8s-app: calico-kube-controllers spec: # The controllers can only have a single active instance. replicas: 1 selector: matchLabels: k8s-app: calico-kube-controllers strategy: type: Recreate template: metadata: name: calico-kube-controllers namespace: kube-system labels: k8s-app: calico-kube-controllers spec: nodeSelector: kubernetes.io/os: linux tolerations: # Mark the pod as a critical add-on for rescheduling. - key: CriticalAddonsOnly operator: Exists - key: node-role.kubernetes.io/master effect: NoSchedule - key: node-role.kubernetes.io/control-plane effect: NoSchedule serviceAccountName: calico-kube-controllers priorityClassName: system-cluster-critical # The controllers must run in the host network namespace so that # it isn't governed by policy that would prevent it from working. hostNetwork: true containers: - name: calico-kube-controllers image: docker.io/calico/kube-controllers:master imagePullPolicy: IfNotPresent env: # The location of the etcd cluster. - name: ETCD_ENDPOINTS valueFrom: configMapKeyRef: name: canal-config key: etcd_endpoints # Location of the CA certificate for etcd. - name: ETCD_CA_CERT_FILE valueFrom: configMapKeyRef: name: canal-config key: etcd_ca # Location of the client key for etcd. - name: ETCD_KEY_FILE valueFrom: configMapKeyRef: name: canal-config key: etcd_key # Location of the client certificate for etcd. - name: ETCD_CERT_FILE valueFrom: configMapKeyRef: name: canal-config key: etcd_cert # Choose which controllers to run. - name: ENABLED_CONTROLLERS value: policy,namespace,serviceaccount,workloadendpoint,node volumeMounts: # Mount in the etcd TLS secrets. - mountPath: /calico-secrets name: etcd-certs livenessProbe: exec: command: - /usr/bin/check-status - -l periodSeconds: 10 initialDelaySeconds: 10 failureThreshold: 6 timeoutSeconds: 10 readinessProbe: exec: command: - /usr/bin/check-status - -r periodSeconds: 10 volumes: # Mount in the etcd TLS secrets with mode 400. # See https://kubernetes.io/docs/concepts/configuration/secret/ - name: etcd-certs secret: secretName: calico-etcd-secrets defaultMode: 0440 --- # Source: calico/templates/configure-canal.yaml # This manifest deploys a Job which performs one time # configuration of Canal. apiVersion: batch/v1 kind: Job metadata: name: configure-canal namespace: kube-system labels: k8s-app: canal spec: template: metadata: name: configure-canal spec: nodeSelector: kubernetes.io/os: linux hostNetwork: true restartPolicy: OnFailure containers: # Writes basic flannel configuration to etcd. - name: configure-flannel image: quay.io/coreos/etcd:v3.5.1 command: - "etcdctl" # Uncomment the following lines if using TLS to etcd. # - "--cert-file=/calico-secrets/etcd-cert" # - "--key-file=/calico-secrets/etcd-key" # - "--ca-file=/calico-secrets/etcd-ca" - "--no-sync" - "set" - "/coreos.com/network/config" - '{ "Network": "10.244.0.0/16", "Backend": {"Type": "vxlan"} }' env: # The location of the etcd cluster. - name: ETCDCTL_PEERS valueFrom: configMapKeyRef: name: canal-config key: etcd_endpoints # The location of the Calico etcd cluster. - name: ETCDCTL_CACERT valueFrom: configMapKeyRef: name: canal-config key: etcd_ca volumeMounts: # Mount in the etcd TLS secrets. - mountPath: /calico-secrets name: etcd-certs volumes: # Mount in the etcd TLS secrets. - name: etcd-certs secret: secretName: calico-etcd-secrets