Propagate NodeSelector and Tolerations to Affinity Assistant

To be able to run Tekton workload on a subset of the Nodes in a cluster,
a NodeSelector or Tolerations can be set on the pods, e.g. by specifying a
custom PodTemplate.

When using the Affinity Assistant, this was not supported to customize.

This commit, also propagate NodeSelector and Tolerations to the Affinity Assistant pod,
so that Tekton workload can be kept to a subset of the Nodes in a cluster, also when
using the Affinity Assistant.

Fixes #2742
/kind feature

Author: jlpettersson

docs/install.md

@@ -245,7 +245,7 @@ The example below customizes the following:
 - the default timeout from 60 minutes to 20 minutes.
 - the default `app.kubernetes.io/managed-by` label is applied to all Pods created to execute `TaskRuns`.
 - the default Pod template to include a node selector to select the node where the Pod will be scheduled by default.
-  For more information, see [`PodTemplate` in `TaskRuns`](./taskruns.md#pod-template) or [`PodTemplate` in `PipelineRuns`](./pipelineruns.md#pod-template).
+  For more information, see [`PodTemplate` in `TaskRuns`](./taskruns.md#specifying-a-pod-template) or [`PodTemplate` in `PipelineRuns`](./pipelineruns.md#specifying-a-pod-template).
 
 ```yaml
 apiVersion: v1
@@ -270,7 +270,7 @@ To customize the behavior of the Pipelines Controller, modify the ConfigMap `fea
 
 - `disable-affinity-assistant` - set this flag to disable the [Affinity Assistant](./workspaces.md#affinity-assistant-and-specifying-workspace-order-in-a-pipeline)
   that is used to provide Node Affinity for `TaskRun` pods that share workspace volume. 
-  The Affinity Assistant pods may be incompatible with NodeSelector and other affinity rules
+  The Affinity Assistant is incompatible with other affinity rules
   configured for `TaskRun` pods.
 
   **Note:** Affinity Assistant use [Inter-pod affinity and anti-affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity)

docs/workspaces.md

@@ -215,7 +215,7 @@ When a `PersistentVolumeClaim` is used as volume source for a `Workspace` in a `
 an Affinity Assistant will be created. The Affinity Assistant acts as a placeholder for `TaskRun` pods
 sharing the same `Workspace`. All `TaskRun` pods within the `PipelineRun` that share the `Workspace`
 will be scheduled to the same Node as the Affinity Assistant pod. This means that Affinity Assistant is incompatible
-with e.g. NodeSelectors or other affinity rules configured for the `TaskRun` pods. The Affinity Assistant
+with e.g. other affinity rules configured for the `TaskRun` pods. The Affinity Assistant
 is deleted when the `PipelineRun` is completed. The Affinity Assistant can be disabled by setting the
 [disable-affinity-assistant](install.md#customizing-basic-execution-parameters) feature gate.
 

pkg/reconciler/pipelinerun/affinity_assistant.go

@@ -117,6 +117,18 @@ func affinityAssistantStatefulSet(name string, pr *v1beta1.PipelineRun, claimNam
 	// We want a singleton pod
 	replicas := int32(1)
 
+	// use tolerations from default podTemplate if specified
+	var tolerations []corev1.Toleration
+	if pr.Spec.PodTemplate != nil {
+		tolerations = pr.Spec.PodTemplate.Tolerations
+	}
+
+	// use nodeSelector from default podTemplate if specified
+	var nodeSelector map[string]string
+	if pr.Spec.PodTemplate != nil {
+		nodeSelector = pr.Spec.PodTemplate.NodeSelector
+	}
+
 	containers := []corev1.Container{{
 		Name: "affinity-assistant",
 
@@ -171,7 +183,9 @@ func affinityAssistantStatefulSet(name string, pr *v1beta1.PipelineRun, claimNam
 					Labels: getStatefulSetLabels(pr, name),
 				},
 				Spec: corev1.PodSpec{
-					Containers: containers,
+					Containers:   containers,
+					Tolerations:  tolerations,
+					NodeSelector: nodeSelector,
 					Affinity: &corev1.Affinity{
 						PodAntiAffinity: &corev1.PodAntiAffinity{
 							PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{repelOtherAffinityAssistantsPodAffinityTerm},

pkg/reconciler/pipelinerun/affinity_assistant_test.go

@@ -82,6 +82,55 @@ func TestCreateAndDeleteOfAffinityAssistant(t *testing.T) {
 	}
 }
 
+func TestThatTolerationsAndNodeSelectorIsPropagatedToAffinityAssistant(t *testing.T) {
+	prWithCustomPodTemplate := &v1beta1.PipelineRun{
+		TypeMeta: metav1.TypeMeta{Kind: "PipelineRun"},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "pipelinerun-with-custom-podtemplate",
+		},
+		Spec: v1beta1.PipelineRunSpec{
+			PodTemplate: &v1beta1.PodTemplate{
+				NodeSelector: map[string]string{
+					"disktype": "ssd",
+				},
+				Tolerations: []corev1.Toleration{{
+					Key:      "key",
+					Operator: "Equal",
+					Value:    "value",
+					Effect:   "NoSchedule",
+				}},
+			},
+		},
+	}
+
+	prWithoutCustomPodTemplate := &v1beta1.PipelineRun{
+		TypeMeta: metav1.TypeMeta{Kind: "PipelineRun"},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "pipelinerun-without-custom-podtemplate",
+		},
+		Spec: v1beta1.PipelineRunSpec{},
+	}
+
+	stsWithTolerationsAndNodeSelector := affinityAssistantStatefulSet("test-assistant", prWithCustomPodTemplate, "mypvc")
+	stsWithoutTolerationsAndNodeSelector := affinityAssistantStatefulSet("test-assistant", prWithoutCustomPodTemplate, "mypvc")
+
+	if len(stsWithTolerationsAndNodeSelector.Spec.Template.Spec.NodeSelector) != 1 {
+		t.Errorf("expected a NodeSelector in the StatefulSet")
+	}
+
+	if len(stsWithTolerationsAndNodeSelector.Spec.Template.Spec.Tolerations) != 1 {
+		t.Errorf("expected Tolerations in the StatefulSet")
+	}
+
+	if len(stsWithoutTolerationsAndNodeSelector.Spec.Template.Spec.NodeSelector) != 0 {
+		t.Errorf("unexpected NodeSelector in the StatefulSet")
+	}
+
+	if len(stsWithoutTolerationsAndNodeSelector.Spec.Template.Spec.Tolerations) != 0 {
+		t.Errorf("unexpected Tolerations in the StatefulSet")
+	}
+}
+
 func TestDisableAffinityAssistant(t *testing.T) {
 	for _, tc := range []struct {
 		description string