Use a kubernetes service account if one is specified (#4039)

Author: ohrite

airflow/plugins/operators/pod_operator.py

@@ -16,6 +16,9 @@ def PodOperator(*args, **kwargs):
     if "secrets" in kwargs:
         kwargs["secrets"] = map(lambda d: Secret(**d), kwargs["secrets"])
 
+    if "SERVICE_ACCOUNT_NAME" in os.environ:
+        kwargs["service_account_name"] = os.environ.get("SERVICE_ACCOUNT_NAME")
+
     location = os.environ.get("POD_LOCATION")
     cluster_name = os.environ.get("POD_CLUSTER_NAME")
     project_id = os.environ.get("GOOGLE_CLOUD_PROJECT")

iac/cal-itp-data-infra-staging/composer/us/environment.tf

@@ -55,7 +55,7 @@ resource "google_composer_environment" "calitp-staging-composer" {
         "POD_LOCATION"                                         = "us-west2",
         "POD_CLUSTER_NAME"                                     = data.terraform_remote_state.gke.outputs.google_container_cluster_airflow-jobs-staging_name,
         "POD_SECRETS_NAMESPACE"                                = local.namespace,
-        "SERVICE_ACCOUNT_NAME"                                 = data.terraform_remote_state.iam.outputs.google_service_account_composer-service-account_email,
+        "SERVICE_ACCOUNT_NAME"                                 = local.service_account_name,
         "CALITP_BUCKET__AGGREGATOR_SCRAPER"                    = "gs://${data.terraform_remote_state.gcs.outputs.google_storage_bucket_calitp-staging-aggregator-scraper_name}",
         "CALITP_BUCKET__AIRTABLE"                              = "gs://${data.terraform_remote_state.gcs.outputs.google_storage_bucket_calitp-staging-airtable_name}",
         "CALITP_BUCKET__AMPLITUDE_BENEFITS_EVENTS"             = "gs://${data.terraform_remote_state.gcs.outputs.google_storage_bucket_calitp-staging-amplitude-benefits-events_name}",

iac/cal-itp-data-infra-staging/composer/us/kubernetes.tf

@@ -38,7 +38,7 @@ resource "kubernetes_priority_class" "dbt-high-priority" {
 
 resource "kubernetes_service_account" "composer-service-account" {
   metadata {
-    name      = "composer-service-account"
+    name      = local.service_account_name
     namespace = local.namespace
     annotations = {
       "iam.gke.io/gcp-service-account" = data.terraform_remote_state.iam.outputs.google_service_account_composer-service-account_email

iac/cal-itp-data-infra-staging/composer/us/variables.tf

@@ -1,6 +1,7 @@
 locals {
-  namespace = "airflow-jobs"
-  secret    = "jobs-data"
+  namespace            = "airflow-jobs"
+  secret               = "jobs-data"
+  service_account_name = "composer-service-account"
 
   # This regular expression corresponds to the Python package name specification
   # https://packaging.python.org/en/latest/specifications/name-normalization/