NHSDigital · Sourceless · Mar 22, 2021 · Mar 17, 2021 · Mar 22, 2021
diff --git a/.gitignore b/.gitignore
@@ -27,6 +27,7 @@ share/python-wheels/
 *.egg
 MANIFEST
 .venv/
+*.orig
 
 # PyInstaller
 #  Usually these files are written by a python script from a template
@@ -148,4 +149,4 @@ env
 .vscode
 
 # ansible testing
-ansible/collections/ansible_collections/nhsd/apigee/tests/output/
+ansible/collections/ansible_collections/nhsd/apigee/tests/output/
diff --git a/ansible/Makefile b/ansible/Makefile
@@ -82,4 +82,7 @@ remove-old-pr-proxies:
 	@poetry run ansible-playbook -i local remove-old-pr-proxies.yml
 
 deploy-api-pipelines: guard-PREFIX guard-REPO guard-SERVICE_CONNECTION_ID
-	PREFIX="$(PREFIX)" ORG="NHSD-APIM" PROJECT="API Platform" REPO="https://github.com/NHSDigital/$(REPO)" SERVICE_CONNECTION="$(SERVICE_CONNECTION_ID)" poetry run ansible-playbook ansible/deploy-api-pipelines.yml
+	PREFIX="$(PREFIX)" ORG="NHSD-APIM" PROJECT="API Platform" REPO="https://github.com/NHSDigital/$(REPO)" SERVICE_CONNECTION="$(SERVICE_CONNECTION_ID)" poetry run ansible-playbook ansible/deploy-api-pipelines.yml
+
+get-developer-emails:
+	@poetry run ansible-playbook -i local get-developer-emails.yml
diff --git a/ansible/filter_plugins/apigee_helpers.py b/ansible/filter_plugins/apigee_helpers.py
@@ -73,6 +73,75 @@ def apigee_remove_proxy_from_product(product: dict, proxy_to_remove):
     return product
 
 
+def apigee_teams_map(teams: List[dict]):
+
+    return {
+        f"{team['id']}@devteam.apigee.io": {
+            "contact": team['pointOfContact'],
+            "members": list(mem['userId'] for mem in team.get('memberships', []))
+        }
+        for team in teams
+    }
+
+
+def apigee_teams_to_point_of_contact(teams: List[dict]):
+
+    return {
+        f"{team['id']}@devteam.apigee.io": team['pointOfContact'] for team in teams
+    }
+
+
+def apigee_teams_to_members(teams: List[dict]):
+
+    return {
+        f"{team['id']}@devteam.apigee.io": list(mem['userId'] for mem in team.get('memberships', [])) for team in teams
+    }
+
+
+def apigee_product_developers(
+    product_app_map: dict, dev_id_to_email: dict, teams_map: dict, product_filter: str = None
+):
+    if not product_app_map:
+        raise ValueError('product_app_map not set')
+
+    if not dev_id_to_email:
+        raise ValueError('dev_id_to_email not set')
+
+    teams_map = teams_map or {}
+    result = {}
+    for product_name, apps in product_app_map.items():
+        if product_filter and not re.match(product_filter, product_name):
+            continue
+        if product_name not in result:
+            result[product_name] = []
+
+        for app in apps:
+            developer = dev_id_to_email[app["developerId"]]
+
+            team = teams_map.get(developer, {})
+
+            entry = {
+                "app": app["appName"],
+                "developer": developer
+            }
+
+            if team:
+                entry['developer'] = team['contact']
+                entry['team'] = team['members']
+
+            result[product_name].append(entry)
+
+    result = {
+        key: sorted(apps, key=lambda x: x['app']) for key, apps in result.items()
+    }
+
+    return result
+
+    # return {
+    #     f"{team['id']}@devteam.apigee.io": list(mem['userId'] for mem in team.get('memberships', [])) for team in teams
+    # }
+
+
 class FilterModule:
 
     @staticmethod
@@ -81,5 +150,9 @@ def filters():
             # jinja2 overrides
             'apigee_apps_to_product_map': apigee_apps_to_product_map,
             'apigee_products_to_api_map': apigee_products_to_api_map,
-            'apigee_remove_proxy_from_product': apigee_remove_proxy_from_product
-        }
+            'apigee_remove_proxy_from_product': apigee_remove_proxy_from_product,
+            'apigee_teams_to_point_of_contact': apigee_teams_to_point_of_contact,
+            'apigee_teams_to_members': apigee_teams_to_members,
+            'apigee_teams_map': apigee_teams_map,
+            'apigee_product_developers': apigee_product_developers
+        }
diff --git a/ansible/get-developer-emails.yml b/ansible/get-developer-emails.yml
@@ -0,0 +1,22 @@
+- name: remove old pr portal apis
+  hosts: 127.0.0.1
+  connection: local
+  gather_facts: yes
+
+  vars:
+    APIGEE_ORGANIZATION: "{{ lookup('env', 'APIGEE_ORGANIZATION') }}"
+    APIGEE_ACCESS_TOKEN: "{{ lookup('env', 'APIGEE_ACCESS_TOKEN') }}"
+
+  pre_tasks:
+    - name: check APIGEE_ORGANIZATION
+      fail:
+        msg: "APIGEE_ORGANIZATION not set"
+      when: not APIGEE_ORGANIZATION
+
+    - name: check APIGEE_ACCESS_TOKEN
+      fail:
+        msg: "APIGEE_ACCESS_TOKEN not set"
+      when: not APIGEE_ACCESS_TOKEN
+
+  roles:
+    - get-developer-emails
diff --git a/ansible/roles/create-api-deployment-pre-reqs/templates/terraform/iam.tf b/ansible/roles/create-api-deployment-pre-reqs/templates/terraform/iam.tf
@@ -24,6 +24,29 @@ data "aws_iam_policy_document" "ecs-execution-role" {
     ]
   }
 
+  statement {
+    actions = [
+      "secretsmanager:GetSecretValue"
+    ]
+
+    resources = [
+      "arn:aws:secretsmanager:${local.region}:${local.account_id}:secret:${var.account}/*"
+    ]
+
+    condition {
+      test = "ForAnyValue:StringLike"
+      values = [
+        "${var.service_id}",
+        "${var.service_id} *",
+        "* ${var.service_id} *",
+        "* ${var.service_id}",
+        "all"
+      ]
+      variable = "secretsmanager:ResourceTag/AllowedServices1"
+    }
+
+  }
+
   statement {
 
     actions = [
@@ -66,6 +89,8 @@ resource "aws_iam_role" "ecs-execution-role" {
   tags = {
     Name   = "ecs-x-${local.env_service_id}"
     source = "terraform"
+    api-service = var.service_id
+    api-environment = var.apigee_environment
   }
 }
 
@@ -108,6 +133,12 @@ data "aws_iam_policy_document" "deploy-user-assume-role" {
 resource "aws_iam_role" "deploy-user" {
   name = "deploy-${local.env_service_id}"
   assume_role_policy = data.aws_iam_policy_document.deploy-user-assume-role.json
+
+  tags = {
+    source = "terraform"
+    api-service = var.service_id
+    api-environment = var.apigee_environment
+  }
 }
 
 resource "aws_iam_role_policy_attachment" "deploy-user" {
@@ -202,8 +233,7 @@ data "aws_iam_policy_document" "deploy-user" {
 
     condition {
       test = "StringEquals"
-      values = [
-      var.service_id]
+      values = [var.service_id]
       variable = "aws:RequestTag/api-service"
     }
 

diff --git a/ansible/roles/create-api-deployment-pre-reqs/templates/terraform/sg.tf b/ansible/roles/create-api-deployment-pre-reqs/templates/terraform/sg.tf
@@ -8,5 +8,7 @@ resource "aws_security_group" "api-deployment" {
   tags = {
     Name   = "api-${local.env_service_id}"
     source = "terraform"
+    api-service = var.service_id
+    api-environment = var.apigee_environment
   }
 }
diff --git a/ansible/roles/get-developer-emails/tasks/main.yml b/ansible/roles/get-developer-emails/tasks/main.yml
@@ -0,0 +1,74 @@
+
+- name: "get developers"
+  uri:
+    url: "{{ developers_uri }}?expand=true"
+    headers:
+      Authorization: "Bearer {{ APIGEE_ACCESS_TOKEN }}"
+    return_content: yes
+  register: get_developers
+
+- name: "map developers"
+  set_fact:
+    dev_id_to_email: "{{ get_developers.json.developer | dict_list_to_map('developerId', 'email') }}"
+
+- name: get products
+  uri:
+    url: "{{ products_uri }}"
+    headers:
+      Authorization: "Bearer {{ APIGEE_ACCESS_TOKEN }}"
+    return_content: yes
+  register: products
+
+- name: list portals
+  uri:
+    url: "{{ portals_base_uri }}?orgname={{ APIGEE_ORGANIZATION }}"
+    headers:
+      Authorization: "Bearer {{ APIGEE_ACCESS_TOKEN }}"
+    return_content: yes
+  register: portals
+
+- name: set portal_id
+  set_fact:
+    portal_id: "{{ portals.json.data[0].id }}"
+
+- name: get audience
+  uri:
+    url: "https://apigee.com/portals/api/sites/{{ portal_id }}/audiencesenabled"
+    headers:
+      Authorization: "Bearer {{ APIGEE_ACCESS_TOKEN }}"
+    return_content: yes
+  register: get_audiences
+
+- name: set audience_id
+  set_fact:
+    audience_id: "{{ get_audiences.json.data.zmsId }}"
+
+- name: get teams
+  uri:
+    url: "https://apigee.com/consumers/api/providers/{{ audience_id }}/teams"
+    headers:
+      Authorization: "Bearer {{ APIGEE_ACCESS_TOKEN }}"
+    return_content: yes
+  register: get_teams
+
+- name: setup developer teams map
+  set_fact:
+    teams_map: "{{ get_teams.json.data | apigee_teams_map }}"
+
+- name: "get apps"
+  uri:
+    url: "{{ apps_uri }}?expand=true"
+    headers:
+      Authorization: "Bearer {{ APIGEE_ACCESS_TOKEN }}"
+    return_content: yes
+  register: get_apps
+
+- name: create product map
+  set_fact:
+    product_app_map: "{{ get_apps.json.app | apigee_apps_to_product_map(product_filter=product_filter) }}"
+
+- name: get product developers
+  set_fact:
+    product_developers: "{{ product_app_map | apigee_product_developers(dev_id_to_email=dev_id_to_email, teams_map=teams_map) }}"
+
+- debug: var=product_developers
diff --git a/ansible/roles/get-developer-emails/tasks/output-emails.yml b/ansible/roles/get-developer-emails/tasks/output-emails.yml
@@ -0,0 +1,7 @@
+
+
+- debug: msg="{{ product_slug }} - {{ item.appName}} - {{ dev_id_to_email[item.developerId] }} - {{ apigee_teams_to_members[dev_id_to_email[item.developerId]] | default(dev_id_to_email[item.developerId]) }}"
+  loop: "{{ product_app_map.get(product_slug, []) }}"
+  loop_control:
+    label: "{{ product_slug }} - {{ item.appName }} - {{ item.appId }}"
+
diff --git a/ansible/roles/get-developer-emails/vars/main.yml b/ansible/roles/get-developer-emails/vars/main.yml
@@ -0,0 +1,16 @@
+org_uri: "https://api.enterprise.apigee.com/v1/organizations/{{ APIGEE_ORGANIZATION }}"
+products_uri: "{{ org_uri }}/apiproducts"
+apps_uri: "{{ org_uri }}/apps"
+developers_uri: "{{ org_uri }}/developers"
+retain_hours:  "{{ (lookup('env', 'retain_hours') or 72) }}"
+product_filter: "{{ lookup('env', 'PRODUCT_FILTER') }}"
+portals_base_uri: "https://apigee.com/portals/api/sites"
+
+#      https://apigee.com/portals/api/sites/nhsd-nonprod-developerportal/audiencesenabled
+#
+#  nhsd-nonprod/portals/nhsd-nonprod-developerportal/accounts/teams
+#https://apigee.com/consumers/api/providers/4ac22135-49bf-4e7e-b79b-93ad2ceb3aa3/teams
+#
+#  https://apigee.com/organizations/nhsd-nonprod/portals/nhsd-nonprod-developerportal/accounts/teams
+#  https://apigee.com/portals/api/sites/nhsd-nonprod-developerportal/accounts/teams
+#                    "{{ portals_base_uri }}/{{ portal_id }}/apidocs"