NHSDigital · matt-mercer · Feb 17, 2021 · Nov 16, 2020 · Nov 16, 2020 · Nov 16, 2020
diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml
@@ -0,0 +1,22 @@
+name: Test deployments
+
+on: [pull_request]
+
+jobs:
+  test_deployments:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout Scripts
+      uses: actions/checkout@v2
+
+    - name: Test Pull Request Deployments
+      env:
+        AZURE_TOKEN: "${{ secrets.AZURE_TOKEN }}"
+      run: |
+        export BRANCH_NAME="${{ github.event.pull_request.head.ref }}"
+        export NOTIFY_COMMIT_SHA="${{ github.event.pull_request.head.sha }}"
+        export UTILS_PR_NUMBER="${{ github.event.pull_request.number }}"
+        echo UTILS_PR_NUMBER=${UTILS_PR_NUMBER}
+        echo BRANCH_NAME=${BRANCH_NAME}
+        echo NOTIFY_COMMIT_SHA=${NOTIFY_COMMIT_SHA}
+        python3 scripts/test_pull_request_deployments.py
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -1,6 +1,6 @@
 name: Build
 
-on: push
+on: [pull_request]
 
 jobs:
   build:

diff --git a/README.md b/README.md
@@ -3,3 +3,8 @@ Scripts and utilities used across API managment platform and services
 
 ## Scripts
 * `template.py` - cli for basic jinja templating
+* `test_pull_request_deployments.py` - cli for testing utils against other repositories
+    * Environment Variables:
+        * `AZURE_TOKEN` - Azure Devops token.
+        * `NOTIFY_COMMIT_SHA` - Git Commit SHA that you want to report to.
+        * `UTILS_PR_NUMBER` - The utils pull request number e.g. '123'
diff --git a/ansible/Makefile b/ansible/Makefile
@@ -15,18 +15,23 @@ guard-%:
 	    exit 1; \
 	fi
 
+CHECK := $(if ${CHECK},--check , )
+VERBOSE := $(if ${VERBOSE},-vvv , )
 
 template-manifest: guard-DIST_DIR
-	@poetry run ansible-playbook -i local template-manifest.yml
+	@poetry run ansible-playbook $(VERBOSE) -i local template-manifest.yml
+
+add-jwks-resource-url-to-app: guard-APIGEE_ENVIRONMENT guard-APIGEE_ACCESS_TOKEN guard-APP_ID
+	@poetry run ansible-playbook $(VERBOSE) --diff $(CHECK) -i local add-jwks-resource-url-to-app.yml
 
 add-apim-guids-policy:
-	@poetry run ansible-playbook --diff -i local add-apim-guids-policy.yml
+	@poetry run ansible-playbook $(VERBOSE) --diff $(CHECK) -i local add-apim-guids-policy.yml
 
 validate-manifest: guard-DIST_DIR
-	@poetry run ansible-playbook -i local validate-manifest.yml
+	@poetry run ansible-playbook $(VERBOSE) -i local validate-manifest.yml
 
 deploy-manifest: guard-APIGEE_ENVIRONMENT guard-APIGEE_ORGANIZATION guard-APIGEE_ACCESS_TOKEN
-	@poetry run ansible-playbook --diff -i local deploy-manifest.yml
+	@poetry run ansible-playbook $(VERBOSE) --diff $(CHECK) -i local deploy-manifest.yml
 
 build-ecs-proxies: guard-build_label guard-service_id guard-CONTAINER_VARS_FILE
 	@poetry run ansible-playbook -i local build-ecs-proxies.yml

diff --git a/ansible/add-jwks-resource-url-to-app.yml b/ansible/add-jwks-resource-url-to-app.yml
@@ -0,0 +1,7 @@
+- name: add jwks_resource_url to app
+  hosts: 127.0.0.1
+  connection: local
+  gather_facts: no
+
+  roles:
+    - nhsd.apigee.add_jwks_resource_url_to_app
diff --git a/...ollections/ansible_collections/nhsd/apigee/plugins/action/add_jwks_resource_url_to_app.py b/...ollections/ansible_collections/nhsd/apigee/plugins/action/add_jwks_resource_url_to_app.py
@@ -0,0 +1,69 @@
+import copy
+from ansible_collections.nhsd.apigee.plugins.module_utils.models.ansible.add_jwks_resource_url import (
+    AddJwksResourceUrlToApp
+)
+from ansible_collections.nhsd.apigee.plugins.module_utils.apigee_action import (
+    ApigeeAction,
+)
+from ansible_collections.nhsd.apigee.plugins.module_utils import utils
+from ansible_collections.nhsd.apigee.plugins.module_utils import constants
+
+
+ATTRIBUTE_NAME = "jwks-resource-url"
+
+
+class ActionModule(ApigeeAction):
+    def run(self, tmp=None, task_vars=None):
+        super(ActionModule, self).run(tmp, task_vars)
+        args, errors = self.validate_args(AddJwksResourceUrlToApp)
+        if errors:
+            return errors
+
+        diff_mode = self._play_context.diff
+        check_mode = self._play_context.check_mode
+
+        before = args._app_data
+        after = copy.deepcopy(before)
+
+        jwks_attribute = {"name": ATTRIBUTE_NAME, "value": str(args.jwks_resource_url)}
+
+        # Delete any existing jwks attributes, for now there can only be one.
+        after["attributes"] = [attr for attr in after["attributes"] if attr["name"] != ATTRIBUTE_NAME]
+        # Append the desired jwks attributes and sort
+        after["attributes"].append(jwks_attribute)
+        after["attributes"] = sorted(after["attributes"], key=lambda attr: attr["name"])
+
+        delta = utils.delta(before, after)
+        result = {"changed": bool(delta), "app": after}
+
+        if diff_mode:
+            result["diff"] = [{"before": before, "after": after}]
+
+        if check_mode:
+            return result
+
+        developer_email = args._app_data["createdBy"]
+        app_name = args._app_data["name"]
+        app_attribute_url = (
+            constants.APIGEE_BASE_URL
+            + f"organizations/{args.organization}/developers/{developer_email}/apps/{app_name}/attributes"
+        )
+        app_data2 = utils.post(app_attribute_url, args.access_token,
+                               json={"attribute": after["attributes"]})
+        if app_data2.get("failed"):
+            return app_data2
+
+        app_url = (
+            constants.APIGEE_BASE_URL
+            + f"organizations/{args.organization}/apps/{args.app_id}"
+        )
+        updated_app_response = utils.get(app_url, args.access_token)
+        if updated_app_response.get("failed"):
+            return updated_app_response
+
+        after = updated_app_response["response"]["body"]
+        if diff_mode:
+            result["diff"][-1]["after"] = after
+
+        result["app"] = after
+        return result
diff --git a/ansible/collections/ansible_collections/nhsd/apigee/plugins/action/deploy_product.py b/ansible/collections/ansible_collections/nhsd/apigee/plugins/action/deploy_product.py
@@ -5,9 +5,9 @@
     DeployProduct,
 )
 from ansible_collections.nhsd.apigee.plugins.module_utils import utils
-
-
-APIGEE_BASE_URL = "https://api.enterprise.apigee.com/v1/"
+from ansible_collections.nhsd.apigee.plugins.module_utils.constants import (
+    APIGEE_BASE_URL,
+)
 
 
 class ActionModule(ApigeeAction):

diff --git a/ansible/collections/ansible_collections/nhsd/apigee/plugins/filter/filters.py b/ansible/collections/ansible_collections/nhsd/apigee/plugins/filter/filters.py
@@ -0,0 +1,21 @@
+from ansible_collections.nhsd.apigee.plugins.module_utils import constants
+
+
+def org_from_env(environment) -> str:
+    """Get nhsd apigee organization name from environment name."""
+    for org, envs in constants.APIGEE_ORG_TO_ENV.items():
+        if environment in envs:
+            return org
+    valid_envs = []
+    for v in constants.APIGEE_ORG_TO_ENV.values():
+        valid_envs = valid_envs + v
+    raise ValueError(f"Unknown environment {environment}, valid environments are {valid_envs}")
+
+
+class FilterModule:
+
+    @staticmethod
+    def filters():
+        return {
+            'org_from_env': org_from_env
+        }
diff --git a/ansible/collections/ansible_collections/nhsd/apigee/plugins/module_utils/constants.py b/ansible/collections/ansible_collections/nhsd/apigee/plugins/module_utils/constants.py
@@ -3,6 +3,17 @@
 APIGEE_BASE_URL = "https://api.enterprise.apigee.com/v1/"
 APIGEE_DAPI_URL = "https://apigee.com/dapi/api/"
 
+APIGEE_ORG_TO_ENV = {
+    "nhsd-nonprod": [
+        "internal-dev",
+        "internal-dev-sandbox",
+        "internal-qa",
+        "internal-qa-sandbox",
+        "ref",
+    ],
+    "nhsd-prod": ["dev", "int", "sandbox", "prod"],
+}
+
 
 def portal_uri(org: typing.Literal["nhsd-nonprod", "nhsd-prod"]) -> str:
     portal_ids = {

diff --git a/...ible_collections/nhsd/apigee/plugins/module_utils/models/ansible/add_jwks_resource_url.py b/...ible_collections/nhsd/apigee/plugins/module_utils/models/ansible/add_jwks_resource_url.py
@@ -0,0 +1,108 @@
+import pydantic
+import typing
+import requests
+
+
+from ansible_collections.nhsd.apigee.plugins.module_utils import utils
+from ansible_collections.nhsd.apigee.plugins.module_utils.constants import (
+    APIGEE_BASE_URL,
+)
+
+_cached_app_data = None
+_environment = None
+_app_id = None
+
+
+def _put_app_data():
+    return _cached_app_data
+
+
+def default_jwks_resource_url(environment=None, app_id=None):
+    if environment is None:
+        global _environment
+        environment = _environment
+    if app_id is None:
+        global _app_id
+        app_id = _app_id
+    return f"https://raw.githubusercontent.com/NHSDigital/identity-service-jwks/main/jwks/{environment}/{app_id}.json"
+
+
+class AddJwksResourceUrlToApp(pydantic.BaseModel):
+    organization: typing.Literal["nhsd-nonprod", "nhsd-prod"]
+    environment: typing.Literal[
+        "internal-dev",
+        "internal-dev-sandbox",
+        "internal-qa",
+        "internal-qa-sandbox",
+        "ref",
+        "dev",
+        "int",
+        "sandbox",
+        "prod",
+    ]
+    access_token: str
+    app_id: pydantic.UUID4
+    jwks_resource_url: pydantic.HttpUrl = pydantic.Field(default_factory=default_jwks_resource_url)
+    _app_data: typing.Dict = pydantic.PrivateAttr(default_factory=_put_app_data)
+
+    @pydantic.validator("environment")
+    def check_org_env_combo(cls, environment, values):
+        org = values.get("organization")
+        if org is None:
+            return
+        non_prod_envs = [
+            "internal-dev",
+            "internal-dev-sandbox",
+            "internal-qa",
+            "internal-qa-sandbox",
+            "ref",
+        ]
+        if org == "nhsd-nonprod" and environment not in non_prod_envs:
+            raise ValueError(
+                f"Invalid environment {environment} for organization {org}"
+            )
+        return environment
+
+    @pydantic.validator("environment")
+    def cache_put(cls, environment):
+        global _environment
+        _environment = environment
+        return environment
+
+    @pydantic.validator("app_id")
+    def check_app_exists(cls, app_id, values):
+        access_token = values.get("access_token")
+        org = values.get("organization")
+        url = f"{APIGEE_BASE_URL}organizations/{org}/apps/{app_id}"
+        app_response = utils.get(url, access_token)
+
+        if app_response.get("failed"):
+            raise ValueError(f"Unable to find app with app_id {app_id} in {org}")
+
+        app_data = app_response["response"]["body"]
+        attributes = app_data.get("attributes", [])
+        jwks_attribs = [a for a in attributes if a["name"] == "jwks-resource-url"]
+        if len(jwks_attribs) > 1:
+            raise ValueError(
+                f"App {app_id} has {len(jwks_attribs)} jwks-resource-url attributes! {[v['value'] for v in jwks_attribs]}"
+            )
+
+        # cache response data
+        global _cached_app_data
+        _cached_app_data = app_data
+        global _app_id
+        _app_id = app_id
+        return app_id
+
+    @pydantic.validator("jwks_resource_url", always=True)
+    def check_jwks_url(cls, jwks_resource_url):
+        resp = requests.get(jwks_resource_url)
+        if resp.status_code != 200:
+            raise ValueError(f"Invalid jwks_resource_url: GET {jwks_resource_url} returned {resp.status_code}")
+        try:
+            resp.json()
+        except Exception:
+            raise ValueError(
+                f"Invalid jwks_resource_url: GET {jwks_resource_url} returned {resp.content.decode()}, which is not valid JSON"
+            )
+        return jwks_resource_url
diff --git a/...llections/nhsd/apigee/plugins/module_utils/models/ansible/apply_pull_request_namespace.py b/...llections/nhsd/apigee/plugins/module_utils/models/ansible/apply_pull_request_namespace.py
@@ -23,11 +23,11 @@ class ApplyPullRequestNamespace(pydantic.BaseModel):
            :class:`ApigeeSpec` objects.
 
     :param pull_request: A string like 'pr-1234' for pull request
-        number 1234.
+        number 1234. Alternatively 'utils-pr-1234' for utils pull request number 1234
     :param manifest: The content of your manifest.yml.
     """
 
-    pull_request: pydantic.constr(regex=r"^pr-[0-9]+$")  # i.e. 'pr-1234'
+    pull_request: pydantic.constr(regex=r"^pr-[0-9]+$|^utils-pr-[0-9]+$")  # i.e. 'pr-1234' or 'utils-pr-1234'
     manifest: Manifest
 
     @pydantic.validator("manifest")

diff --git a/...le_collections/nhsd/apigee/plugins/module_utils/models/ansible/deploy_api_catalog_item.py b/...le_collections/nhsd/apigee/plugins/module_utils/models/ansible/deploy_api_catalog_item.py
@@ -1,8 +1,10 @@
 import pydantic
+import typing
+
 from ansible_collections.nhsd.apigee.plugins.module_utils.models.apigee.apidoc import ApigeeApidoc
 
 
 class DeployApidoc(pydantic.BaseModel):
     api_catalog_item: ApigeeApidoc
-    organization: str
+    organization: typing.Literal["nhsd-nonprod", "nhsd-prod"]
     access_token: str
diff --git a/...ons/ansible_collections/nhsd/apigee/plugins/module_utils/models/ansible/deploy_product.py b/...ons/ansible_collections/nhsd/apigee/plugins/module_utils/models/ansible/deploy_product.py
@@ -1,8 +1,13 @@
+import typing
+
 import pydantic
-from ansible_collections.nhsd.apigee.plugins.module_utils.models.apigee.product import ApigeeProduct
+
+from ansible_collections.nhsd.apigee.plugins.module_utils.models.apigee.product import (
+    ApigeeProduct,
+)
 
 
 class DeployProduct(pydantic.BaseModel):
-    organization: str
+    organization: typing.Literal["nhsd-nonprod", "nhsd-prod"]
     access_token: str
     product: ApigeeProduct
diff --git a/...ctions/ansible_collections/nhsd/apigee/plugins/module_utils/models/ansible/deploy_spec.py b/...ctions/ansible_collections/nhsd/apigee/plugins/module_utils/models/ansible/deploy_spec.py
@@ -1,8 +1,10 @@
 import pydantic
+import typing
+
 from ansible_collections.nhsd.apigee.plugins.module_utils.models import apigee
 
 
 class DeploySpec(pydantic.BaseModel):
     spec: apigee.spec.ApigeeSpec
-    organization: str
+    organization: typing.Literal["nhsd-nonprod", "nhsd-prod"]
     access_token: str