NHSDigital · strutt · Jan 22, 2021 · Jan 19, 2021 · Jan 19, 2021 · Jan 20, 2021
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -29,8 +29,10 @@ jobs:
         run: poetry install
         if: steps.cache.outputs.cache-hit != 'true'
 
-      - name: Test ansible collection
-        env:
-          PYTHONPATH: collections
-        working-directory: ansible
-        run: poetry run pytest
+      - name: Unit test ansible collection
+        working-directory: ansible/collections/ansible_collections/nhsd/apigee
+        run: poetry run ansible-test units --python=3.8
+
+      - name: Integration test ansible collection
+        working-directory: ansible/collections/ansible_collections/nhsd/apigee
+        run: poetry run ansible-test integration --python=3.8
diff --git a/...ollections/ansible_collections/nhsd/apigee/plugins/action/apply_pull_request_namespace.py b/...ollections/ansible_collections/nhsd/apigee/plugins/action/apply_pull_request_namespace.py
@@ -12,4 +12,4 @@ def run(self, tmp=None, task_vars=None):
         args, errors = self.validate_args(ApplyPullRequestNamespace)
         if errors:
             return errors
-        return {"apigee": args.apigee.dict(), "changed": False}
+        return {"manifest": args.manifest.dict(), "changed": False}
diff --git a/ansible/collections/ansible_collections/nhsd/apigee/plugins/action/validate_manifest.py b/ansible/collections/ansible_collections/nhsd/apigee/plugins/action/validate_manifest.py
@@ -12,4 +12,4 @@ def run(self, tmp=None, task_vars=None):
         args, errors = self.validate_args(ValidateManifest)
         if errors:
             return errors
-        return {"apigee": args.apigee.dict(), "changed": False}
+        return {"manifest": args.manifest.dict(), "changed": False}
diff --git a/...llections/nhsd/apigee/plugins/module_utils/models/ansible/apply_pull_request_namespace.py b/...llections/nhsd/apigee/plugins/module_utils/models/ansible/apply_pull_request_namespace.py
@@ -1,7 +1,6 @@
 import pydantic
 
-from ansible_collections.nhsd.apigee.plugins.module_utils.models.manifest.apigee import ManifestApigee
-from ansible_collections.nhsd.apigee.plugins.module_utils.models.manifest.meta import ManifestMeta
+from ansible_collections.nhsd.apigee.plugins.module_utils.models.manifest.manifest import Manifest
 
 
 class ApplyPullRequestNamespace(pydantic.BaseModel):
@@ -25,28 +24,26 @@ class ApplyPullRequestNamespace(pydantic.BaseModel):
 
     :param pull_request: A string like 'pr-1234' for pull request
         number 1234.
-    :param meta: The 'meta' item your manifest.yml.
-    :param apigee: The 'apigee' item from your manifest.yml
+    :param manifest: The content of your manifest.yml.
     """
 
     pull_request: pydantic.constr(regex=r"^pr-[0-9]+$")  # i.e. 'pr-1234'
-    meta: ManifestMeta
-    apigee: ManifestApigee
+    manifest: Manifest
 
-    @pydantic.validator("apigee")
-    def apply_namespace(cls, apigee, values):
-        api_name = values["meta"].api.name
+    @pydantic.validator("manifest")
+    def apply_namespace(cls, manifest, values):
+        api_name = manifest.meta.api.name
 
-        apigee.environments = [
-            env for env in apigee.environments if env.name.startswith("internal-dev")
+        manifest.apigee.environments = [
+            env for env in manifest.apigee.environments if env.name.startswith("internal-dev")
         ]
 
         # here we want:
         # canary-api-internal-dev         -> canary-api-pr-1234
         # canary-api-internal-dev-sandbox -> canary-api-pr-1234-sandbox
         old = "internal-dev"
         new = values["pull_request"]
-        for env in apigee.environments:
+        for env in manifest.apigee.environments:
             for product in env.products:
                 product.name = product.name.replace(old, new, 1)
                 product.proxies = [
@@ -64,4 +61,4 @@ def apply_namespace(cls, apigee, values):
                 )
                 entry.specId = entry.specId.replace(old, new, 1)
 
-        return apigee
+        return manifest
diff --git a/.../ansible_collections/nhsd/apigee/plugins/module_utils/models/ansible/validate_manifest.py b/.../ansible_collections/nhsd/apigee/plugins/module_utils/models/ansible/validate_manifest.py
@@ -1,11 +1,8 @@
 import os
 import re
 import pydantic
-from ansible_collections.nhsd.apigee.plugins.module_utils.models.manifest.apigee import (
-    ManifestApigee,
-)
-from ansible_collections.nhsd.apigee.plugins.module_utils.models.manifest.meta import (
-    ManifestMeta,
+from ansible_collections.nhsd.apigee.plugins.module_utils.models.manifest.manifest import (
+    Manifest,
 )
 
 
@@ -33,15 +30,15 @@ def correct_namespace(name, api_name, env_name) -> bool:
 
 
 class ValidateManifest(pydantic.BaseModel):
-    meta: ManifestMeta
-    service_name: str = ""
     dist_dir: pydantic.DirectoryPath = ""
-    apigee: ManifestApigee
+    manifest: Manifest
+    service_name: str = ""
 
     @pydantic.validator("service_name")
     def check_service_name(cls, service_name, values):
         if service_name:
-            meta = values.get("meta")
+            manifest = values.get("manifest")
+            meta = manifest.meta
             if not meta:
                 return
             api_name = meta.api.name
@@ -50,25 +47,27 @@ def check_service_name(cls, service_name, values):
                     f"pipeline defined SERVICE_NAME ('{service_name}') does not begin with manifest defined meta.api.name ('{api_name}')"
                 )
 
-    @pydantic.validator("apigee", pre=True)
-    def prepend_dist_dir_to_spec_paths(cls, apigee, values):
+    @pydantic.validator("manifest", pre=True)
+    def prepend_dist_dir_to_spec_paths(cls, manifest, values):
         dist_dir = values.get("dist_dir")
-        if dist_dir:
-            for env_dict in apigee["environments"]:
-                for spec_dict in env_dict["specs"]:
-                    path = spec_dict.get("path")
-                    if path is not None:
-                        spec_dict["path"] = os.path.join(dist_dir, path)
-        return apigee
+        print(dist_dir)
+        if not dist_dir:
+            return manifest
+        apigee = manifest["apigee"]
+        for env_dict in apigee["environments"]:
+            for spec_dict in env_dict["specs"]:
+                path = spec_dict.get("path")
+                if path is not None:
+                    spec_dict["path"] = os.path.join(dist_dir, path)
+        return manifest
 
-    @pydantic.validator("apigee")
-    def check_namespacing(cls, apigee, values):
-        meta = values.get("meta")
-        if not meta:
+    @pydantic.validator("manifest")
+    def check_namespacing(cls, manifest, values):
+        if not manifest.meta:
             return
-        api_name = meta.api.name
+        api_name = manifest.meta.api.name
 
-        for env in apigee.environments:
+        for env in manifest.apigee.environments:
             if env is None:
                 continue
             for product in env.products:
@@ -81,4 +80,4 @@ def check_namespacing(cls, apigee, values):
                     raise ValueError(
                         f"{spec.name} does not conform to namespace for {api_name}-*{env.name}"
                     )
-        return apigee
+        return manifest
diff --git a/...le/collections/ansible_collections/nhsd/apigee/plugins/module_utils/models/apigee/spec.py b/...le/collections/ansible_collections/nhsd/apigee/plugins/module_utils/models/apigee/spec.py
@@ -10,6 +10,8 @@ class ApigeeSpec(pydantic.BaseModel):
 
     @pydantic.validator("content", always=True)
     def load_content(cls, content, values):
+        if content is not None:
+            return content
         path = values.get("path")
         if not path:  # When path does not validate.
             return None

diff --git a/...lections/ansible_collections/nhsd/apigee/plugins/module_utils/models/manifest/manifest.py b/...lections/ansible_collections/nhsd/apigee/plugins/module_utils/models/manifest/manifest.py
@@ -0,0 +1,9 @@
+import pydantic
+
+from ansible_collections.nhsd.apigee.plugins.module_utils.models.manifest.apigee import ManifestApigee
+from ansible_collections.nhsd.apigee.plugins.module_utils.models.manifest.meta import ManifestMeta
+
+
+class Manifest(pydantic.BaseModel):
+    apigee: ManifestApigee
+    meta: ManifestMeta
diff --git a/.../collections/ansible_collections/nhsd/apigee/plugins/module_utils/models/manifest/meta.py b/.../collections/ansible_collections/nhsd/apigee/plugins/module_utils/models/manifest/meta.py
@@ -1,11 +1,17 @@
-import typing
 import pydantic
 
+SCHEMA_VERSION = "1.0.0"
+
 
 class ManifestMetaApi(pydantic.BaseModel):
     name: pydantic.constr(regex=r"^[a-z]+(-[a-z]+)*$")
     id: pydantic.UUID4
 
+    def dict(self, **kwargs):
+        native = super().dict(**kwargs)
+        native.update({"id": str(native["id"])})
+        return native
+
 
 class ManifestMeta(pydantic.BaseModel):
     schema_version: pydantic.constr(regex=r"[0-9]+(\.[0-9]+){0,2}")
@@ -14,15 +20,15 @@ class ManifestMeta(pydantic.BaseModel):
     @pydantic.validator("schema_version")
     def validate_schema_version(cls, schema_version):
         semantic_parts = schema_version.split(".")
+
+        MAJOR, MINOR, PATCH = [int(x) for x in SCHEMA_VERSION.split(".")]
         major = int(semantic_parts[0])
-        minor = 0 if len(semantic_parts) < 2 else int(semantic_parts[1])
-        patch = 0 if len(semantic_parts) < 3 else int(semantic_parts[2])
-
-        if major != 1:
-            raise ValueError(f"Invalid major version {major} for schema_version")
-        if minor != 0:
-            raise ValueError(f"Invalid minor version {minor} for major schema_version {major}")
-        if patch != 0:
-            raise ValueError(f"Invalid patch version {patch} for major/minor schema_version {major}.{minor}")
+        # minor = int(MAJOR) if len(semantic_parts) < 2 else int(semantic_parts[1])
+        # patch = int(PATCH) if len(semantic_parts) < 3 else int(semantic_parts[2])
+
+        if major != MAJOR:
+            raise ValueError(
+                f"Current schema version is {SCHEMA_VERSION}. All minor and patch changes are backwards compatible."
+            )
 
         return schema_version
diff --git a/ansible/collections/ansible_collections/nhsd/apigee/roles/deploy_manifest/tasks/main.yml b/ansible/collections/ansible_collections/nhsd/apigee/roles/deploy_manifest/tasks/main.yml
@@ -0,0 +1,75 @@
+- name: ensure valid APIGEE_ORGANIZATION
+  fail:
+    msg: Invalid APIGEE_ORGANIZATION
+  when: APIGEE_ORGANIZATION not in APIGEE_ORGANIZATIONS
+
+- name: ensure valid APIGEE_ENVIRONMENT
+  fail:
+    msg: Invalid APIGEE_ENVIRONMENT
+  when: APIGEE_ENVIRONMENT not in APIGEE_ENVIRONMENTS[APIGEE_ORGANIZATION]
+
+  # NOTE: Once the manifest handles proxies this can be removed!
+  # it must remain (for now) to handle the frequent case where
+  # multile Azure pipeline stages deploy to the same Apigee
+  # environment.
+- name: ensure SERVICE_NAME
+  fail:
+    msg: Invalid SERVICE_NAME
+  when: not SERVICE_NAME
+
+- name: load manifest
+  set_fact:
+    manifest: "{{ lookup('file', (DIST_DIR, 'manifest.yml') | path_join) | from_yaml }}"
+
+- name: validate manifest
+  nhsd.apigee.validate_manifest:
+    service_name: "{{ SERVICE_NAME }}"
+    dist_dir: "{{ DIST_DIR }}"
+    manifest: "{{ manifest }}"
+  register: validated_manifest
+
+- name: apply pr namespace
+  nhsd.apigee.apply_pull_request_namespace:
+    manifest: "{{ validated_manifest['manifest'] }}"
+    pull_request: "{{ PULL_REQUEST }}"
+  register: pr_manifest
+  when: PULL_REQUEST != ""
+
+- name: select regular or pr manifest 
+  set_fact:
+    manifest: "{{ pr_manifest['manifest'] | default(validated_manifest['manifest']) }}"
+
+- name: select environment
+  set_fact:
+    apigee_environment: "{{ manifest.apigee.environments | selectattr('name', '==', APIGEE_ENVIRONMENT) | list | first }}"
+
+- name: deploy apigee products
+  nhsd.apigee.deploy_product:
+    product: "{{ item }}"
+    organization: "{{ APIGEE_ORGANIZATION }}"
+    access_token: "{{ APIGEE_ACCESS_TOKEN }}"
+  loop: "{{ apigee_environment.products }}"
+  # Since proxies are uploaded and deployed per-Azure pipeline
+  # stage, this prevents a race condition on the first deployment,
+  # since products can only be created if they reference proxies
+  # that exist.  This safety check can be removed when the
+  # manifest manages proxies too.
+  when: item.name | regex_search('^' + SERVICE_NAME + '-' + PULL_REQUEST | default(APIGEE_ENVIRONMENT))
+
+- name: deploy apigee specs
+  nhsd.apigee.deploy_spec:
+    spec: "{{ item }}"
+    organization: "{{ APIGEE_ORGANIZATION }}"
+    access_token: "{{ APIGEE_ACCESS_TOKEN }}"
+  loop: "{{ apigee_environment.specs }}"
+
+- name: deploy api catalog
+  nhsd.apigee.deploy_api_catalog_item:
+    api_catalog_item: "{{ item }}"
+    organization: "{{ APIGEE_ORGANIZATION }}"
+    access_token: "{{ APIGEE_ACCESS_TOKEN }}"
+  loop: "{{ apigee_environment.api_catalog }}"
+  # See comment on product loop control. (I believe APIDocs
+  # actually can reference products that don't exist. But this is
+  # cleaner.)
+  when: item.edgeAPIProductName | regex_search('^' + SERVICE_NAME + '-' + PULL_REQUEST | default(APIGEE_ENVIRONMENT))
diff --git a/ansible/collections/ansible_collections/nhsd/apigee/roles/deploy_manifest/vars/main.yml b/ansible/collections/ansible_collections/nhsd/apigee/roles/deploy_manifest/vars/main.yml
@@ -0,0 +1,11 @@
+APIGEE_ORGANIZATIONS: [nhsd-nonprod, nhsd-prod]
+APIGEE_ENVIRONMENTS:
+  nhsd-nonprod: [internal-dev, internal-dev-sandbox, internal-qa, internal-qa-sandbox, ref]
+  nhsd-prod: [dev, sandbox, int, prod]
+
+APIGEE_ENVIRONMENT: "{{ lookup('env','APIGEE_ENVIRONMENT') }}"
+APIGEE_ORGANIZATION: "{{ lookup('env', 'APIGEE_ORGANIZATION') }}"
+APIGEE_ACCESS_TOKEN: "{{ lookup('env', 'APIGEE_ACCESS_TOKEN') }}"
+SERVICE_NAME: "{{ lookup('env', 'SERVICE_NAME') }}"
+PULL_REQUEST: "{{ lookup('env', 'PULL_REQUEST') }}"
+DIST_DIR: "{{ lookup('env', 'DIST_DIR') }}"
diff --git a/ansible/collections/ansible_collections/nhsd/apigee/roles/template_manifest/tasks/main.yml b/ansible/collections/ansible_collections/nhsd/apigee/roles/template_manifest/tasks/main.yml
@@ -0,0 +1,28 @@
+- name: set full path to manifest file
+  set_fact:
+    manifest_template: "{{ (DIST_DIR, 'manifest_template.yml') | path_join }}"
+    manifest_file: "{{ (DIST_DIR, 'manifest.yml') | path_join }}"
+
+- name: load template streams
+  set_fact:
+    template_streams: "{{ lookup('file', manifest_template ).split('\n---\n') | list }}"
+
+- name: Set template vars
+  set_fact:
+    "{{ item.key }}": "{{ item.value }}"
+  loop: "{{ template_streams | first | from_yaml | dict2items }}"
+
+- name: template manifest
+  set_fact:
+    manifest: "{{ lookup('template', manifest_template) | from_yaml_all | list | last }}"
+
+- name: validate templated manifest
+  nhsd.apigee.validate_manifest:
+    manifest: "{{ manifest }}"
+    service_name: "{{ SERVICE_NAME }}"
+    dist_dir: "{{ DIST_DIR }}"
+
+- name: write template to file
+  copy:
+    content: "{{ manifest | to_nice_yaml(indent=2) }}"
+    dest: "{{ manifest_file }}"
diff --git a/ansible/collections/ansible_collections/nhsd/apigee/roles/template_manifest/vars/main.yml b/ansible/collections/ansible_collections/nhsd/apigee/roles/template_manifest/vars/main.yml
@@ -0,0 +1,2 @@
+SERVICE_NAME: "{{ lookup('env', 'SERVICE_NAME')}}"
+DIST_DIR: "{{ lookup('env', 'DIST_DIR') }}"
diff --git a/ansible/collections/ansible_collections/nhsd/apigee/roles/validate_manifest/tasks/main.yml b/ansible/collections/ansible_collections/nhsd/apigee/roles/validate_manifest/tasks/main.yml
@@ -0,0 +1,18 @@
+
+- name: check DIST_DIR
+  fail:
+    msg: "DIST_DIR not defined"
+  when: DIST_DIR is not defined
+
+- name: set full path to manifest file
+  set_fact:
+    manifest_file: "{{ (DIST_DIR, 'manifest.yml') | path_join }}"
+
+- name: load manifest
+  set_fact:
+    manifest: "{{ lookup('file', manifest_file ) | from_yaml }}"
+
+- name: validate manifest
+  nhsd.apigee.validate_manifest:
+    manifest: "{{ manifest }}"
+    dist_dir: "{{ DIST_DIR }}"
diff --git a/ansible/collections/ansible_collections/nhsd/apigee/roles/validate_manifest/vars/main.yml b/ansible/collections/ansible_collections/nhsd/apigee/roles/validate_manifest/vars/main.yml
@@ -0,0 +1 @@
+DIST_DIR: "{{ lookup('env', 'DIST_DIR') }}"
diff --git a/.../tests/integration/targets/test_validate_manifest/files/canary-api-v1.0.0/canary-api.json b/.../tests/integration/targets/test_validate_manifest/files/canary-api-v1.0.0/canary-api.json
@@ -0,0 +1 @@
+{}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		SERVICE_NAME: "{{ lookup('env', 'SERVICE_NAME')}}"
		DIST_DIR: "{{ lookup('env', 'DIST_DIR') }}"