Replace values in Yaml file with value in dictionary

[PelleH]

Hi, I have a problem with one script. (Python3 ver 3.6.x)
I have a file "deployment_custom.txt" containing values to set in a Yaml file. One value per line.
Then I have the Yaml file, "deployment.yml".

Here are a snippet from "deployment_custom.txt"

spec components synapse config postgresql port 123
spec components synapse config delegatedAuth oidc clientID q11xxx

Here are a snippet from "deployment.yml"

spec:
  components:
    synapse:
      config:
        delegatedAuth:
          oidc:
            - backchannelLogoutEnabled: true
              clientId: 111111111111
              idpId: proxy1
        postgresql:
          database: synapse
          host: postgresql
          passwordSecretKey: postgresPassword
          sslMode: disable
          user: syn_user
          port: 5436[/quote]

My script do fix the first line "port 123".
The problem starts with the dictionary oidc containing clientId. How to update that value?

from ruamel.yaml import YAML
import sys

def load_custom_values(file_path):
    with open(file_path, "r") as custom_file:
        return [line.strip() for line in custom_file]

def load_yaml(file_path):
    yaml = YAML()
    with open(file_path, "r") as yaml_file:
        return yaml.load(yaml_file)

def edit_yaml(data, keys, new_value):
    if len(keys) == 1:
        data[keys[0]] = int(new_value) if new_value.isdigit() else new_value
    else:
        edit_yaml(data[keys[0]], keys[1:], new_value)

def apply_custom_values(yaml_data, custom_values):
    for custom_value in custom_values:
        parts = custom_value.split()
        if len(parts) >= 3:
            section = parts[0]
            if section in yaml_data:
                keys = parts[1:-1]
                new_value = parts[-1]
                edit_yaml(yaml_data[section], keys, new_value)
    return yaml_data

def save_yaml(data, file_path):
    yaml = YAML()
    yaml.indent(mapping=2, sequence=4, offset=2)  # Ensure proper indentation
    with open(file_path, "w") as modified_file:
        yaml.dump(data, modified_file)

def fix_deployment_file(org):
    custom_file_path = f"./{org}/deployment_custom.txt"
    yaml_file_path = f"./{org}/deployment.yml"

    custom_values = load_custom_values(custom_file_path)
    yaml_data = load_yaml(yaml_file_path)
    modified_yaml_data = apply_custom_values(yaml_data, custom_values)
    save_yaml(modified_yaml_data, yaml_file_path)

if __name__ == "__main__":
    org = sys.argv[1] if len(sys.argv) > 1 else None
    if not org:
        print("Please provide the name of ORG.")
        sys.exit(1)
    fix_deployment_file(org)

How to solve the problematic part with dictionary values?

/Pelle

[alexjordan]

Hi, I have a problem with one script. (Python3 ver 3.6.x)
I have a file "deployment_custom.txt" containing values to set in a Yaml file. One value per line.
Then I have the Yaml file, "deployment.yml".

Here are a snippet from "deployment_custom.txt"

spec components synapse config postgresql port 123
spec components synapse config delegatedAuth oidc clientID q11xxx

Here are a snippet from "deployment.yml"

spec:
  components:
    synapse:
      config:
        delegatedAuth:
          oidc:
            - backchannelLogoutEnabled: true
              clientId: 111111111111
              idpId: proxy1
        postgresql:
          database: synapse
          host: postgresql
          passwordSecretKey: postgresPassword
          sslMode: disable
          user: syn_user
          port: 5436[/quote]

My script do fix the first line "port 123".
The problem starts with the dictionary oidc containing clientId. How to update that value?

from ruamel.yaml import YAML
import sys

def load_custom_values(file_path):
    with open(file_path, "r") as custom_file:
        return [line.strip() for line in custom_file]

def load_yaml(file_path):
    yaml = YAML()
    with open(file_path, "r") as yaml_file:
        return yaml.load(yaml_file)

def edit_yaml(data, keys, new_value):
    if len(keys) == 1:
        data[keys[0]] = int(new_value) if new_value.isdigit() else new_value
    else:
        edit_yaml(data[keys[0]], keys[1:], new_value)

def apply_custom_values(yaml_data, custom_values):
    for custom_value in custom_values:
        parts = custom_value.split()
        if len(parts) >= 3:
            section = parts[0]
            if section in yaml_data:
                keys = parts[1:-1]
                new_value = parts[-1]
                edit_yaml(yaml_data[section], keys, new_value)
    return yaml_data

def save_yaml(data, file_path):
    yaml = YAML()
    yaml.indent(mapping=2, sequence=4, offset=2)  # Ensure proper indentation
    with open(file_path, "w") as modified_file:
        yaml.dump(data, modified_file)

def fix_deployment_file(org):
    custom_file_path = f"./{org}/deployment_custom.txt"
    yaml_file_path = f"./{org}/deployment.yml"

    custom_values = load_custom_values(custom_file_path)
    yaml_data = load_yaml(yaml_file_path)
    modified_yaml_data = apply_custom_values(yaml_data, custom_values)
    save_yaml(modified_yaml_data, yaml_file_path)

if __name__ == "__main__":
    org = sys.argv[1] if len(sys.argv) > 1 else None
    if not org:
        print("Please provide the name of ORG.")
        sys.exit(1)
    fix_deployment_file(org)

How to solve the problematic part with dictionary values?

/Pelle

I have done somewhat similar for one of the , where nested dictionaries, such as oidc: clientId, need updating dynamically, modifying YAML files using Python can streamline the process. Below is an improved version of your script that correctly updates nested dictionary values:

Updated Python Solution
python
Copy
Edit

from ruamel.yaml import YAML
import sys

def load_custom_values(file_path):
    with open(file_path, "r") as custom_file:
        return [line.strip().split() for line in custom_file if line.strip()]

def load_yaml(file_path):
    yaml = YAML()
    with open(file_path, "r") as yaml_file:
        return yaml.load(yaml_file)

def edit_yaml(data, keys, new_value):
    """Recursively updates YAML nested dictionaries."""
    if len(keys) == 1:
        data[keys[0]] = int(new_value) if new_value.isdigit() else new_value
    elif isinstance(data.get(keys[0]), list):  
        for item in data[keys[0]]:  
            if isinstance(item, dict) and keys[1] in item:
                item[keys[1]] = new_value  
    else:
        edit_yaml(data[keys[0]], keys[1:], new_value)

def apply_custom_values(yaml_data, custom_values):
    for parts in custom_values:
        if len(parts) >= 3:
            keys = parts[:-1]  # All elements except the last one are keys
            new_value = parts[-1]  # Last element is the value
            edit_yaml(yaml_data, keys, new_value)
    return yaml_data

def save_yaml(data, file_path):
    yaml = YAML()
    yaml.indent(mapping=2, sequence=4, offset=2)  # Ensure proper indentation
    with open(file_path, "w") as modified_file:
        yaml.dump(data, modified_file)

def fix_deployment_file(org):
    custom_file_path = f"./{org}/deployment_custom.txt"
    yaml_file_path = f"./{org}/deployment.yml"

    custom_values = load_custom_values(custom_file_path)
    yaml_data = load_yaml(yaml_file_path)
    modified_yaml_data = apply_custom_values(yaml_data, custom_values)
    save_yaml(modified_yaml_data, yaml_file_path)

if __name__ == "__main__":
    org = sys.argv[1] if len(sys.argv) > 1 else None
    if not org:
        print("Please provide the name of ORG.")
        sys.exit(1)
    fix_deployment_file(org)

This script ensures that even deeply nested YAML structures can be updated dynamically, preventing configuration errors that could disrupt automated systems.

Larz60+ write Feb-11-2025, 09:56 AM:
additional spam link removed

buran write Feb-10-2025, 07:51 PM:
spam link removed
Please, use proper tags when post code, traceback, output, etc. This time I have added tags for you.
See BBcode help for more info.