Python3 script for the constructor of the config file with Jinja in Unix/Linux

Recently at work there was a need to have a single YAML configuration file with all types and ENV variables for them. In this case, that script could parse it and substitute all occurrences of all incoming files. This is very useful when you have many types of machines with different settings. I solved the problem, wrote a script, all very happy.

Python3 script for the constructor of the config file with Jinja in Unix/Linux

To begin with, that become Python and other Goodies:

Installing python in Unix/Linux in the process of writing (drafts)

Installing pip/setuptools/wheel under Unix/Linux

Other useful topics:

Installing/Using pexpect and python in Unix/Linux

Installing virtualenv + virtualenvwrapper in Unix/Linux

Pip3 has now put packages:

$ pip3 install-U pyyaml argparse

Open the file, for example, I so called:

$ vim templater_py3.py

The contents will be as follows:

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import argparse
import time
import os
import pathlib
import glob
import yaml

from jinja2 import Environment, StrictUndefined
from jinja2.loaders import FileSystemLoader
from jinja2.exceptions import UndefinedError, TemplateRuntimeError, TemplateNotFound


class Bgcolors:
 def __init__(self):
 self.colors = {
 'PURPURE': '33[95m',
 'BLUE': '33[94m',
 'GREEN': '33[92m',
 'YELLOW': '33[93m',
 'RED': '33[91m',
 'ENDC': '33[0m',
 'BOLD': '33[1m',
 'UNDERLINE': '33[4m'
}


def direcroty(in_put, output):

 array_files = []

 if in_put is not None:
 if os.path.exists(os.path.dirname(in_put)):
 if os.path.isdir(os.path.dirname(in_put)):
 print(Bgcolors().colors['GREEN'] + 'Input directory is: {}'.format(pathlib.Path(in_put.strip())),
Bgcolors().colors['ENDC'])
 for file in glob.glob(os.path.join(in_put, '*')):
 if os.path.isfile(file):
array_files.append(pathlib.Path(file.strip()))
else:
 print('Input file is: {}'.format(pathlib.Path(in_put.strip())))
 if os.path.isfile(pathlib.Path(in_put.strip())):
array_files.append(pathlib.Path(in_put.strip()))
else:
 print(Bgcolors().colors['RED'] +
 'Use correct PATH for configs or check your file: {}'.format(in_put), 'nIt doesnt exist!',
Bgcolors().colors['ENDC'])
exit(0)
else:
 print(Bgcolors().colors['RED'] +
 'Use correct PATH for configs or check your file: {}'.format(in_put), 'nIt doesnt exist!',
Bgcolors().colors['ENDC'])
exit(0)
else:
 print(Bgcolors().colors['RED'], 'Please set [--in_put-file] or [--in_put-dir] or use [--help] to get a help',
Bgcolors().colors['ENDC'])
exit(0)

 if output is not None:
 if os.path.exists(os.path.dirname(output)):
 print(Bgcolors().colors['GREEN'] + 'Output folder: ', pathlib.Path(output.strip()), 'n',
Bgcolors().colors['ENDC'])
else:
try:
os.stat(pathlib.Path(os.path.dirname(output)))
 print(Bgcolors().colors['GREEN'] + 'Output folder: ', pathlib.Path(output.strip()), 'n',
Bgcolors().colors['ENDC'])
 except Exception:
os.mkdir(output)
 print(Bgcolors().colors['GREEN'] + 'Output folder: ', pathlib.Path(output.strip()), 'n',
Bgcolors().colors['ENDC'])
else:
 print(Bgcolors().colors['RED'], 'Please set [--output-dir] or use [--help] to get a help',
Bgcolors().colors['ENDC'])
exit(0)

 # print(array_files)

 return array_files


def jinja_render(in_put, output, env, env_type, yml_files):

 array_files = direcroty(in_put, output)
 sections = ['default', env, env_type]
 # add properties that we know about
 config = {'ENVIRONMENT_TYPE': env_type, 'ENVIRONMENT': env}

 # Create the jinja2 environment.
 # Notice the use of trim_blocks, which greatly helps control whitespace.
 j2_env = Environment(
loader=FileSystemLoader("),
trim_blocks=True,
autoescape=False,
undefined=StrictUndefined)

 if yml_files is not None:
 for yml_file in yml_files:
 with open(yml_file) as stream:
try:
 # print(yaml.safe_load(stream))
 cfg_yaml = yaml.safe_load(stream.read())
 for key in sections:
 props = cfg_yaml.get(key)
 # print('{0} : {1}'.format(key,props), len(props))
 if props is not None:
 # print(yaml.dump(props, default_flow_style=False))
config.update(props)
 except yaml.YAMLError as exc:
print(exc)
 for envriron in os.environ:
 config.update({envriron: os.environ.get(envriron)})
 # print(envriron, os.environ.get(envriron))
else:
 print(Bgcolors().colors['RED'], 'Please set [--yml] or use [--help] to get a help',
Bgcolors().colors['ENDC'])
exit(0)

 print(yaml.dump(config, default_flow_style=False))
 errs = []

 for file in array_files:
 # file_name = os.path.basename(file)
 head, file_name = os.path.split(file.as_posix())
 template = j2_env.get_template(file.as_posix())
try:
 out_content = template.render(env=os.environ, **config)
 out_file = output + '/' + file_name
try:
 f = open(out_file, 'wb')
f.write(out_content.encode('utf-8'))
f.close()
 print(Bgcolors().colors['GREEN'] + 'The [{}] the file has been written.'.format(out_file)+
Bgcolors().colors['ENDC'])
 except ValueError:
 print('I cant write to file: ', ValueError)
 # print('file:', file,'n', out_content)
 except UndefinedError as e:
 if e not in errs:
errs.append(str(e))
 # errs.append(str(e).split("'")[1])
 if len(errs) > 0:
print('~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~')
 print('Total errors: [{}]'.format(len(errs)))
print(errs)
print('~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~')

 return jinja_render


def main():
 start__time = time.time()
 parser = argparse.ArgumentParser(prog='python3 script_name.py -h',
 usage='python3 script_name.py {ARGS}',
add_help=True,
prefix_chars='--/',
 epilog="'created by Vitalii Natarov"')
 parser.add_argument('--version', action='version', version='v0.2.0')
 parser.add_argument('--input-file', '--input-dir', dest='input', help='Set input file or folder to find configs',
default='./templates')
 parser.add_argument('--output-dir', dest='output', help='Set a output folder for outgoing files',
default='./output')
 parser.add_argument('--env', dest='env', help='Environment: tst, sit01, prd, etc', default=None)
 parser.add_argument('--env-type', dest='env_type', help='Type Environment: prod or nonprod', default=None)
 parser.add_argument('--upgrade', dest='yml_files', nargs="+", help='Config property yml files', default=None)

 results = parser.parse_args()

 in_put = os.path.expanduser(results.input)
 output = os.path.expanduser(results.output)
 env = results.env
 env_type = results.env_type
 yml_files = results.yml_files

 jinja_render(in_put, output, env, env_type, yml_files)

 end__time = round(time.time() - start__time, 2)
 print("--- %s seconds ---" % end__time)
 print(Bgcolors().colors['GREEN'], "============================================================",
Bgcolors().colors['ENDC'])
 print(Bgcolors().colors['GREEN'], "==========================FINISHED==========================",
Bgcolors().colors['ENDC'])
 print(Bgcolors().colors['GREEN'], "============================================================",
Bgcolors().colors['ENDC'])


if __name__ == '__main__':
main()

To get help on the scripts, do:

$ python3 templater_py3.py -h
usage: python3 script_name.py {ARGS}

optional arguments:
 -h, --help show this help message and exit
 --version show the program's version number and exit
 --input-file INPUT, --input-dir INPUT
 Set input file or folder to find configs
 --output-dir OUTPUT Set a output folder for outgoing files
 --env ENV Environment: tst, sit01, prd, etc
 --env-type Type ENV_TYPE Environment: prod or nonprod
 --yml YML_FILES [YML_FILES ...]
 Property yml Config files

created by Vitalii Natarov

Where:

  • —input-file or —input-dir — Input data (folder or file) to handle jinja. Ie to Jinja replaced all occurrences in the configuration file or other files.
  • —output-dir — output will be stored in this folder.
  • —env — Name of environment (service1, service2).
  • —env-type — the Type of environment (prod, nonprod, dev).
  • —yml — YML file with all settings for all types of ENV and ENV in General.

I will give an example using this script:

$ python3 templater_py3.py --input-dir=templates --yml configProps.yml --env=test --env-type=nonprod --ouput-dir=output

If necessary, the script understands that it is necessary to create the output folder for this, it is not necessary to create.

This script was tested on:

  • MacOS with python 3.7.
  • CentOS 7 with python 3.4, and 3.5.

A little later, lay out the script on Github and will update the article.

That’s all, “Python3 script for the constructor of the config file with Jinja in Unix/Linux” is completed.

Source: linux-notes.org

(Visited 48 times, 1 visits today)