From 051d67659a1b56a2cd52afc050993ebe850b3d47 Mon Sep 17 00:00:00 2001 From: Arthur Diniz Date: Fri, 10 Jul 2020 06:42:19 -0300 Subject: [PATCH 01/10] Enable test with pytest in action Signed-off-by: Arthur Diniz --- .github/workflows/build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 26411c8..3d82e31 100755 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,7 +29,7 @@ jobs: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # The GitHub editor is 127 chars wide flake8 . --count --max-complexity=10 --max-line-length=127 --statistics - # - name: Test with pytest - # run: | - # pip install pytest - # pytest + - name: Test with pytest + run: | + pip install -r requirements-test.txt + pytest -v tests/ From 4fd80614a02da3a5b72e6857ad70004cd4928343 Mon Sep 17 00:00:00 2001 From: Arthur Diniz Date: Fri, 10 Jul 2020 06:43:08 -0300 Subject: [PATCH 02/10] Document how to run tests in README Signed-off-by: Arthur Diniz --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index 09354df..8e7f04b 100755 --- a/README.md +++ b/README.md @@ -144,6 +144,17 @@ deactivate pip3 install --editable . ``` +#### Run tests +First make sure to install the test dependencies: +``` +pip3 install -r requirements-test.txt +``` + +To run the test suite: +```bash +pytest -v tests/ +``` + #### Manualy generate binary ```bash pyinstaller --clean --hidden-import one.__main__ cli.py --onefile --noconsole -n one From 69c020478c8f4cad8dcef301ad960ef6392f2afd Mon Sep 17 00:00:00 2001 From: Arthur Diniz Date: Fri, 10 Jul 2020 06:43:24 -0300 Subject: [PATCH 03/10] Adds required libs to run tests Signed-off-by: Arthur Diniz --- requirements-test.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 requirements-test.txt diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000..be1a0cf --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,3 @@ +mock +pytest +click \ No newline at end of file From 1327cf3d2fcb753f5793185ee87ccf386dc1ea6f Mon Sep 17 00:00:00 2001 From: Arthur Diniz Date: Fri, 10 Jul 2020 06:45:50 -0300 Subject: [PATCH 04/10] Add test to terraform modules check feature Signed-off-by: Arthur Diniz --- one/utils/terraform_modules.py | 91 ++++++++++++--------- requirements-test.txt | 3 +- tests/__init__.py | 0 tests/test_terraform_modules_check.py | 113 ++++++++++++++++++++++++++ 4 files changed, 169 insertions(+), 38 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/test_terraform_modules_check.py diff --git a/one/utils/terraform_modules.py b/one/utils/terraform_modules.py index 6850d0e..ef1d1dd 100644 --- a/one/utils/terraform_modules.py +++ b/one/utils/terraform_modules.py @@ -5,48 +5,65 @@ from os import path -def terraform_modules_check(): - file_path = '.terraform/modules/modules.json' - if not path.exists(file_path): +def get_modules_list(url='https://modules.dnx.one/api.json'): + response = requests.get(url) + if response.status_code != 200: raise SystemExit + return response.json() - response = requests.get('https://modules.dnx.one/api.json') - if response.status_code != 200: + +def check_file_path(file_path): + if not path.exists(file_path): raise SystemExit - api = response.json() - with open('.terraform/modules/modules.json') as modules_json_file: + +def terraform_modules_check(file_path='.terraform/modules/modules.json', api=None): + check_file_path(file_path) + + json_api = api or get_modules_list() + + results = {} + + with open(file_path) as modules_json_file: data = json.load(modules_json_file) click.echo( click.style('\nInitializing DNX modules check...', bold=True) ) - for module in data['Modules']: - - if not module['Source']: - continue - - split = re.split(r'[./]\s*', module['Source']) - if len(split) >= 5 and split[4] == 'DNXLabs': - name = re.split(r'[./]\s*', module['Source'])[5] - version = module['Source'].split('=')[1] - key = module['Key'] - api_version = '' - try: - api_version = api['modules'][name]['tag_name'] - except KeyError: - click.echo( - click.style('ERROR ', fg='red') + - 'Could not find module ' + name + ' at DNX modules API.' - ) - if api_version != version: - click.echo( - '- ' + name + '/' + key + ': ' + - click.style(version, fg='yellow') + - ' ~> ' + - click.style(api_version, fg='green') - ) - else: - click.echo( - '- ' + name + '/' + key + ': ' + - click.style(version, fg='green') - ) + try: + for module in data['Modules']: + + if not module['Source']: + continue + split = re.split(r'[./]\s*', module['Source']) + if len(split) >= 5 and split[4] == 'DNXLabs': + name = re.split(r'[./]\s*', module['Source'])[5] + version = module['Source'].split('=')[1] + key = module['Key'] + api_version = '' + try: + api_version = json_api['modules'][name]['tag_name'] + except KeyError: + click.echo( + click.style('ERROR ', fg='red') + + 'Could not find module ' + name + ' at DNX modules API.' + ) + if api_version != version: + results[name] = {"key": key, "version": version, "api_version": api_version} + click.echo( + '- ' + name + '/' + key + ': ' + + click.style(version, fg='yellow') + + ' ~> ' + + click.style(api_version, fg='green') + ) + else: + results[name] = {"key": key, "version": version} + click.echo( + '- ' + name + '/' + key + ': ' + + click.style(version, fg='green') + ) + except KeyError: + click.echo( + click.style('ERROR ', fg='red') + + 'Couldn not find data from local terraform modules' + ) + return results diff --git a/requirements-test.txt b/requirements-test.txt index be1a0cf..987ac04 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,3 +1,4 @@ mock pytest -click \ No newline at end of file +click +flake8 \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_terraform_modules_check.py b/tests/test_terraform_modules_check.py new file mode 100644 index 0000000..f92771e --- /dev/null +++ b/tests/test_terraform_modules_check.py @@ -0,0 +1,113 @@ +from one.utils.terraform_modules import terraform_modules_check +from click.testing import CliRunner + +runner = CliRunner() + + +def test_terraform_module_same_version(): + api_mock_data = """{ + "Modules":[ + { + "Key": "terraform-aws-test", + "Source": "git::https://github.com/DNXLabs/terraform-aws-test.git?ref=1.0.0" + } + ] + }""" + + terraform_mock_data = { + "modules": { + "terraform-aws-test": { + "html_url": "https://github.com/DNXLabs/terraform-aws-test/releases/tag/1.0.0", + "tag_name": "1.0.0" + } + } + } + + with runner.isolated_filesystem(): + with open('modules.json', 'w') as f: + f.write(api_mock_data) + results = terraform_modules_check(file_path='modules.json', api=terraform_mock_data) + print(results) + assert results == {'terraform-aws-test': {'key': 'terraform-aws-test', 'version': '1.0.0'}} + + +def test_terraform_module_new_version(): + api_mock_data = """{ + "Modules":[ + { + "Key":"terraform-aws-test", + "Source":"git::https://github.com/DNXLabs/terraform-aws-test.git?ref=1.0.0" + } + ] + }""" + + terraform_mock_data = { + "modules": { + "terraform-aws-test": { + "html_url": "https://github.com/DNXLabs/terraform-aws-test/releases/tag/1.0.0", + "tag_name": "1.0.1" + } + } + } + + with runner.isolated_filesystem(): + with open('modules.json', 'w') as f: + f.write(api_mock_data) + results = terraform_modules_check(file_path='modules.json', api=terraform_mock_data) + print(results) + assert results == {'terraform-aws-test': {'key': 'terraform-aws-test', 'version': '1.0.0', 'api_version': '1.0.1'}} + + +def test_empty_modules_list(): + api_mock_data = """{ + "Modules":[] + }""" + + terraform_mock_data = {} + + with runner.isolated_filesystem(): + with open('modules.json', 'w') as f: + f.write(api_mock_data) + results = terraform_modules_check(file_path='modules.json', api=terraform_mock_data) + print(results) + assert results == {} + + +def test_terraform_module_local_dir_pointer(): + api_mock_data = """{ + "Modules":[ + { + "Key":"", + "Source":"", + "Dir":"." + } + ] + }""" + + terraform_mock_data = {} + + with runner.isolated_filesystem(): + with open('modules.json', 'w') as f: + f.write(api_mock_data) + results = terraform_modules_check(file_path='modules.json', api=terraform_mock_data) + print(results) + assert results == {} + + +def test_empty_missing_module_source(): + api_mock_data = """{ + "Modules":[ + { + "Key":"terraform-aws-test" + } + ] + }""" + + terraform_mock_data = {} + + with runner.isolated_filesystem(): + with open('modules.json', 'w') as f: + f.write(api_mock_data) + results = terraform_modules_check(file_path='modules.json', api=terraform_mock_data) + print(results) + assert results == {} From dc6b9ffa540a3d80757878cd13e4f63fb1d1a240 Mon Sep 17 00:00:00 2001 From: Arthur Diniz Date: Fri, 10 Jul 2020 06:57:47 -0300 Subject: [PATCH 05/10] Document how to run lint in README Signed-off-by: Arthur Diniz --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8e7f04b..52c220a 100755 --- a/README.md +++ b/README.md @@ -144,7 +144,7 @@ deactivate pip3 install --editable . ``` -#### Run tests +#### Run tests and lint First make sure to install the test dependencies: ``` pip3 install -r requirements-test.txt @@ -155,6 +155,11 @@ To run the test suite: pytest -v tests/ ``` +To run the lint check: +```bash +flake8 . --count --max-complexity=10 --max-line-length=127 --statistics --exclude env +``` + #### Manualy generate binary ```bash pyinstaller --clean --hidden-import one.__main__ cli.py --onefile --noconsole -n one From 92c3bc42a88709706e35383135ef409a25d0f8f9 Mon Sep 17 00:00:00 2001 From: Arthur Diniz Date: Sun, 12 Jul 2020 21:19:45 -0300 Subject: [PATCH 06/10] Replace manual setup command to Makefile Signed-off-by: Arthur Diniz --- .gitignore | 3 ++- Makefile | 40 ++++++++++++++++++++++++++++++++++++++++ README.md | 34 ++++++++++------------------------ 3 files changed, 52 insertions(+), 25 deletions(-) create mode 100644 Makefile diff --git a/.gitignore b/.gitignore index 1d93495..7644efa 100755 --- a/.gitignore +++ b/.gitignore @@ -7,5 +7,6 @@ one.spec one/__init__.pyc one/__main__.pyc one/one.pyc -env +.venv one_cli.egg-info +.pytest_cache diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..74282b2 --- /dev/null +++ b/Makefile @@ -0,0 +1,40 @@ +all: .venv install + + +.venv: + virtualenv -p /usr/bin/python3 .venv + + +install: .venv + .venv/bin/pip install --editable . + + +clean: + rm -rf .venv + rm -rf build + rm -rf dist + rm -rf one_cli.egg-info + rm -rf .pytest_cache + rm -rf __pycache__ + find -iname "*.pyc" -delete + + +run: install + .venv/bin/python cli.py $(filter-out $@,$(MAKECMDGOALS)) + + +build: .venv + .venv/bin/pip install -U PyInstaller + .venv/bin/pyinstaller --clean --hidden-import one.__main__ cli.py --onefile --noconsole -n one + + +.requirements-test-lint: + .venv/bin/pip install -r requirements-test.txt + + +test: .venv .requirements-test-lint + .venv/bin/pytest -v tests/ + + +flake8: .venv .requirements-test-lint + .venv/bin/flake8 . --count --max-complexity=10 --max-line-length=127 --statistics --exclude .venv \ No newline at end of file diff --git a/README.md b/README.md index 52c220a..cdce357 100755 --- a/README.md +++ b/README.md @@ -125,44 +125,30 @@ workspaces: #### Dependencies - Python 3 - -#### Python Virtual Environment -```bash -# Create environment -python3 -m venv env - -# To activate the environment -source env/bin/activate - -# When you finish you can exit typing -deactivate -``` +- Make +- virtualenv #### Install dependencies - ```bash -pip3 install --editable . -``` - -#### Run tests and lint -First make sure to install the test dependencies: -``` -pip3 install -r requirements-test.txt +make install ``` +#### Run tests and flake8 To run the test suite: ```bash -pytest -v tests/ +make test ``` -To run the lint check: +To run the flake8 lint check: ```bash -flake8 . --count --max-complexity=10 --max-line-length=127 --statistics --exclude env +make flake8 ``` #### Manualy generate binary +> Notice that when the command finishes successfully two folders will be generated in the project (**build** and **dist**). The CLI binary file can be found at **dist**. + ```bash -pyinstaller --clean --hidden-import one.__main__ cli.py --onefile --noconsole -n one +make build ``` ## Plugin System From cddbd79bde9fc1db70546f5d08892035b5f0c321 Mon Sep 17 00:00:00 2001 From: Arthur Diniz Date: Sun, 12 Jul 2020 21:21:10 -0300 Subject: [PATCH 07/10] Refactor init command in a controller Signed-off-by: Arthur Diniz --- one/commands/init.py | 35 ++----------------------- one/controller/init.py | 59 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 33 deletions(-) create mode 100644 one/controller/init.py diff --git a/one/commands/init.py b/one/commands/init.py index ab079a6..73cb2ce 100644 --- a/one/commands/init.py +++ b/one/commands/init.py @@ -1,38 +1,7 @@ import click -from PyInquirer import prompt -import yaml -from one.utils.prompt import style -from one.__init__ import CONFIG_FILE -from one.prompt.init import CREATION_QUESTION, IMAGE_QUESTIONS, WORKSPACE_QUESTIONS +from one.controller.init import InitController @click.command(help='Create config file for CLI in current directory.') def init(): - create_answer = prompt(CREATION_QUESTION, style=style) - create_workspace = create_answer['create'].lower() - workspaces = {} - if create_workspace == 'y' or not create_workspace: - image_answers = prompt(IMAGE_QUESTIONS, style=style) - images = { - 'terraform': image_answers['terraform'], - 'gsuite': image_answers['gsuite'], - 'azure': image_answers['azure'] - } - - while True: - workspace_answers = prompt(WORKSPACE_QUESTIONS, style=style) - if workspace_answers['assume_role'].lower() == 'y' or not workspace_answers['assume_role']: - assume_role = True - else: - assume_role = False - workspace = { - 'aws-role': workspace_answers['AWS_ROLE'], - 'aws-account-id': workspace_answers['AWS_ACCOUNT_ID'], - 'assume-role': assume_role - } - workspaces[workspace_answers['WORKSPACE']] = workspace - if workspace_answers['new_workspace'].lower() == 'n': - break - with open(CONFIG_FILE, 'w') as file: - content = {'images': images, 'workspaces': workspaces} - yaml.dump(content, file) + InitController().init() diff --git a/one/controller/init.py b/one/controller/init.py new file mode 100644 index 0000000..b984d7a --- /dev/null +++ b/one/controller/init.py @@ -0,0 +1,59 @@ +import yaml +from PyInquirer import prompt +from one.__init__ import CONFIG_FILE +from one.prompt.init import CREATION_QUESTION, IMAGE_QUESTIONS, WORKSPACE_QUESTIONS +from one.utils.prompt import style + + +class InitController(): + + workspaces = {} + images = {} + create_workspace = 'y' + + def __init__(self): + pass + + def init(self): + try: + self.prompt_create_workspace_questions() + + if self.create_workspace == 'y': + self.prompt_image_questions() + self.prompt_credential_questions() + self.write_config() + except KeyError: + raise SystemExit + + def prompt_create_workspace_questions(self): + answer = prompt(CREATION_QUESTION, style=style) + self.create_workspace = answer['create'].lower() + + def prompt_image_questions(self): + image_answers = prompt(IMAGE_QUESTIONS, style=style) + self.images = { + 'terraform': image_answers['terraform'], + 'azure': image_answers['azure'], + 'gsuite': image_answers['gsuite'] + } + + def prompt_credential_questions(self): + while True: + workspace_answers = prompt(WORKSPACE_QUESTIONS, style=style) + if workspace_answers['assume_role'].lower() == 'y': + assume_role = True + else: + assume_role = False + workspace = { + 'aws-role': workspace_answers['AWS_ROLE'], + 'aws-account-id': workspace_answers['AWS_ACCOUNT_ID'], + 'assume-role': assume_role + } + self.workspaces[workspace_answers['WORKSPACE']] = workspace + if workspace_answers['new_workspace'].lower() == 'n': + break + + def write_config(self): + with open(CONFIG_FILE, 'w') as file: + content = {'images': self.images, 'workspaces': self.workspaces} + yaml.dump(content, file) From edcb65761d1df29f10f3b56b768031b72f9a90ef Mon Sep 17 00:00:00 2001 From: Arthur Diniz Date: Sun, 12 Jul 2020 22:09:26 -0300 Subject: [PATCH 08/10] Refactor workspace command inside controller Signed-off-by: Arthur Diniz --- one/__init__.py | 1 + one/commands/workspace.py | 32 ++++--------------- one/controller/workspace.py | 63 +++++++++++++++++++++++++++++++++++++ one/utils/config.py | 12 ------- 4 files changed, 70 insertions(+), 38 deletions(-) create mode 100644 one/controller/workspace.py diff --git a/one/__init__.py b/one/__init__.py index ab3f741..cc9e29e 100755 --- a/one/__init__.py +++ b/one/__init__.py @@ -6,3 +6,4 @@ CLI_ROOT = home + '/.one' CONFIG_FILE = './one.yaml' +DEFAULT_WORKSPACE = '.one.workspace' diff --git a/one/commands/workspace.py b/one/commands/workspace.py index 492849a..95dc10f 100755 --- a/one/commands/workspace.py +++ b/one/commands/workspace.py @@ -1,7 +1,7 @@ import click -from one.utils.prompt import style -from PyInquirer import prompt -from one.utils.config import get_workspaces +from one.controller.workspace import WorkspaceController + +workspace_controller = WorkspaceController() @click.group(help='Manage workspaces.') @@ -10,30 +10,10 @@ def workspace(): @workspace.command(name='list', help='List all workspaces.') -def list_workspaces(): - workspaces = get_workspaces() - for workspace in workspaces: - click.echo('- ' + workspace) +def _list(): + workspace_controller._list() @workspace.command(help='Change environment variables to another workspace.') def change(): - workspaces_obj = [] - workspaces = get_workspaces() - for workspace in workspaces: - workspaces_obj.append({'name': workspace}) - - questions = [ - { - 'type': 'list', - 'message': 'Select workspace', - 'name': 'workspace', - 'choices': workspaces_obj - } - ] - - answers = prompt(questions, style=style) - - f = open('.one.workspace', 'w') - f.write('WORKSPACE=' + answers['workspace'] + '\n') - f.close() + workspace_controller.change() diff --git a/one/controller/workspace.py b/one/controller/workspace.py new file mode 100644 index 0000000..513896a --- /dev/null +++ b/one/controller/workspace.py @@ -0,0 +1,63 @@ +import yaml +import click +from PyInquirer import prompt +from os import path +from one.__init__ import CONFIG_FILE, DEFAULT_WORKSPACE +from one.utils.prompt import style + + +class WorkspaceController(): + + workspaces = [] + + def __init__(self): + pass + + def _list(self): + self.get_workspaces() + for workspace in self.workspaces: + click.echo('- ' + workspace) + + def prompt_workspaces_list(self, workspaces_obj): + questions = [ + { + 'type': 'list', + 'message': 'Select workspace', + 'name': 'workspace', + 'choices': workspaces_obj + } + ] + + answers = prompt(questions, style=style) + return answers['workspace'] + + def change(self): + workspaces_obj = self.format_workspace_list() + if workspaces_obj: + try: + selected_workspace = self.prompt_workspaces_list(workspaces_obj) + content = 'WORKSPACE=%s\n' % (selected_workspace) + self.write_default_workspace(content) + except IndexError: + raise SystemExit + + def get_workspaces(self): + if path.exists(CONFIG_FILE): + with open(CONFIG_FILE) as file: + docs = yaml.load(file, Loader=yaml.BaseLoader) + for workspace_key in docs['workspaces'].keys(): + self.workspaces.append(workspace_key) + file.close() + + return self.workspaces + + def format_workspace_list(self): + workspaces_obj = [] + self.get_workspaces() + for workspace in self.workspaces: + workspaces_obj.append({'name': workspace}) + return workspaces_obj + + def write_default_workspace(self, content): + with open(DEFAULT_WORKSPACE, 'w') as file: + file.write(content) diff --git a/one/utils/config.py b/one/utils/config.py index 7cb6c1b..1b66a23 100755 --- a/one/utils/config.py +++ b/one/utils/config.py @@ -30,18 +30,6 @@ def get_config_value(key, default=None): return value -def get_workspaces(): - workspaces = [] - if path.exists(CONFIG_FILE): - with open(CONFIG_FILE) as file: - docs = yaml.load(file, Loader=yaml.BaseLoader) - for workspace_key in docs['workspaces'].keys(): - workspaces.append(workspace_key) - file.close() - - return workspaces - - def get_workspace_value(workspace_name, variable, default=None): value = default if path.exists(CONFIG_FILE): From c4abcd3ab273228000b897282342e85796595857 Mon Sep 17 00:00:00 2001 From: Arthur Diniz Date: Sun, 12 Jul 2020 22:10:51 -0300 Subject: [PATCH 09/10] Adds one.yaml and .one.workspace to gitignore Signed-off-by: Arthur Diniz --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 7644efa..a02ebce 100755 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,6 @@ one/one.pyc .venv one_cli.egg-info .pytest_cache + +one.yaml +.one.workspace \ No newline at end of file From 3829a6258f09a0e43b8a2d8c134de296f3b6681b Mon Sep 17 00:00:00 2001 From: Arthur Diniz Date: Sun, 12 Jul 2020 22:11:57 -0300 Subject: [PATCH 10/10] Add one.spec to makefile Signed-off-by: Arthur Diniz --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 74282b2..ee02eaa 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,7 @@ clean: rm -rf one_cli.egg-info rm -rf .pytest_cache rm -rf __pycache__ + rm -rf one.spec find -iname "*.pyc" -delete