From 148f4c7f63ec2a743fa875cf64c7be5fdb8d801f Mon Sep 17 00:00:00 2001 From: Peter Burkholder Date: Fri, 15 Aug 2025 13:58:23 -0400 Subject: [PATCH 1/7] Works to query table, etc --- cost-estimator/air_table.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100755 cost-estimator/air_table.py diff --git a/cost-estimator/air_table.py b/cost-estimator/air_table.py new file mode 100755 index 00000000..9753586f --- /dev/null +++ b/cost-estimator/air_table.py @@ -0,0 +1,21 @@ +#! /usr/bin/env python3 + +import os +import pprint + +from pyairtable import Api + +api = Api(os.environ['AIRTABLE_API_KEY']) + + +QUOTE_BASE_ID='appprdUNzFPO9avLd' +RESOURCE_ENTRY_TABLE_ID='tbl5fX9qzivwgMnD2' +RESOURCE_PRICING_TABLE_ID='tblr5evoP1pKGcUxl' + +resource_prices = api.table(QUOTE_BASE_ID, RESOURCE_PRICING_TABLE_ID) + +price_dict = {} +for rp in resource_prices.all(fields=['Name']): + print(rp['id']) + print(rp['fields']['Name']) + From d37ef371537f8bdcda4594c676e48671a58cce36 Mon Sep 17 00:00:00 2001 From: Peter Burkholder Date: Fri, 15 Aug 2025 15:43:26 -0400 Subject: [PATCH 2/7] Populate resource dict, works --- cost-estimator/air_table.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/cost-estimator/air_table.py b/cost-estimator/air_table.py index 9753586f..712336d7 100755 --- a/cost-estimator/air_table.py +++ b/cost-estimator/air_table.py @@ -11,11 +11,19 @@ QUOTE_BASE_ID='appprdUNzFPO9avLd' RESOURCE_ENTRY_TABLE_ID='tbl5fX9qzivwgMnD2' RESOURCE_PRICING_TABLE_ID='tblr5evoP1pKGcUxl' +RESOURCE_SUMMARY_TABLE_ID='tblCj7JYRsYlqtruU' resource_prices = api.table(QUOTE_BASE_ID, RESOURCE_PRICING_TABLE_ID) price_dict = {} for rp in resource_prices.all(fields=['Name']): - print(rp['id']) - print(rp['fields']['Name']) + price_dict[rp['fields']['Name']] = (rp['id']) + +pprint.pp(price_dict) + +resource_summaries = api.table(QUOTE_BASE_ID, RESOURCE_SUMMARY_TABLE_ID) +pprint.pp(resource_summaries) + +for records in resource_summaries.iterate(page_size=100, max_records=1000): + print(records) \ No newline at end of file From abd682e3fc908244438ebebd8386ce0de9e74762 Mon Sep 17 00:00:00 2001 From: Peter Burkholder Date: Fri, 15 Aug 2025 16:26:31 -0400 Subject: [PATCH 3/7] Updates summary table in AT --- cost-estimator/estimate-costs.py | 36 +++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/cost-estimator/estimate-costs.py b/cost-estimator/estimate-costs.py index ea6f14ad..6b2ecac6 100755 --- a/cost-estimator/estimate-costs.py +++ b/cost-estimator/estimate-costs.py @@ -14,6 +14,7 @@ import argparse import math import re +import pyairtable class AWSResource: @@ -468,7 +469,8 @@ def test_authenticated(service): class Account: - def __init__(self, orgs, space_names): + def __init__(self, name, orgs, space_names): + self.name = name self.org_names = orgs self.space_names = space_names if space_names else [] @@ -486,6 +488,7 @@ def __init__(self, orgs, space_names): self.input_workbook_file = None self.output_workbook_file = None self.reporter = Reporter() + self.airtable = Airtable() def report_orgs(self): for org_name in self.org_names: @@ -536,6 +539,15 @@ def report_summary(self, reporter): for key, value in sorted(self.es_total_instance_plans.items()): reporter.log(f" {key}: {value}") + def upload_airtable_report(self, airtable, account_name): + headline = f"Cost estimate for org: {self.org_names}" + if len(self.space_names) > 0: + headline += f", spaces: {self.space_names}" + + airtable.summary_table.create( + {"Source": "cost-estimator", "Account": account_name, "Description": headline} + ) + def generate_cost_estimate(self, reporter): estimate_map = { # Usage @@ -631,6 +643,18 @@ def generate_cost_estimate(self, reporter): workbook.save(filename=self.output_workbook_file) print(f"Saved cost estimate to: {self.output_workbook_file}") +class Airtable: + QUOTE_BASE_ID='appprdUNzFPO9avLd' + RESOURCE_ENTRY_TABLE_ID='tbl5fX9qzivwgMnD2' + RESOURCE_PRICING_TABLE_ID='tblr5evoP1pKGcUxl' + RESOURCE_SUMMARY_TABLE_ID='tblCj7JYRsYlqtruU' + + def __init__(self): + self.api = pyairtable.Api(os.environ['AIRTABLE_API_KEY']) + self.summary_table = self.api.table(self.QUOTE_BASE_ID, self.RESOURCE_SUMMARY_TABLE_ID) + + + class Reporter: """ @@ -725,10 +749,11 @@ def main(): print("space names only allowed when specifying a single organization") exit(1) + account_name = org_names[-1] if args.account_name: - output_file = args.account_name + ".xlsx" - else: - output_file = org_names[-1] + ".xlsx" + account_name = args.account_name + output_file = account_name + ".xlsx" + if not os.path.exists(cost_estimate_file): print( @@ -744,7 +769,7 @@ def main(): print(f"Info: Authenticated, starting...", file=sys.stderr) - acct = Account(orgs=org_names, space_names=space_names) + acct = Account(name=account_name,orgs=org_names, space_names=space_names) acct.report_orgs() if len(org_names) > 1: acct.report_summary(acct.reporter) @@ -752,6 +777,7 @@ def main(): acct.input_workbook_file = cost_estimate_file acct.output_workbook_file = output_file acct.generate_cost_estimate(acct.reporter) + acct.upload_airtable_report(acct.airtable, acct.name) if __name__ == "__main__": From 2101ebc410ccdfb8e405210c721d55a01625d839 Mon Sep 17 00:00:00 2001 From: Peter Burkholder Date: Fri, 15 Aug 2025 17:14:14 -0400 Subject: [PATCH 4/7] No XLS write, batches input for AT --- cost-estimator/estimate-costs.py | 70 ++++++++++++++++++++++++++++++-- 1 file changed, 66 insertions(+), 4 deletions(-) diff --git a/cost-estimator/estimate-costs.py b/cost-estimator/estimate-costs.py index 6b2ecac6..8fd3c3a5 100755 --- a/cost-estimator/estimate-costs.py +++ b/cost-estimator/estimate-costs.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import pprint import subprocess import sys import os.path @@ -544,9 +545,64 @@ def upload_airtable_report(self, airtable, account_name): if len(self.space_names) > 0: headline += f", spaces: {self.space_names}" - airtable.summary_table.create( + summary_record_id = airtable.summary_table.create( {"Source": "cost-estimator", "Account": account_name, "Description": headline} - ) + )['id'] + print("Created AT Summary Record: ", summary_record_id) + + price_lookup= airtable.price_dict() + print("Fetched AT Price Records: ", len(price_lookup)) + + airtable_batch = [] + + # memory-quota + if len(self.space_names) == 0: + at_memory_quota = self.memory_quota / 1024 + else: + # If we are producing an estimate for a set of space(s), then the memory usage is + # whatever memory is used by those spaces. Round up the memory usage to the nearest + # integer because we charge for memory on a per GB basis, so any partial use of a GB + # should be treated as a whole GB for accounting purposes + at_memory_quota = math.ceil( + self.memory_usage / 1024 + ) + airtable_batch.append({ + "Resource Summary": [summary_record_id], + "Resource Pricelist": [price_lookup['memory-quota']], + "Units": at_memory_quota + }) + # rds-storage + if self.rds_total_allocation > 0: + airtable_batch.append({ + "Resource Summary": [summary_record_id], + "Resource Pricelist": [price_lookup['rds-storage']], + "Units": math.ceil(self.rds_total_allocation) + }) + # s3-storage + if self.s3_total_storage > 0: + airtable_batch.append({ + "Resource Summary": [summary_record_id], + "Resource Pricelist": [price_lookup['s3-storage']], + "Units": math.ceil(self.s3_total_storage / ( 1024 * 1024 * 1024 )) + }) + + # es-storage + if self.es_total_volume_storage > 0: + airtable_batch.append({ + "Resource Summary": [summary_record_id], + "Resource Pricelist": [price_lookup['es-storage']], + "Units": math.ceil(self.es_total_volume_storage) + }) + # rds-plans + if len(self.rds_total_instance_plans) > 0: + for plan, units in sorted(self.rds_total_instance_plans.items()): + airtable_batch.append({ + "Resource Summary": [summary_record_id], + "Resource Pricelist": [price_lookup[plan]], + "Units": units + }) + + pprint.pp(airtable_batch) def generate_cost_estimate(self, reporter): estimate_map = { @@ -652,8 +708,14 @@ class Airtable: def __init__(self): self.api = pyairtable.Api(os.environ['AIRTABLE_API_KEY']) self.summary_table = self.api.table(self.QUOTE_BASE_ID, self.RESOURCE_SUMMARY_TABLE_ID) - + self.pricing_table = self.api.table(self.QUOTE_BASE_ID, self.RESOURCE_PRICING_TABLE_ID) + self.record_table = self.api.table(self.QUOTE_BASE_ID, self.RESOURCE_ENTRY_TABLE_ID) + def price_dict(self): + price_dict = {} + for rp in self.pricing_table.all(fields=['Name']): + price_dict[rp['fields']['Name']] = (rp['id']) + return price_dict class Reporter: @@ -776,7 +838,7 @@ def main(): acct.input_workbook_file = cost_estimate_file acct.output_workbook_file = output_file - acct.generate_cost_estimate(acct.reporter) +# acct.generate_cost_estimate(acct.reporter) acct.upload_airtable_report(acct.airtable, acct.name) From 0e561ea43c425bd866d2cbe48ff97a089e3360f8 Mon Sep 17 00:00:00 2001 From: Peter Burkholder Date: Fri, 15 Aug 2025 17:25:39 -0400 Subject: [PATCH 5/7] Batch commit works --- cost-estimator/estimate-costs.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cost-estimator/estimate-costs.py b/cost-estimator/estimate-costs.py index 8fd3c3a5..3117f7a0 100755 --- a/cost-estimator/estimate-costs.py +++ b/cost-estimator/estimate-costs.py @@ -601,8 +601,10 @@ def upload_airtable_report(self, airtable, account_name): "Resource Pricelist": [price_lookup[plan]], "Units": units }) + result = airtable.resource_table.batch_create(airtable_batch) + print("AT batch result", result) + - pprint.pp(airtable_batch) def generate_cost_estimate(self, reporter): estimate_map = { @@ -709,7 +711,7 @@ def __init__(self): self.api = pyairtable.Api(os.environ['AIRTABLE_API_KEY']) self.summary_table = self.api.table(self.QUOTE_BASE_ID, self.RESOURCE_SUMMARY_TABLE_ID) self.pricing_table = self.api.table(self.QUOTE_BASE_ID, self.RESOURCE_PRICING_TABLE_ID) - self.record_table = self.api.table(self.QUOTE_BASE_ID, self.RESOURCE_ENTRY_TABLE_ID) + self.resource_table = self.api.table(self.QUOTE_BASE_ID, self.RESOURCE_ENTRY_TABLE_ID) def price_dict(self): price_dict = {} From ea0abd0b2ac3d905592163c0e6751c6fc6c4a058 Mon Sep 17 00:00:00 2001 From: Peter Burkholder Date: Fri, 15 Aug 2025 17:48:43 -0400 Subject: [PATCH 6/7] Does Redis and ES now too --- cost-estimator/estimate-costs.py | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/cost-estimator/estimate-costs.py b/cost-estimator/estimate-costs.py index 3117f7a0..4e71173c 100755 --- a/cost-estimator/estimate-costs.py +++ b/cost-estimator/estimate-costs.py @@ -540,18 +540,17 @@ def report_summary(self, reporter): for key, value in sorted(self.es_total_instance_plans.items()): reporter.log(f" {key}: {value}") - def upload_airtable_report(self, airtable, account_name): + def upload_airtable_estimate(self, airtable, account_name): headline = f"Cost estimate for org: {self.org_names}" if len(self.space_names) > 0: headline += f", spaces: {self.space_names}" + + price_lookup= airtable.price_dict() + print(f"Fetched {len(price_lookup)} Price Records from AirTable") summary_record_id = airtable.summary_table.create( {"Source": "cost-estimator", "Account": account_name, "Description": headline} )['id'] - print("Created AT Summary Record: ", summary_record_id) - - price_lookup= airtable.price_dict() - print("Fetched AT Price Records: ", len(price_lookup)) airtable_batch = [] @@ -601,9 +600,24 @@ def upload_airtable_report(self, airtable, account_name): "Resource Pricelist": [price_lookup[plan]], "Units": units }) + # redis-plans + if len(self.redis_total_instance_plans) > 0: + for plan, units in sorted(self.redis_total_instance_plans.items()): + airtable_batch.append({ + "Resource Summary": [summary_record_id], + "Resource Pricelist": [price_lookup[plan]], + "Units": units + }) + # es-plans + if len(self.es_total_instance_plans) > 0: + for plan, units in sorted(self.es_total_instance_plans.items()): + airtable_batch.append({ + "Resource Summary": [summary_record_id], + "Resource Pricelist": [price_lookup[plan]], + "Units": units + }) result = airtable.resource_table.batch_create(airtable_batch) - print("AT batch result", result) - + print(f"Created {len(result)} AirTable records") def generate_cost_estimate(self, reporter): @@ -840,8 +854,8 @@ def main(): acct.input_workbook_file = cost_estimate_file acct.output_workbook_file = output_file -# acct.generate_cost_estimate(acct.reporter) - acct.upload_airtable_report(acct.airtable, acct.name) + acct.generate_cost_estimate(acct.reporter) + acct.upload_airtable_estimate(acct.airtable, acct.name) if __name__ == "__main__": From 3dd4eadfa6a47e34bbb33af115853135497cc8da Mon Sep 17 00:00:00 2001 From: Peter Burkholder Date: Fri, 15 Aug 2025 17:49:03 -0400 Subject: [PATCH 7/7] No longer need example airtable script --- cost-estimator/air_table.py | 29 ----------------------------- 1 file changed, 29 deletions(-) delete mode 100755 cost-estimator/air_table.py diff --git a/cost-estimator/air_table.py b/cost-estimator/air_table.py deleted file mode 100755 index 712336d7..00000000 --- a/cost-estimator/air_table.py +++ /dev/null @@ -1,29 +0,0 @@ -#! /usr/bin/env python3 - -import os -import pprint - -from pyairtable import Api - -api = Api(os.environ['AIRTABLE_API_KEY']) - - -QUOTE_BASE_ID='appprdUNzFPO9avLd' -RESOURCE_ENTRY_TABLE_ID='tbl5fX9qzivwgMnD2' -RESOURCE_PRICING_TABLE_ID='tblr5evoP1pKGcUxl' -RESOURCE_SUMMARY_TABLE_ID='tblCj7JYRsYlqtruU' - -resource_prices = api.table(QUOTE_BASE_ID, RESOURCE_PRICING_TABLE_ID) - -price_dict = {} -for rp in resource_prices.all(fields=['Name']): - price_dict[rp['fields']['Name']] = (rp['id']) - - -pprint.pp(price_dict) - -resource_summaries = api.table(QUOTE_BASE_ID, RESOURCE_SUMMARY_TABLE_ID) -pprint.pp(resource_summaries) - -for records in resource_summaries.iterate(page_size=100, max_records=1000): - print(records) \ No newline at end of file