diff --git a/.github/workflows/build-map-data.yaml b/.github/workflows/build-map-data.yaml index 5a42a3a..2eb56a0 100644 --- a/.github/workflows/build-map-data.yaml +++ b/.github/workflows/build-map-data.yaml @@ -14,11 +14,13 @@ on: default: '' jobs: - build: + download: runs-on: ubuntu-latest - env: - PUSHOVER_API_KEY: ${{ secrets.PUSHOVER_API_KEY }} - PUSHOVER_USER_KEY: ${{ secrets.PUSHOVER_USER_KEY }} + outputs: + exit_code: ${{ steps.dl-shp.outputs.exit_code }} + shp_year: ${{ steps.info.outputs.shp_year }} + state_shp: ${{ steps.info.outputs.state_shp }} + county_shp: ${{ steps.info.outputs.county_shp }} steps: - name: Checkout @@ -38,6 +40,38 @@ jobs: run: | python data-raw/scripts/shapefiles.py ${{ inputs.year }} + - name: Save shapefile info + id: info + run: | + echo "shp_year=${{ env.shp_year }}" >> "$GITHUB_OUTPUT" + echo "state_shp=${{ env.state_shp }}" >> "$GITHUB_OUTPUT" + echo "county_shp=${{ env.county_shp }}" >> "$GITHUB_OUTPUT" + + - name: Upload shapefiles + if: steps.dl-shp.outputs.exit_code == '0' + uses: actions/upload-artifact@v4 + with: + name: shapefiles + path: data-raw/shapefiles/${{ env.shp_year }} + + process: + runs-on: ubuntu-latest + needs: download + if: needs.download.outputs.exit_code == '0' + outputs: + pr_url: ${{ steps.info.outputs.pr_url }} + pr_number: ${{ steps.info.outputs.pr_number }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Download shapefiles + uses: actions/download-artifact@v4 + with: + name: shapefiles + path: data-raw/shapefiles/${{ needs.download.outputs.shp_year }} + - name: Setup R uses: r-lib/actions/setup-r@v2 @@ -48,9 +82,9 @@ jobs: - name: Modify shapefiles env: - STATE_SHP: ${{ env.state_shp }} - COUNTY_SHP: ${{ env.county_shp }} - YEAR: ${{ env.shp_year }} + STATE_SHP: ${{ needs.download.outputs.state_shp }} + COUNTY_SHP: ${{ needs.download.outputs.county_shp }} + YEAR: ${{ needs.download.outputs.shp_year }} run: | input_dir <- file.path("data-raw", "shapefiles", Sys.getenv("YEAR")) output_dir <- file.path("inst", "extdata", Sys.getenv("YEAR")) @@ -80,7 +114,7 @@ jobs: - name: Determine pull request parameters id: pr-params env: - YEAR: ${{ env.shp_year }} + YEAR: ${{ needs.download.outputs.shp_year }} run: | echo "branch_name=data-update/$YEAR" >> "$GITHUB_OUTPUT" echo "pr_title=Add $YEAR map data" >> "$GITHUB_OUTPUT" @@ -100,7 +134,7 @@ jobs: token: ${{ secrets.BOT_PAT }} author: ${{ secrets.BOT_USER }} committer: ${{ secrets.BOT_USER }} - commit-message: "[automated] Add ${{ env.shp_year }} map data based on available shapefiles" + commit-message: "[automated] Add ${{ needs.download.outputs.shp_year }} map data based on available shapefiles" branch: ${{ steps.pr-params.outputs.branch_name }} title: ${{ steps.pr-params.outputs.pr_title }} body: ${{ steps.pr-body.outputs.result }} @@ -109,17 +143,45 @@ jobs: labels: data update delete-branch: true + - name: Save PR info + id: info + run: | + echo "pr_url=${{ steps.open-pr.outputs.pull-request-url }}" >> "$GITHUB_OUTPUT" + echo "pr_number=${{ steps.open-pr.outputs.pull-request-number }}" >> "$GITHUB_OUTPUT" + + notify: + runs-on: ubuntu-latest + needs: [download, process] + if: always() + env: + PUSHOVER_API_KEY: ${{ secrets.PUSHOVER_API_KEY }} + PUSHOVER_USER_KEY: ${{ secrets.PUSHOVER_USER_KEY }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.9' + cache: 'pip' + + - name: Install Python dependencies + run: pip install -r data-raw/scripts/requirements.txt + - name: Send success notification + if: needs.download.outputs.exit_code == '0' && needs.process.result == 'success' run: | - python data-raw/scripts/pushover.py "✅ usmapdata has updated its data files, a PR review is needed: PR #${{ steps.open-pr.outputs.pull-request-number }}" + python data-raw/scripts/pushover.py "✅ usmapdata has updated its data files, a PR review is needed: PR #${{ needs.process.outputs.pr_number }}" - name: Send data not found notification - if: ${{ failure() && steps.dl-shp.outputs.exit_code == '404' }} + if: needs.download.outputs.exit_code == '404' run: | - python data-raw/scripts/pushover.py "⚠️ usmapdata failed to find map data files for ${{ env.shp_year }}." "LOW" + python data-raw/scripts/pushover.py "⚠️ usmapdata failed to find map data files for ${{ needs.download.outputs.shp_year }}." "LOW" - name: Send failure notification - if: ${{ failure() && steps.dl-shp.outputs.exit_code != '404' }} + if: needs.download.outputs.exit_code != '0' && needs.download.outputs.exit_code != '404' run: | - python data-raw/scripts/pushover.py "❌ usmapdata failed to update map data files. (error: ${{ steps.dl-shp.outputs.exit_code }})" "LOW" + python data-raw/scripts/pushover.py "❌ usmapdata failed to update map data files. (error: ${{ needs.download.outputs.exit_code }})" "LOW" diff --git a/data-raw/certs/www2-census-gov-chain.pem b/data-raw/certs/www2-census-gov-chain.pem deleted file mode 100644 index 989eae6..0000000 --- a/data-raw/certs/www2-census-gov-chain.pem +++ /dev/null @@ -1,92 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIEUDCCA/egAwIBAgIQaDrV1+TMjrWOBiXILF9rhzAKBggqhkjOPQQDAjBSMQsw -CQYDVQQGEwJVUzEZMBcGA1UECgwQQ0xPVURGTEFSRSwgSU5DLjEoMCYGA1UEAwwf -Q2xvdWRmbGFyZSBUTFMgSXNzdWluZyBFQ0MgQ0EgMTAeFw0yNTA0MDMxNjQzMzda -Fw0yNjA0MDMxNjQ4MzVaMBoxGDAWBgNVBAMMD3d3dzIuY2Vuc3VzLmdvdjBZMBMG -ByqGSM49AgEGCCqGSM49AwEHA0IABPln+y1wtdnOs3sPFcf6B5cE5I92d8/F4jyd -pPszdhAxeOrBvu8hdtNAfBIEQahXw25tHyJ8a5PHQPLnW+wy3MWjggLlMIIC4TAM -BgNVHRMBAf8EAjAAMB8GA1UdIwQYMBaAFJzECXJHGBd7pxqJs5I11eEDjP6SMGwG -CCsGAQUFBwEBBGAwXjA5BggrBgEFBQcwAoYtaHR0cDovL2kuY2YtYi5zc2wuY29t -L0Nsb3VkZmxhcmUtVExTLUktRTEuY2VyMCEGCCsGAQUFBzABhhVodHRwOi8vby5j -Zi1iLnNzbC5jb20wGgYDVR0RBBMwEYIPd3d3Mi5jZW5zdXMuZ292MCMGA1UdIAQc -MBowCAYGZ4EMAQIBMA4GDCsGAQQBgqkwAQMBATAdBgNVHSUEFjAUBggrBgEFBQcD -AgYIKwYBBQUHAwEwPgYDVR0fBDcwNTAzoDGgL4YtaHR0cDovL2MuY2YtYi5zc2wu -Y29tL0Nsb3VkZmxhcmUtVExTLUktRTEuY3JsMA4GA1UdDwEB/wQEAwIHgDAPBgkr -BgEEAYLaSywEAgUAMIIBfwYKKwYBBAHWeQIEAgSCAW8EggFrAWkAdwAOV5S8866p -PjMbLJkHs/eQ35vCPXEyJd0hqSWsYcVOIQAAAZX8ko+bAAAEAwBIMEYCIQCNc92s -rBx9YkFERIfu2ZoldiZHEqfkItZ818VIrEVNDAIhAOP3IF7IXgObk2Rr1vGTdO3R -Jmrc5rFeR3TE4oCzOWqwAHcASZybad4dfOz8Nt7Nh2SmuFuvCoeAGdFVUvvp6ynd -+MMAAAGV/JKPmQAABAMASDBGAiEAp9V1pHP+SKUonoy7foADCOfCdTpc2nj/Yq/V -2lAT2nICIQCK577pOoUWXLiq+1TKnhg/lw0ypV6Sondkg6+k0t6/XgB1AMs49xWJ -fIShRF9bwd37yW7ymlnNRwppBYWwyxTDFFjnAAABlfySj+YAAAQDAEYwRAIgHUDA -Zr6xm1Lzdk5EFSxdT6RWmR15dTuq/OOUWzO3So0CIGV9yT4exupT8q0Q+b14ybVc -oXGru2Kbm61LxMqgvcADMAoGCCqGSM49BAMCA0cAMEQCID4A0hXYQDqtjZHwDuWl -YCfg02kAhRobK0bOSjqEwUr6AiAJnCOWKa1qK8fu143WgcLJR11/Aj3XGBWINcJd -0xLtuA== ------END CERTIFICATE----- ------BEGIN CERTIFICATE----- -MIIC5DCCAmqgAwIBAgIQLD+iaS9BE707f+W2BLSdTTAKBggqhkjOPQQDAzBPMQsw -CQYDVQQGEwJVUzEYMBYGA1UECgwPU1NMIENvcnBvcmF0aW9uMSYwJAYDVQQDDB1T -U0wuY29tIFRMUyBUcmFuc2l0IEVDQyBDQSBSMjAeFw0yMzEwMzExNzE3NDlaFw0z -MzEwMjgxNzE3NDhaMFIxCzAJBgNVBAYTAlVTMRkwFwYDVQQKDBBDTE9VREZMQVJF -LCBJTkMuMSgwJgYDVQQDDB9DbG91ZGZsYXJlIFRMUyBJc3N1aW5nIEVDQyBDQSAx -MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEByHHIHytNSzTS+F3JA7hHMDGd2cp -cY9i3MLTKmE6DJTKc6JwvW50pwKodvd2Qj4RAAy2jSejsVgw5jeh6syt3KOCASMw -ggEfMBIGA1UdEwEB/wQIMAYBAf8CAQAwHwYDVR0jBBgwFoAUMqLH2FiL/3/APPJV -aTPszswfvJcwSAYIKwYBBQUHAQEEPDA6MDgGCCsGAQUFBzAChixodHRwOi8vY2Vy -dC5zc2wuY29tL1NTTC5jb20tVExTLVQtRUNDLVIyLmNlcjARBgNVHSAECjAIMAYG -BFUdIAAwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUFBwMBMD0GA1UdHwQ2MDQw -MqAwoC6GLGh0dHA6Ly9jcmxzLnNzbC5jb20vU1NMLmNvbS1UTFMtVC1FQ0MtUjIu -Y3JsMB0GA1UdDgQWBBScxAlyRxgXe6caibOSNdXhA4z+kjAOBgNVHQ8BAf8EBAMC -AYYwCgYIKoZIzj0EAwMDaAAwZQIxAL0Sk3RweR6uG1aSHF3JgHQptubP9xoZyUmz -HSa+SSdY5wTGSx5qAowrLPCpLio2PAIwXQGgYzf5QzD/1Bsu87WrUcIVtLixr5KQ -wKBaFAyIJ7OOiWgW0HV/NA1UeuSe0zmN ------END CERTIFICATE----- ------BEGIN CERTIFICATE----- -MIID0DCCArigAwIBAgIRAK2NLfZGgaDTZEfqqU+ic8EwDQYJKoZIhvcNAQELBQAw -ezELMAkGA1UEBhMCR0IxGzAZBgNVBAgMEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4G -A1UEBwwHU2FsZm9yZDEaMBgGA1UECgwRQ29tb2RvIENBIExpbWl0ZWQxITAfBgNV -BAMMGEFBQSBDZXJ0aWZpY2F0ZSBTZXJ2aWNlczAeFw0yNDA2MjEwMDAwMDBaFw0y -ODEyMzEyMzU5NTlaME8xCzAJBgNVBAYTAlVTMRgwFgYDVQQKDA9TU0wgQ29ycG9y -YXRpb24xJjAkBgNVBAMMHVNTTC5jb20gVExTIFRyYW5zaXQgRUNDIENBIFIyMHYw -EAYHKoZIzj0CAQYFK4EEACIDYgAEZOd9mQNTXJEe6vjYI62hvyziY4nvKGj27dfw -7Ktorncr5HaXG1Dr21koLW+4NrmrjZfKTCKe7onZAj/9enM6kI0rzC86N4PaDbQt -RRtzcgllX3ghPeeLZj9H/Qkp1hQPo4IBJzCCASMwHwYDVR0jBBgwFoAUoBEKIz6W -8Qfs4q8p74Klf9AwpLQwHQYDVR0OBBYEFDKix9hYi/9/wDzyVWkz7M7MH7yXMA4G -A1UdDwEB/wQEAwIBhjASBgNVHRMBAf8ECDAGAQH/AgEBMB0GA1UdJQQWMBQGCCsG -AQUFBwMBBggrBgEFBQcDAjAjBgNVHSAEHDAaMAgGBmeBDAECATAOBgwrBgEEAYKp -MAEDAQEwQwYDVR0fBDwwOjA4oDagNIYyaHR0cDovL2NybC5jb21vZG9jYS5jb20v -QUFBQ2VydGlmaWNhdGVTZXJ2aWNlcy5jcmwwNAYIKwYBBQUHAQEEKDAmMCQGCCsG -AQUFBzABhhhodHRwOi8vb2NzcC5jb21vZG9jYS5jb20wDQYJKoZIhvcNAQELBQAD -ggEBAB4oL4ChKaKGZVZK8uAXjj8wvFdm45uvhU/t14QeH5bwETeKiQQXBga4/Nyz -zvpfuoEycantX+tHl/muwpmuHT0Z6IKYoICaMxOIktcTF4qHvxQW2WItHjOglrTj -qlXJXVL+3HCO60TEloSX8eUGsqfLQkc//z3Lb4gz117+fkDbnPt8+2REq3SCvaAG -hlh/lWWfHqTAiHed/qqzBSYqqvfjNlhIfXnPnhfAv/PpOUO1PmxCEAEYrg+VoS+O -+EBd1zkT0V7CfrPpj30cAMs2h+k4pPMwcLuB3Ku4TncBTRyt5K0gbJ3pQ0Rk9Hmu -wOz5QAZ+2n1q4TlApJzBfwFrCDg= ------END CERTIFICATE----- ------BEGIN CERTIFICATE----- -MIIEMjCCAxqgAwIBAgIBATANBgkqhkiG9w0BAQUFADB7MQswCQYDVQQGEwJHQjEb -MBkGA1UECAwSR3JlYXRlciBNYW5jaGVzdGVyMRAwDgYDVQQHDAdTYWxmb3JkMRow -GAYDVQQKDBFDb21vZG8gQ0EgTGltaXRlZDEhMB8GA1UEAwwYQUFBIENlcnRpZmlj -YXRlIFNlcnZpY2VzMB4XDTA0MDEwMTAwMDAwMFoXDTI4MTIzMTIzNTk1OVowezEL -MAkGA1UEBhMCR0IxGzAZBgNVBAgMEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UE -BwwHU2FsZm9yZDEaMBgGA1UECgwRQ29tb2RvIENBIExpbWl0ZWQxITAfBgNVBAMM -GEFBQSBDZXJ0aWZpY2F0ZSBTZXJ2aWNlczCCASIwDQYJKoZIhvcNAQEBBQADggEP -ADCCAQoCggEBAL5AnfRu4ep2hxxNRUSOvkbIgwadwSr+GB+O5AL686tdUIoWMQua -BtDFcCLNSS1UY8y2bmhGC1Pqy0wkwLxyTurxFa70VJoSCsN6sjNg4tqJVfMiWPPe -3M/vg4aijJRPn2jymJBGhCfHdr/jzDUsi14HZGWCwEiwqJH5YZ92IFCokcdmtet4 -YgNW8IoaE+oxox6gmf049vYnMlhvB/VruPsUK6+3qszWY19zjNoFmag4qMsXeDZR -rOme9Hg6jc8P2ULimAyrL58OAd7vn5lJ8S3frHRNG5i1R8XlKdH5kBjHYpy+g8cm -ez6KJcfA3Z3mNWgQIJ2P2N7Sw4ScDV7oL8kCAwEAAaOBwDCBvTAdBgNVHQ4EFgQU -oBEKIz6W8Qfs4q8p74Klf9AwpLQwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQF -MAMBAf8wewYDVR0fBHQwcjA4oDagNIYyaHR0cDovL2NybC5jb21vZG9jYS5jb20v -QUFBQ2VydGlmaWNhdGVTZXJ2aWNlcy5jcmwwNqA0oDKGMGh0dHA6Ly9jcmwuY29t -b2RvLm5ldC9BQUFDZXJ0aWZpY2F0ZVNlcnZpY2VzLmNybDANBgkqhkiG9w0BAQUF -AAOCAQEACFb8AvCb6P+k+tZ7xkSAzk/ExfYAWMymtrwUSWgEdujm7l3sAg9g1o1Q -GE8mTgHj5rCl7r+8dFRBv/38ErjHT1r0iWAFf2C3BUrz9vHCv8S5dIa2LX1rzNLz -Rt0vxuBqw8M0Ayx9lt1awg6nCpnBBYurDC/zXDrPbDdVCYfeU0BsWO/8tqtlbgT2 -G9w84FoVxp7Z8VlIMCFlA2zs6SFz7JsDoeA3raAVGI/6ugLOpyypEBMs1OUIJqsi -l2D4kF501KKaU73yqWjgom7C12yxow+ev+to51byrvLjKzg6CYG1a4XXvi3tPxq3 -smPi9WIsgtRqAEFQ8TmDn5XpNpaYbg== ------END CERTIFICATE----- diff --git a/data-raw/scripts/config.ini b/data-raw/scripts/config.ini index f47f894..629dfae 100644 --- a/data-raw/scripts/config.ini +++ b/data-raw/scripts/config.ini @@ -1,6 +1,5 @@ [shapefiles] url = https://www2.census.gov/geo/tiger/GENZ{year}/shp/cb_{year}_us_{entity}_{res}.zip -cert = www2-census-gov-chain.pem current_year = 2024 entities = state,county res = 20m diff --git a/data-raw/scripts/shapefiles.py b/data-raw/scripts/shapefiles.py index e79c195..da434d9 100644 --- a/data-raw/scripts/shapefiles.py +++ b/data-raw/scripts/shapefiles.py @@ -4,36 +4,48 @@ import requests import shutil import sys +import tempfile from zipfile import ZipFile class DownloadError(Exception): - def __init__(self, message, code=None): + def __init__(self, message, code): super().__init__(message) self.code = code -def _download_and_extract(file_url: str, extract_dir: str, cert_url: str) -> bool: - response = requests.get(file_url, verify = cert_url) - LOCAL_FILE = "download.zip" +def _download_and_extract(file_url: str, extract_dir: str): + response = requests.get(file_url, timeout=300) - if response.status_code == 200: - with open(LOCAL_FILE, "wb") as f: - f.write(response.content) - print(f"{LOCAL_FILE} downloaded from {file_url}.") + if response.status_code != 200: + raise DownloadError(f"Failed to download {file_url}.", code=response.status_code) + + with tempfile.NamedTemporaryFile(suffix='.zip', delete=False) as tmp_file: + tmp_filename = tmp_file.name + tmp_file.write(response.content) + print(f"Files downloaded from {file_url} to {tmp_filename}.") - with ZipFile(LOCAL_FILE, "r") as z: + try: + with ZipFile(tmp_filename, "r") as z: z.extractall(extract_dir) - print(f"{LOCAL_FILE} extracted to {extract_dir}.") + print(f"{tmp_filename} extracted to {extract_dir}.") + finally: + os.remove(tmp_filename) - os.remove(LOCAL_FILE) - else: - raise DownloadError(f"Failed to download {file_url}.", code=response.status_code) +def _exit(sys_code: int, gh_code: int=None): + """ + Exits with the given code(s). + + Parameters: + sys_code: The exit code to call sys.exit() with. + gh_code (optional): The code to set in the GitHub output. + If None, uses sys_code. + """ + gh_code = sys_code if gh_code is None else gh_code -def _failed(code: int): if (gh_env := os.getenv("GITHUB_OUTPUT")): with open(gh_env, "a") as f: - f.write(f"exit_code={code}\n") + f.write(f"exit_code={gh_code}\n") - sys.exit(code) + sys.exit(sys_code) def download_shapefiles(selected_year=None): """ @@ -54,7 +66,6 @@ def download_shapefiles(selected_year=None): SECTION = "shapefiles" url_template = config.get(SECTION, "url") - cert_file = config.get(SECTION, "cert") current_year = config.getint(SECTION, "current_year") entities = config.get(SECTION, "entities").split(",") res = config.get(SECTION, "res") @@ -68,21 +79,18 @@ def download_shapefiles(selected_year=None): with open(gh_env, "a") as f: f.write(f"shp_year={year}\n") - # create cert file URL - cert_url = os.path.join(script_dir, "..", "certs", cert_file) - # create output directory extract_dir = os.path.join(script_dir, "..", "shapefiles", str(year)) if os.path.exists(extract_dir): shutil.rmtree(extract_dir) - shutil.os.makedirs(extract_dir) + os.makedirs(extract_dir) try: # attempt shapefile downloads for entity in entities: url = url_template.format(year=year, entity=entity, res=res) - _download_and_extract(url, extract_dir, cert_url) + _download_and_extract(url, extract_dir) if (gh_env := os.getenv("GITHUB_ENV")): with open(gh_env, "a") as f: @@ -93,16 +101,20 @@ def download_shapefiles(selected_year=None): config.set(SECTION, "current_year", f"{year}") with open(config_file, "w") as f: config.write(f) + + _exit(0) except DownloadError as e: if e.code == 404: # i.e. shapefiles not found print(f"The shapefiles for {year} were not found. Better luck next time!") + # "files not found" is not considered a system failure + _exit(sys_code=0, gh_code=404) else: # other download errors print(e) + _exit(e.code) - _failed(e.code) except Exception as e: print(e) - _failed(-1) + _exit(-1) if __name__ == "__main__":