diff --git a/.github/workflows/delete dev env.yml b/.github/workflows/delete dev env.yml index e9c143ab618..f850adca2e1 100644 --- a/.github/workflows/delete dev env.yml +++ b/.github/workflows/delete dev env.yml @@ -24,6 +24,14 @@ jobs: with: fetch-depth: 0 + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: '3.14' + + - name: Install Python Dependency + run: pip install requests + - name: Remove Lua Dev Env Modules env: WIKI_USER: ${{ secrets.LP_BOTUSER }} @@ -33,4 +41,5 @@ jobs: LUA_DEV_ENV_NAME: "dev/${{ github.event.inputs.luadevenv }}" INCLUDE_COMMONS: ${{ github.event.inputs.includecommons }} INCLUDE_SUB_ENVS: ${{ github.event.inputs.includesubenvs }} - run: bash scripts/remove_dev.sh + PYTHONUNBUFFERED: 1 + run: python3 ./scripts/remove_dev.py diff --git a/.github/workflows/deploy test.yml b/.github/workflows/deploy test.yml index 2ae268f6480..ec3ca33d011 100644 --- a/.github/workflows/deploy test.yml +++ b/.github/workflows/deploy test.yml @@ -27,6 +27,16 @@ jobs: files: | lua/wikis/**/*.lua + - name: Setup Python + if: steps.changed-files.outputs.any_changed == 'true' + uses: actions/setup-python@v6 + with: + python-version: '3.14' + + - name: Install Python Dependency + if: steps.changed-files.outputs.any_changed == 'true' + run: pip install requests + - name: Personal Lua Deploy if: steps.changed-files.outputs.any_changed == 'true' env: @@ -35,4 +45,5 @@ jobs: WIKI_UA_EMAIL: ${{ secrets.LP_UA_EMAIL }} WIKI_BASE_URL: ${{ secrets.LP_BASE_URL }} LUA_DEV_ENV_NAME: "/dev/${{ github.event.inputs.luadevenv }}" - run: bash scripts/deploy.sh "${{ steps.changed-files.outputs.all_changed_files }}" + PYTHONUNBUFFERED: 1 + run: python3 ./scripts/deploy.py ${{ steps.changed-files.outputs.all_changed_files }} diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index da02bb258de..2abab23cd81 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -29,6 +29,16 @@ jobs: stylesheets/**/*.scss javascript/**/*.js + - name: Setup Python + if: steps.res-changed-files.outputs.any_changed == 'true' || steps.lua-changed-files.outputs.any_changed == 'true' + uses: actions/setup-python@v6 + with: + python-version: '3.14' + + - name: Install Python Dependency + if: steps.res-changed-files.outputs.any_changed == 'true' || steps.lua-changed-files.outputs.any_changed == 'true' + run: pip install requests + - name: Resource Deploy if: steps.res-changed-files.outputs.any_changed == 'true' env: @@ -37,7 +47,8 @@ jobs: WIKI_UA_EMAIL: ${{ secrets.LP_UA_EMAIL }} WIKI_BASE_URL: ${{ secrets.LP_BASE_URL }} DEPLOY_TRIGGER: ${{ github.event_name }} - run: bash scripts/deploy_res.sh "${{ steps.res-changed-files.outputs.all_changed_files }}" + PYTHONUNBUFFERED: 1 + run: python3 ./scripts/deploy_res.py ${{ steps.res-changed-files.outputs.all_changed_files }} - name: Lua Deploy if: steps.lua-changed-files.outputs.any_changed == 'true' @@ -47,7 +58,8 @@ jobs: WIKI_UA_EMAIL: ${{ secrets.LP_UA_EMAIL }} WIKI_BASE_URL: ${{ secrets.LP_BASE_URL }} DEPLOY_TRIGGER: ${{ github.event_name }} - run: bash scripts/deploy.sh "${{ steps.lua-changed-files.outputs.all_changed_files }}" + PYTHONUNBUFFERED: 1 + run: python3 ./scripts/deploy.py ${{ steps.lua-changed-files.outputs.all_changed_files }} - name: Lua Protect if: steps.lua-changed-files.outputs.added_files_count != 0 || steps.lua-changed-files.outputs.renamed_files_count != 0 @@ -56,4 +68,5 @@ jobs: WIKI_PASSWORD: ${{ secrets.LP_BOTPASSWORD }} WIKI_UA_EMAIL: ${{ secrets.LP_UA_EMAIL }} WIKI_BASE_URL: ${{ secrets.LP_BASE_URL }} - run: bash scripts/protect.sh "${{ steps.lua-changed-files.outputs.added_files }} ${{ steps.lua-changed-files.outputs.renamed_files }}" + PYTHONUNBUFFERED: 1 + run: python3 ./scripts/protect.py ${{ steps.lua-changed-files.outputs.added_files }} ${{ steps.lua-changed-files.outputs.renamed_files }} diff --git a/.github/workflows/protect new wiki modules.yml b/.github/workflows/protect new wiki modules.yml index 369ff76f973..6bb1caafa9a 100644 --- a/.github/workflows/protect new wiki modules.yml +++ b/.github/workflows/protect new wiki modules.yml @@ -16,6 +16,14 @@ jobs: steps: - uses: actions/checkout@v6 + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: '3.14' + + - name: Install Python Dependency + run: pip install requests + - name: Lua Protect env: WIKI_USER: ${{ secrets.LP_BOTUSER }} @@ -23,4 +31,5 @@ jobs: WIKI_UA_EMAIL: ${{ secrets.LP_UA_EMAIL }} WIKI_BASE_URL: ${{ secrets.LP_BASE_URL }} WIKI_TO_PROTECT: ${{ github.event.inputs.wiki }} - run: bash scripts/protect.sh + PYTHONUNBUFFERED: 1 + run: python3 ./scripts/protect.py diff --git a/.github/workflows/protect new wiki templates.yml b/.github/workflows/protect new wiki templates.yml index ae18459b5c0..3b1542f5a2b 100644 --- a/.github/workflows/protect new wiki templates.yml +++ b/.github/workflows/protect new wiki templates.yml @@ -16,6 +16,14 @@ jobs: steps: - uses: actions/checkout@v6 + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: '3.14' + + - name: Install Python Dependency + run: pip install requests + - name: Template Protect env: WIKI_USER: ${{ secrets.LP_BOTUSER }} @@ -23,4 +31,5 @@ jobs: WIKI_UA_EMAIL: ${{ secrets.LP_UA_EMAIL }} WIKI_BASE_URL: ${{ secrets.LP_BASE_URL }} WIKI_TO_PROTECT: ${{ github.event.inputs.wiki }} - run: bash scripts/protect_templates.sh + PYTHONUNBUFFERED: 1 + run: python3 ./scripts/protect_templates.py diff --git a/.github/workflows/pythoncheck.yml b/.github/workflows/pythoncheck.yml new file mode 100644 index 00000000000..ad756d5a1ec --- /dev/null +++ b/.github/workflows/pythoncheck.yml @@ -0,0 +1,22 @@ +name: Python Code Style + +on: + pull_request: + paths: + - '**.py' + - '.github/workflows/*.yml' + workflow_dispatch: + +jobs: + python-code-style: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v6 + - name: Run linter + uses: astral-sh/ruff-action@v3 + with: + src: scripts/ + - name: Check styling + run: ruff format --check --diff scripts/ diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index f90f3c9a3f2..0d2d29c3d8b 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -15,6 +15,14 @@ jobs: with: fetch-depth: 2 + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: '3.14' + + - name: Install Python Dependency + run: pip install requests + - name: Resource Deploy env: WIKI_USER: ${{ secrets.LP_BOTUSER }} @@ -22,7 +30,8 @@ jobs: WIKI_UA_EMAIL: ${{ secrets.LP_UA_EMAIL }} WIKI_BASE_URL: ${{ secrets.LP_BASE_URL }} DEPLOY_TRIGGER: ${{ github.event_name }} - run: bash scripts/deploy_res.sh + PYTHONUNBUFFERED: 1 + run: python3 ./scripts/deploy_res.py - name: Lua Deploy env: @@ -31,4 +40,5 @@ jobs: WIKI_UA_EMAIL: ${{ secrets.LP_UA_EMAIL }} WIKI_BASE_URL: ${{ secrets.LP_BASE_URL }} DEPLOY_TRIGGER: ${{ github.event_name }} - run: bash scripts/deploy.sh + PYTHONUNBUFFERED: 1 + run: python3 ./scripts/deploy.py diff --git a/.gitignore b/.gitignore index 07f9d297949..4b5b6746381 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,6 @@ /package-lock.json /luacov.report.out /luacov.stats.out +**/__pycache__/ +.env +*.ck diff --git a/.vscode/settings.json b/.vscode/settings.json index 919b327f401..acda9e63d38 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -12,6 +12,8 @@ "css.validate": false, "less.validate": false, "scss.validate": false, + "python.envFile": "${workspaceFolder}/.env", + "python.terminal.useEnvFile": true, "workbench.editor.customLabels.patterns": { "**/wikis/*/*.lua": "${dirname}: ${filename}", "**/wikis/*/*/*.lua": "${dirname(-4)}: ${dirname}/${filename}", diff --git a/scripts/deploy.py b/scripts/deploy.py new file mode 100644 index 00000000000..662139e7f3b --- /dev/null +++ b/scripts/deploy.py @@ -0,0 +1,73 @@ +import itertools +import os +import pathlib +import re +import sys + +from typing import Iterable + +import requests + +from deploy_util import ( + get_git_deploy_reason, + deploy_file_to_wiki, + read_cookie_jar, + read_file_from_path, + write_to_github_summary_file, +) +from login_and_get_token import get_token + +HEADER_PATTERN = re.compile( + r"\A---\n" r"-- @Liquipedia\n" r"-- page=(?P[^\n]*)\n" +) + + +def deploy_all_files_for_wiki( + wiki: str, file_paths: Iterable[pathlib.Path], deploy_reason: str +) -> bool: + all_modules_deployed = True + token = get_token(wiki) + with requests.Session() as session: + session.cookies = read_cookie_jar(wiki) + for file_path in file_paths: + print(f"::group::Checking {str(file_path)}") + file_content = read_file_from_path(file_path) + header_match = HEADER_PATTERN.match(file_content) + if not header_match: + print("...skipping - no magic comment found") + write_to_github_summary_file(f"{str(file_path)} skipped") + else: + page = header_match.groupdict()["pageName"] + ( + os.getenv("LUA_DEV_ENV_NAME") or "" + ) + module_deployed, _ = deploy_file_to_wiki( + session, file_path, file_content, wiki, page, token, deploy_reason + ) + all_modules_deployed &= module_deployed + print("::endgroup::") + return all_modules_deployed + + +def main(): + all_modules_deployed = True + lua_files: Iterable[pathlib.Path] + git_deploy_reason: str + if len(sys.argv[1:]) == 0: + lua_files = pathlib.Path("./lua/wikis/").rglob("*.lua") + git_deploy_reason = "Automated Weekly Re-Sync" + else: + lua_files = [pathlib.Path(arg) for arg in sys.argv[1:]] + git_deploy_reason = get_git_deploy_reason() + + for wiki, files in itertools.groupby(sorted(lua_files), lambda path: path.parts[2]): + all_modules_deployed &= deploy_all_files_for_wiki( + wiki, list(files), git_deploy_reason + ) + + if not all_modules_deployed: + print("::warning::Some modules were not deployed!") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/deploy.sh b/scripts/deploy.sh deleted file mode 100644 index c1ec013d29c..00000000000 --- a/scripts/deploy.sh +++ /dev/null @@ -1,92 +0,0 @@ -#!/bin/bash - -userAgent="GitHub Autodeploy Bot/1.1.0 (${WIKI_UA_EMAIL})" -pat='\-\-\-\ -\-\- @Liquipedia\ -\-\- page=([^ -]*)\ -' - -. ./scripts/login_and_get_token.sh - -if [[ -n "$1" ]]; then - luaFiles=$1 - gitDeployReason="\"$(git log -1 --pretty='%h %s')\"" -else - luaFiles=$(find lua/wikis -type f -name '*.lua') - gitDeployReason='Automated Weekly Re-Sync' -fi - -allModulesDeployed=true -for luaFile in $luaFiles; do - if [[ -n "$1" ]]; then - luaFile="./$luaFile" - fi - echo "::group::Checking $luaFile" - fileContents=$(cat "$luaFile") - - [[ $fileContents =~ $pat ]] - - if [[ "${BASH_REMATCH[1]}" == "" ]]; then - echo '...skipping - no magic comment found' - echo "${luaFile} skipped" >> $GITHUB_STEP_SUMMARY - else - # Extract wiki name from path, e.g. ./lua/wikis/commons/Widget/Match/Page/Header.lua -> commons - wiki=$(echo "$luaFile" | awk -F'/' '{for(i=1;i<=NF;i++) if($i=="wikis") print $(i+1)}') - page="${BASH_REMATCH[1]}${LUA_DEV_ENV_NAME}" - - echo '...magic comment found - updating wiki...' - - echo "...wiki = $wiki" - echo "...page = $page" - wikiApiUrl="${WIKI_BASE_URL}/${wiki}/api.php" - ckf="cookie_${wiki}.ck" - - getToken ${wiki} - - # Edit page - rawResult=$( - curl \ - -s \ - -b "$ckf" \ - -c "$ckf" \ - --data-urlencode "title=${page}" \ - --data-urlencode "text=${fileContents}" \ - --data-urlencode "summary=Git: ${gitDeployReason}" \ - --data-urlencode "bot=true" \ - --data-urlencode "recreate=true" \ - --data-urlencode "token=${token}" \ - -H "User-Agent: ${userAgent}" \ - -H 'Accept-Encoding: gzip' \ - -X POST "${wikiApiUrl}?format=json&action=edit" \ - | gunzip - ) - result=$(echo "$rawResult" | jq ".edit.result" -r) - if [[ "${result}" == "Success" ]]; then - nochange=$(echo "$rawResult" | jq ".edit.nochange" -r) - echo "...${result}" - if [[ "${nochange}" == "" ]] && [[ "${DEPLOY_TRIGGER}" == "push" ]]; then - echo "::notice file=${luaFile}::No change made" - elif [[ "${nochange}" != "" ]] && [[ "${DEPLOY_TRIGGER}" != "push" ]]; then - echo "::warning file=${luaFile}::File changed" - fi - echo '...done' - echo ":information_source: ${luaFile} successfully deployed" >> $GITHUB_STEP_SUMMARY - else - echo "::warning file=${luaFile}::failed to deploy" - echo ":warning: ${luaFile} failed to deploy" >> $GITHUB_STEP_SUMMARY - allModulesDeployed=false - fi - - # Don't get rate limited - sleep 4 - fi - echo '::endgroup::' - - if [ "$allModulesDeployed" != true ]; then - echo "::warning::Some modules were not deployed!" - exit 1 - fi -done - -rm -f cookie_* diff --git a/scripts/deploy_res.py b/scripts/deploy_res.py new file mode 100644 index 00000000000..2b30f76a840 --- /dev/null +++ b/scripts/deploy_res.py @@ -0,0 +1,104 @@ +import itertools +import pathlib +import sys +import subprocess + +from typing import Iterable + +import requests + +from deploy_util import ( + HEADER, + get_git_deploy_reason, + get_wiki_api_url, + deploy_file_to_wiki, + read_cookie_jar, + read_file_from_path, +) +from login_and_get_token import get_token + + +def deploy_resources( + res_type: str, file_paths: Iterable[pathlib.Path], deploy_reason: str +) -> tuple[bool, bool]: + all_deployed = True + changes_made = False + token = get_token("commons") + with requests.Session() as session: + session.cookies = read_cookie_jar("commons") + for file_path in file_paths: + print(f"::group::Checking {str(file_path)}") + file_content = read_file_from_path(file_path) + page = ( + f"MediaWiki:Common.{'js' if res_type == '.js' else 'css'}/" + + file_path.name + ) + print(f"...page = {page}") + deploy_result = deploy_file_to_wiki( + session, file_path, file_content, "commons", page, token, deploy_reason + ) + all_deployed = all_deployed and deploy_result[0] + changes_made = changes_made or deploy_result[1] + print("::endgroup::") + return (all_deployed, changes_made) + + +def update_cache(): + with requests.Session() as session: + session.cookies = read_cookie_jar("commons") + cache_result = session.post( + get_wiki_api_url("commons"), + headers=HEADER, + params={"format": "json", "action": "updatelpmwmessageapi"}, + data={ + "messagename": "Resourceloaderarticles-cacheversion", + "value": subprocess.check_output(["git", "log", "-1", "--pretty=%h"]) + .decode() + .strip(), + }, + ).json() + if ( + cache_result["updatelpmwmessageapi"].get("message") + == "Successfully changed the message value" + ): + print("Resource cache version updated succesfully!") + else: + print("::error::Resource cache version unable to be updated!") + exit(1) + + +def main(): + all_deployed: bool = True + changes_made: bool = False + resource_files: Iterable[pathlib.Path] + git_deploy_reason: str + if len(sys.argv[1:]) == 0: + resource_files = itertools.chain( + pathlib.Path("./javascript/").rglob("*.js"), + pathlib.Path("./stylesheets/").rglob("*.scss"), + ) + git_deploy_reason = "Automated Weekly Re-Sync" + else: + resource_files = [pathlib.Path(arg) for arg in sys.argv[1:]] + git_deploy_reason = get_git_deploy_reason() + + for res_type, files in itertools.groupby( + sorted(resource_files), lambda path: path.suffix + ): + group_all_deployed, group_changes_made = deploy_resources( + res_type, list(files), git_deploy_reason + ) + all_deployed = all_deployed and group_all_deployed + changes_made = changes_made or group_changes_made + + if not all_deployed: + print( + "::error::Some files were not deployed; resource cache version not updated!" + ) + exit(1) + elif changes_made: + update_cache() + + +if __name__ == "__main__": + main() diff --git a/scripts/deploy_res.sh b/scripts/deploy_res.sh deleted file mode 100644 index 8cfcb9a533e..00000000000 --- a/scripts/deploy_res.sh +++ /dev/null @@ -1,102 +0,0 @@ -#!/bin/bash - -userAgent="GitHub Autodeploy Bot/1.1.0 (${WIKI_UA_EMAIL})" - -. ./scripts/login_and_get_token.sh - -if [[ -n "$1" ]]; then - files=$1 - gitDeployReason="\"$(git log -1 --pretty='%h %s')\"" -else - files=$(find stylesheets javascript -type f -name '*.scss' -o -name '*.js') - gitDeployReason='Automated Weekly Re-Sync' -fi - -wikiApiUrl="${WIKI_BASE_URL}/commons/api.php" -ckf="cookie_commons.ck" - -allDeployed=true -changesMade=false -for file in $files; do - if [[ -n "$1" ]]; then - file="./$file" - fi - echo "::group::Checking $file" - fileContents=$(cat "$file") - fileName=$(basename "$file") - - if [[ $file == *.js ]]; then - page="MediaWiki:Common.js/${fileName}" - else - page="MediaWiki:Common.css/${fileName}" - fi - - echo "...page = $page" - - # Edit page - getToken "commons" - rawResult=$( - curl \ - -s \ - -b "$ckf" \ - -c "$ckf" \ - --data-urlencode "title=${page}" \ - --data-urlencode "text=${fileContents}" \ - --data-urlencode "summary=Git: ${gitDeployReason}" \ - --data-urlencode "bot=true" \ - --data-urlencode "recreate=true" \ - --data-urlencode "token=${token}" \ - -H "User-Agent: ${userAgent}" \ - -H 'Accept-Encoding: gzip' \ - -X POST "${wikiApiUrl}?format=json&action=edit" \ - | gunzip - ) - result=$(echo "$rawResult" | jq ".edit.result" -r) - newRevId=$(echo "$rawResult" | jq ".edit.newrevid" -r) - if [[ "${result}" == "Success" ]]; then - if [[ "${newRevId}" != "null" ]]; then - changesMade=true - if [[ "${DEPLOY_TRIGGER}" != "push" ]]; then - echo "::warning file=${file}::File changed" - fi - fi - echo "...${result}" - echo '...done' - echo ":information_source: ${file} successfully deployed" >> $GITHUB_STEP_SUMMARY - else - echo "::warning file=${file}::failed to deploy" - echo ":warning: ${file} failed to deploy" >> $GITHUB_STEP_SUMMARY - allDeployed=false - fi - echo '::endgroup::' - - # Don't get rate limited - sleep 4 -done - -if [ "$allDeployed" != true ]; then - echo "::error::Some files were not deployed; resource cache version not updated!" - exit 1 -elif [ "$changesMade" == true ]; then - cacheResult=$( - curl \ - -s \ - -b "$ckf" \ - -c "$ckf" \ - --data-urlencode "messagename=Resourceloaderarticles-cacheversion" \ - --data-urlencode "value=$(git log -1 --pretty='%h')" \ - -H "User-Agent: ${userAgent}" \ - -H 'Accept-Encoding: gzip' \ - -X POST "${wikiApiUrl}?format=json&action=updatelpmwmessageapi" \ - | gunzip \ - | jq ".updatelpmwmessageapi.message" -r - ) - if [[ "${cacheResult}" == "Successfully changed the message value" ]]; then - echo "Resource cache version updated succesfully!" - else - echo "::error::Resource cache version unable to be updated!" - exit 1 - fi -fi - -rm -f cookie_* diff --git a/scripts/deploy_util.py b/scripts/deploy_util.py new file mode 100644 index 00000000000..32552af2238 --- /dev/null +++ b/scripts/deploy_util.py @@ -0,0 +1,146 @@ +import functools +import http.cookiejar +import os +import pathlib +import subprocess +import time + +import requests + +__all__ = [ + "DEPLOY_TRIGGER", + "HEADER", + "SLEEP_DURATION", + "deploy_file_to_wiki", + "get_git_deploy_reason", + "get_wiki_api_url", + "get_wikis", + "read_cookie_jar", + "read_file_from_path", + "write_to_github_summary_file", +] + +DEPLOY_TRIGGER = os.getenv("DEPLOY_TRIGGER") +GITHUB_STEP_SUMMARY_FILE = os.getenv("GITHUB_STEP_SUMMARY") +USER_AGENT = f"GitHub Autodeploy Bot/2.0.0 ({os.getenv('WIKI_UA_EMAIL')})" +WIKI_BASE_URL = os.getenv("WIKI_BASE_URL") +HEADER = { + "User-Agent": USER_AGENT, + "accept": "application/json", + "Accept-Encoding": "gzip", +} +SLEEP_DURATION = 4 + + +def get_wikis() -> set[str]: + response = requests.get( + "https://liquipedia.net/api.php", + headers=HEADER, + ) + wikis = response.json() + time.sleep(SLEEP_DURATION) + return set(wikis["allwikis"].keys()) + + +@functools.cache +def get_wiki_api_url(wiki: str) -> str: + return f"{WIKI_BASE_URL}/{wiki}/api.php" + + +def get_git_deploy_reason(): + return ( + subprocess.check_output(["git", "log", "-1", "--pretty='%h %s'"]) + .decode() + .strip() + ) + + +def deploy_file_to_wiki( + session: requests.Session, + file_path: pathlib.Path, + file_content: str, + wiki: str, + target_page: str, + token: str, + deploy_reason: str, +) -> tuple[bool, bool]: + change_made = False + deployed = True + response = session.post( + get_wiki_api_url(wiki), + headers=HEADER, + params={"format": "json", "action": "edit"}, + data={ + "title": target_page, + "text": file_content, + "summary": f"Git: {deploy_reason}", + "bot": "true", + "recreate": "true", + "token": token, + }, + ).json() + edit_info = response.get("edit") + error_info = response.get("error") + + # Handle API errors or unexpected response structure + if error_info is not None or edit_info is None: + print(f"::warning file={str(file_path)}::failed to deploy (API error)") + details = "" + if isinstance(error_info, dict): + code = error_info.get("code") + info = error_info.get("info") + detail_parts = [] + if code: + detail_parts.append(f"code={code}") + if info: + detail_parts.append(f"info={info}") + if detail_parts: + details = " (" + ", ".join(detail_parts) + ")" + write_to_github_summary_file( + f":warning: {str(file_path)} failed to deploy due to API error{details}" + ) + deployed = False + time.sleep(SLEEP_DURATION) + return deployed, change_made + + result = edit_info.get("result") + new_rev_id = edit_info.get("newrevid") + if result == "Success": + if new_rev_id is not None: + change_made = True + if DEPLOY_TRIGGER != "push": + print(f"::warning file={str(file_path)}::File changed") + print(f"...{result}") + print("...done") + write_to_github_summary_file( + f":information_source: {str(file_path)} successfully deployed" + ) + + else: + print(f"::warning file={str(file_path)}::failed to deploy") + write_to_github_summary_file(f":warning: {str(file_path)} failed to deploy") + deployed = False + time.sleep(SLEEP_DURATION) + return deployed, change_made + + +def read_cookie_jar(wiki: str) -> http.cookiejar.FileCookieJar: + ckf = f"cookie_{wiki}.ck" + cookie_jar = http.cookiejar.LWPCookieJar(filename=ckf) + try: + cookie_jar.load(ignore_discard=True) + except OSError: + pass + return cookie_jar + + +def read_file_from_path(file_path: pathlib.Path) -> str: + with file_path.open("r") as file: + return file.read() + + +def write_to_github_summary_file(text: str): + if not GITHUB_STEP_SUMMARY_FILE: + return + with open(GITHUB_STEP_SUMMARY_FILE, "a") as summary: + summary.write(f"{text}\n") diff --git a/scripts/login_and_get_token.py b/scripts/login_and_get_token.py new file mode 100644 index 00000000000..d91b92afd10 --- /dev/null +++ b/scripts/login_and_get_token.py @@ -0,0 +1,64 @@ +import functools +import os +import time + +import requests + +from deploy_util import HEADER, SLEEP_DURATION, get_wiki_api_url, read_cookie_jar + +__all__ = ["get_token"] + +WIKI_USER = os.getenv("WIKI_USER") +WIKI_PASSWORD = os.getenv("WIKI_PASSWORD") + +loggedin: set[str] = set() + + +def login(wiki: str): + if wiki in loggedin: + return + cookie_jar = read_cookie_jar(wiki) + print(f"...logging in on {wiki}") + with requests.Session() as session: + session.cookies = cookie_jar + token_response = session.post( + get_wiki_api_url(wiki), + headers=HEADER, + params={ + "format": "json", + "action": "query", + "meta": "tokens", + "type": "login", + }, + ).json() + session.post( + get_wiki_api_url(wiki), + headers=HEADER, + data={ + "lgname": WIKI_USER, + "lgpassword": WIKI_PASSWORD, + "lgtoken": token_response["query"]["tokens"]["logintoken"], + }, + params={"format": "json", "action": "login"}, + ) + loggedin.add(wiki) + cookie_jar.save(ignore_discard=True) + time.sleep(SLEEP_DURATION) + + +@functools.cache +def get_token(wiki: str) -> str: + login(wiki) + + with requests.Session() as session: + session.cookies = read_cookie_jar(wiki) + token_response = session.post( + get_wiki_api_url(wiki), + headers=HEADER, + params={ + "format": "json", + "action": "query", + "meta": "tokens", + }, + ).json() + return token_response["query"]["tokens"]["csrftoken"] diff --git a/scripts/login_and_get_token.sh b/scripts/login_and_get_token.sh deleted file mode 100644 index 7394ff57ea9..00000000000 --- a/scripts/login_and_get_token.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/bin/bash - -userAgent="GitHub Autodeploy Bot/1.1.0 (${WIKI_UA_EMAIL})" - -declare -A loggedin - -# logs into a specified wiki -# $1 -> the wiki to log into -login() { - wiki=$1 - - if [[ ${loggedin[${wiki}]} != 1 ]]; then - ckf="cookie_${wiki}.ck" - wikiApiUrl="${WIKI_BASE_URL}/${wiki}/api.php" - echo "...logging in on \"${wiki}\"" - loginToken=$( - curl \ - -s \ - -b "$ckf" \ - -c "$ckf" \ - -d "format=json&action=query&meta=tokens&type=login" \ - -H "User-Agent: ${userAgent}" \ - -H 'Accept-Encoding: gzip' \ - -X POST "$wikiApiUrl" \ - | gunzip \ - | jq ".query.tokens.logintoken" -r - ) - curl \ - -s \ - -b "$ckf" \ - -c "$ckf" \ - --data-urlencode "lgname=${WIKI_USER}" \ - --data-urlencode "lgpassword=${WIKI_PASSWORD}" \ - --data-urlencode "lgtoken=${loginToken}" \ - -H "User-Agent: ${userAgent}" \ - -H 'Accept-Encoding: gzip' \ - -X POST "${wikiApiUrl}?format=json&action=login" \ - | gunzip \ - > /dev/null - loggedin[$wiki]=1 - # Don't get rate limited - sleep 4 - fi -} - -# gets an edit/protect token for a specified wiki -# if not already logged in on the wiki also logs in on the wiki -# $1 -> the wiki to get the token for -getToken(){ - wiki=$1 - wikiApiUrl="${WIKI_BASE_URL}/${wiki}/api.php" - ckf="cookie_${wiki}.ck" - - login $wiki - - token=$( - curl \ - -s \ - -b "$ckf" \ - -c "$ckf" \ - -d "format=json&action=query&meta=tokens" \ - -H "User-Agent: ${userAgent}" \ - -H 'Accept-Encoding: gzip' \ - -X POST "$wikiApiUrl" \ - | gunzip \ - | jq ".query.tokens.csrftoken" -r - ) -} diff --git a/scripts/protect.py b/scripts/protect.py new file mode 100644 index 00000000000..5bc3387e7aa --- /dev/null +++ b/scripts/protect.py @@ -0,0 +1,60 @@ +import os +import pathlib +import sys + +from typing import Iterable + +from deploy_util import get_wikis +from protect_page import ( + protect_non_existing_page, + protect_existing_page, + handle_protect_errors, +) + +WIKI_TO_PROTECT = os.getenv("WIKI_TO_PROTECT") + + +def check_for_local_version(module: str, wiki: str): + if wiki == "commons": + return False + return pathlib.Path(f"./lua/wikis/{wiki}/{module}.lua").exists() + + +def protect_if_has_no_local_version(module: str, wiki: str): + page = "Module:" + module + if not check_for_local_version(module, wiki): + protect_non_existing_page(page, wiki) + + +def main(): + lua_files: Iterable[pathlib.Path] + if len(sys.argv[1:]) > 0: + lua_files = [pathlib.Path(arg) for arg in sys.argv[1:]] + elif WIKI_TO_PROTECT: + lua_files = pathlib.Path("./lua/wikis/").rglob("*.lua") + else: + print("Nothing to protect") + exit(0) + + for file_to_protect in sorted(lua_files): + print(f"::group::Checking {str(file_to_protect)}") + wiki = file_to_protect.parts[2] + module = "/".join(file_to_protect.parts[3:])[:-4] + page = "Module:" + module + if WIKI_TO_PROTECT: + if wiki == WIKI_TO_PROTECT: + protect_existing_page(page, wiki) + elif wiki == "commons": + protect_if_has_no_local_version(module, WIKI_TO_PROTECT) + elif wiki != "commons": + protect_existing_page(page, wiki) + else: # commons case + protect_existing_page(page, wiki) + for deploy_wiki in get_wikis(): + protect_if_has_no_local_version(module, deploy_wiki) + print("::endgroup::") + handle_protect_errors() + + +if __name__ == "__main__": + main() diff --git a/scripts/protect.sh b/scripts/protect.sh deleted file mode 100644 index b7d68f53335..00000000000 --- a/scripts/protect.sh +++ /dev/null @@ -1,116 +0,0 @@ -#!/bin/bash - -userAgent="GitHub Autodeploy Bot/1.1.0 (${WIKI_UA_EMAIL})" - -. ./scripts/protect_page.sh - -declare -a allWikis -declare -a regexErrors=() - -regex="^\.?/?lua/wikis/([a-z0-9]+)/(.*)\.lua$" - -if [[ -n "$1" ]] then - filesToProtect=$1 -elif [[ -n ${WIKI_TO_PROTECT} ]]; then - filesToProtect=$(find lua/wikis -type f -name '*.lua') -else - echo "Nothing to protect" - exit 0 -fi - -allLuaFiles=$(find lua -type f -name '*.lua') - -allWikis=$( - curl \ - -s \ - -b "$ckf" \ - -c "$ckf" \ - -H "User-Agent: ${userAgent}" \ - -H 'Accept-Encoding: gzip' \ - -X GET "https://liquipedia.net/api.php?action=listwikis" \ - | gunzip \ - | jq '.allwikis | keys[]' -r -) -# Don't get rate limited -sleep 4 - -# checks if for a specified (commons) module there exists a local version of said module on the specified wiki in the git repo -# $1 -> module (without namespace prefix) -# $2 -> wiki -checkForLocalVersion() { - if [[ $2 == "commons" ]]; then - hasNoLocalVersion=false - elif [[ $allLuaFiles == *"lua/wikis/${2}/${1}.lua"* ]] || [[ $filesToProtect == *"lua/wikis/${2}/${1}.lua"* ]]; then - hasNoLocalVersion=false - else - hasNoLocalVersion=true - fi -} - -# protects a module against creation if it has no local version according to git -# $1 -> module (without namespace prefix) -# $2 -> wiki -protectIfHasNoLocalVersion() { - module="${1}" - page="Module:${module}" - wiki="${2}" - checkForLocalVersion $module $wiki - if $hasNoLocalVersion; then - protectNonExistingPage $page $wiki - fi -} - -for fileToProtect in $filesToProtect; do - echo "::group::Checking $fileToProtect" - if [[ $fileToProtect =~ $regex ]]; then - wiki=${BASH_REMATCH[1]} - module=${BASH_REMATCH[2]} - page="Module:${module}" - - if [[ -n ${WIKI_TO_PROTECT} ]]; then - if [[ $wiki == ${WIKI_TO_PROTECT} ]]; then - protectExistingPage $page ${WIKI_TO_PROTECT} - elif [[ $wiki == "commons" ]]; then - protectIfHasNoLocalVersion $module ${WIKI_TO_PROTECT} - fi - elif [[ "commons" != $wiki ]]; then - protectExistingPage $page $wiki - else # commons case - protectExistingPage $page $wiki - - for deployWiki in $allWikis; do - protectIfHasNoLocalVersion $module $deployWiki - done - fi - else - echo '::warning::skipping - regex failed' - regexErrors+=($fileToProtect) - fi - echo '::endgroup::' -done - -if [[ ${#regexErrors[@]} -ne 0 ]]; then - echo "::warning::Some regexes failed" - echo ":warning: Some regexes failed" >> $GITHUB_STEP_SUMMARY - echo "::group::Files the regex failed on" - for value in "${regexErrors[@]}"; do - echo "... ${failedRegex}" - done - echo "::endgroup::" -fi - -if [[ ${#protectErrors[@]} -ne 0 ]]; then - echo "::warning::Some modules could not be protected" - echo ":warning: Some modules could not be protected" >> $GITHUB_STEP_SUMMARY - echo "::group::Failed protections" - for value in "${protectErrors[@]}"; do - echo "... ${value}" - done - echo "::endgroup::" -fi - -rm -f cookie_* - -if [[ ${#protectErrors[@]} -ne 0 ]] || [[ ${#regexErrors[@]} -ne 0 ]]; then - exit 1 -fi diff --git a/scripts/protect_page.py b/scripts/protect_page.py new file mode 100644 index 00000000000..9eeb90e44d0 --- /dev/null +++ b/scripts/protect_page.py @@ -0,0 +1,103 @@ +import time + +from typing import Literal + +import requests + +from deploy_util import ( + HEADER, + SLEEP_DURATION, + get_wiki_api_url, + read_cookie_jar, + write_to_github_summary_file, +) +from login_and_get_token import get_token + +__all__ = [ + "protect_non_existing_page", + "protect_existing_page", + "handle_protect_errors", +] + +protect_errors = list() + + +def protect_page(page: str, wiki: str, protect_mode: Literal["edit", "create"]): + protect_options: str + if protect_mode == "edit": + protect_options = "edit=allow-only-sysop|move=allow-only-sysop" + elif protect_mode == "create": + protect_options = "create=allow-only-sysop" + else: + raise ValueError(f"invalid protect mode: {protect_mode}") + print(f"...wiki = {wiki}") + print(f"...page = {page}") + token = get_token(wiki) + with requests.Session() as session: + session.cookies = read_cookie_jar(wiki) + response = session.post( + get_wiki_api_url(wiki), + headers=HEADER, + params={"format": "json", "action": "protect"}, + data={ + "title": page, + "protections": protect_options, + "reason": "Git maintained", + "expiry": "infinite", + "bot": "true", + "token": token, + }, + ).json() + + time.sleep(SLEEP_DURATION) + if response.get("error"): + print( + f"::warning::could not ({protect_mode}) protect {page} on {wiki}: {response['error']['info']}" + ) + protect_errors.append(f"{protect_mode}:{wiki}:{page}") + return + protections = response["protect"].get("protections") + for protection in protections: + if protection[protect_mode] == "allow-only-sysop": + return + print(f"::warning::could not ({protect_mode}) protect {page} on {wiki}") + protect_errors.append(f"{protect_mode}:{wiki}:{page}") + + +def check_if_page_exists(page: str, wiki: str) -> bool: + with requests.Session() as session: + session.cookies = read_cookie_jar(wiki) + + result = session.post( + get_wiki_api_url(wiki), + headers=HEADER, + params={"format": "json", "action": "query"}, + data={"titles": page, "prop": "info"}, + ).json() + + time.sleep(SLEEP_DURATION) + return "-1" not in result["query"]["pages"] + + +def protect_non_existing_page(page: str, wiki: str): + if check_if_page_exists(page, wiki): + print(f"::warning::{page} already exists on {wiki}") + protect_errors.append(f"create:{wiki}:{page}") + else: + protect_page(page, wiki, "create") + + +def protect_existing_page(page: str, wiki: str): + protect_page(page, wiki, "edit") + + +def handle_protect_errors(): + if len(protect_errors) == 0: + return + print("::warning::Some templates could not be protected") + write_to_github_summary_file(":warning: Some templates could not be protected") + print("::group::Failed protections") + for protect_error in protect_errors: + print(f"... {protect_error}") + print("::endgroup::") + exit(1) diff --git a/scripts/protect_page.sh b/scripts/protect_page.sh deleted file mode 100644 index 3b78f6c6aeb..00000000000 --- a/scripts/protect_page.sh +++ /dev/null @@ -1,115 +0,0 @@ -#!/bin/bash - -userAgent="GitHub Autodeploy Bot/1.1.0 (${WIKI_UA_EMAIL})" - -. ./scripts/login_and_get_token.sh - -declare -a protectErrors=() - -# protects a specified page on a specified wiki with the specified protect mode -# $1 -> page (inkl namespace prefix) -# $2 -> wiki -# $3 -> protect mode ('edit' || 'create') -protectPage() { - page="${1}" - wiki=$2 - protectMode=$3 - - if [[ ${protectMode} == 'edit' ]]; then - protectOptions="edit=allow-only-sysop|move=allow-only-sysop" - elif [[ ${protectMode} == 'create' ]]; then - protectOptions="create=allow-only-sysop" - else - echo "::warning:: invalid protect mode: ${protectMode}" - exit 1 - fi - - echo "...wiki = $wiki" - echo "...page = ${page}" - wikiApiUrl="${WIKI_BASE_URL}/${wiki}/api.php" - ckf="cookie_${wiki}.ck" - - getToken $wiki - - rawProtectResult=$( - curl \ - -s \ - -b "$ckf" \ - -c "$ckf" \ - --data-urlencode "title=${page}" \ - --data-urlencode "protections=${protectOptions}" \ - --data-urlencode "reason=Git maintained" \ - --data-urlencode "expiry=infinite" \ - --data-urlencode "bot=true" \ - --data-urlencode "token=${token}" \ - -H "User-Agent: ${userAgent}" \ - -H 'Accept-Encoding: gzip' \ - -X POST "${wikiApiUrl}?format=json&action=protect" \ - | gunzip - ) - # Don't get rate limited - sleep 4 - - result=$(echo "$rawProtectResult" | jq ".protect.protections.[].${protectMode}" -r) - if [[ $result != *"allow-only-sysop"* ]]; then - echo "::warning::could not (${protectMode}) protect ${page} on ${wiki}" - protectErrorMsg="${protectMode}:${wiki}:${page}" - protectErrors+=("${protectErrorMsg}") - fi -} - -# checks if a specified page on a specified wiki exists -# $1 -> page (inkl namespace prefix) -# $2 -> wiki -checkIfPageExists() { - page="${1}" - wiki="${2}" - wikiApiUrl="${WIKI_BASE_URL}/${wiki}/api.php" - ckf="cookie_${wiki}.ck" - - rawResult=$( - curl \ - -s \ - -b "$ckf" \ - -c "$ckf" \ - --data-urlencode "titles=${page}" \ - --data-urlencode "prop=info" \ - -H "User-Agent: ${userAgent}" \ - -H 'Accept-Encoding: gzip' \ - -X POST "${wikiApiUrl}?format=json&action=query" \ - | gunzip - ) - - # Don't get rate limited - sleep 4 - - if [[ $rawResult == *'missing":"'* ]]; then - pageExists=false - else - pageExists=true - fi -} - -# protects a specified page on a specified wiki against creation -# if the page already exists it will issue a warning -# $1 -> page (inkl namespace prefix) -# $2 -> wiki -protectNonExistingPage() { - page="${1}" - wiki=$2 - - checkIfPageExists "${page}" $wiki - if $pageExists; then - echo "::warning::$page already exists on $wiki" - protectErrors+=("create:${wiki}:${page}") - else - protectPage "${page}" "${wiki}" "create" - fi -} - -# protects a specified page on a specified wiki against editing/moving -# $1 -> page (inkl namespace prefix) -# $2 -> wiki -protectExistingPage() { - protectPage "${1}" "${2}" "edit" -} diff --git a/scripts/protect_templates.py b/scripts/protect_templates.py new file mode 100644 index 00000000000..488b93be20b --- /dev/null +++ b/scripts/protect_templates.py @@ -0,0 +1,28 @@ +import os + +from protect_page import ( + protect_non_existing_page, + protect_existing_page, + handle_protect_errors, +) + +WIKI_TO_PROTECT = os.getenv("WIKI_TO_PROTECT") + + +def main(): + with open("./templates/templatesToProtect", "r") as templates_to_protect: + for template_name in templates_to_protect.read().splitlines(): + if len(template_name.strip()) == 0: + continue + template = "Template:" + template_name + print(f"::group::Checking {WIKI_TO_PROTECT}:{template}") + if WIKI_TO_PROTECT == "commons": + protect_existing_page(template, WIKI_TO_PROTECT) + else: + protect_non_existing_page(template, WIKI_TO_PROTECT) + print("::endgroup::") + handle_protect_errors() + + +if __name__ == "__main__": + main() diff --git a/scripts/protect_templates.sh b/scripts/protect_templates.sh deleted file mode 100644 index 181203e73ab..00000000000 --- a/scripts/protect_templates.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -. ./scripts/protect_page.sh - -readarray filesToProtect < "./templates/templatesToProtect" - -for fileToProtect in "${filesToProtect[@]}"; do - template="Template:${fileToProtect}" - echo "::group::Checking ${WIKI_TO_PROTECT}:${template}" - if [[ "commons" == ${WIKI_TO_PROTECT} ]]; then - protectExistingPage "${template}" ${WIKI_TO_PROTECT} - else - protectNonExistingPage "${template}" ${WIKI_TO_PROTECT} - fi - echo '::endgroup::' -done - -rm -f cookie_* - -if [[ ${#protectErrors[@]} -ne 0 ]]; then - echo "::warning::Some templates could not be protected" - echo ":warning: Some templates could not be protected" >> $GITHUB_STEP_SUMMARY - echo "::group::Failed protections" - for value in "${protectErrors[@]}"; do - echo "... ${value}" - done - echo "::endgroup::" - exit 1 -fi diff --git a/scripts/remove_dev.py b/scripts/remove_dev.py new file mode 100644 index 00000000000..c8918c96652 --- /dev/null +++ b/scripts/remove_dev.py @@ -0,0 +1,102 @@ +import os +import time + +import requests + +from deploy_util import ( + HEADER, + SLEEP_DURATION, + get_wiki_api_url, + get_wikis, + read_cookie_jar, + write_to_github_summary_file, +) +from login_and_get_token import get_token, login + +LUA_DEV_ENV_NAME = os.getenv("LUA_DEV_ENV_NAME") + +remove_errors: list[str] = list() + + +def remove_page(session: requests.Session, page: str, wiki: str): + print(f"deleting {wiki}:{page}") + token = get_token(wiki) + + result = session.post( + get_wiki_api_url(wiki), + headers=HEADER, + params={ + "format": "json", + "action": "delete", + }, + data={ + "title": page, + "reason": f"Remove {LUA_DEV_ENV_NAME}", + "token": token, + }, + ).json() + time.sleep(SLEEP_DURATION) + + if "delete" not in result.keys(): + print(f"::warning::could not delete {page} on {wiki}") + write_to_github_summary_file(f":warning: could not delete {page} on {wiki}") + remove_errors.append(f"{wiki}:{page}") + + +def search_and_remove(wiki: str): + with requests.Session() as session: + search_result = session.post( + get_wiki_api_url(wiki), + headers=HEADER, + params={"format": "json", "action": "query"}, + data={ + "list": "search", + "srsearch": f"intitle:{LUA_DEV_ENV_NAME}", + "srnamespace": 828, + "srlimit": 5000, + "srprop": "", + }, + ).json() + time.sleep(SLEEP_DURATION) + + # Handle API error responses and missing or empty search results safely. + if "error" in search_result: + error_info = search_result.get("error") + print(f"::warning::search API error on {wiki}: {error_info}") + write_to_github_summary_file( + f":warning: search API error on {wiki}: {error_info}" + ) + return + + pages = search_result.get("query", {}).get("search") or [] + if len(pages) == 0: + return + + login(wiki) + session.cookies = read_cookie_jar(wiki) + + for page in pages: + if os.getenv("INCLUDE_SUB_ENVS") == "true" or page["title"].endswith( + LUA_DEV_ENV_NAME + ): + remove_page(session, page["title"], wiki) + + +def main(): + for wiki in get_wikis(): + if wiki == "commons" and os.getenv("INCLUDE_COMMONS") != "true": + continue + search_and_remove(wiki) + if len(remove_errors) == 0: + exit(0) + print("::warning::Could not delete some pages on some wikis") + write_to_github_summary_file("::warning::Could not delete some pages on some wikis") + print("::group::Failed protections") + for remove_error in remove_errors: + print(remove_error) + print("endgroup") + exit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/remove_dev.sh b/scripts/remove_dev.sh deleted file mode 100644 index f1f7536b493..00000000000 --- a/scripts/remove_dev.sh +++ /dev/null @@ -1,112 +0,0 @@ -#!/bin/bash - -userAgent="GitHub Autodeploy Bot/1.1.0 (${WIKI_UA_EMAIL})" - -. ./scripts/login_and_get_token.sh - -ckf="cookie_base.ck" -declare -a removeErrors -declare -a allWikis -allWikis=$( - curl \ - -s \ - -b "$ckf" \ - -c "$ckf" \ - -H "User-Agent: ${userAgent}" \ - -H 'Accept-Encoding: gzip' \ - -X GET "https://liquipedia.net/api.php?action=listwikis" \ - | gunzip \ - | jq '.allwikis | keys[]' -r -) -# Don't get rate limited -sleep 4 - -removePage() { - page="${1}" - wiki=$2 - - echo "deleting ${wiki}:${page}" - - wikiApiUrl="${WIKI_BASE_URL}/${wiki}/api.php" - ckf="cookie_${wiki}.ck" - - getToken $wiki - - rawRemoveResult=$( - curl \ - -s \ - -b "$ckf" \ - -c "$ckf" \ - --data-urlencode "title=${page}" \ - --data-urlencode "reason=Remove ${LUA_DEV_ENV_NAME}" \ - --data-urlencode "token=${token}" \ - -H "User-Agent: ${userAgent}" \ - -H 'Accept-Encoding: gzip' \ - -X POST "${wikiApiUrl}?format=json&action=delete" \ - | gunzip - ) - # Don't get rate limited - sleep 8 - - if [[ $rawRemoveResult != *"delete"* ]]; then - echo "::warning::could not delete ${page} on ${wiki}" - echo "::warning::could not delete ${page} on ${wiki}" >> $GITHUB_STEP_SUMMARY - removeErrorMsg="${wiki}:${page}" - removeErrors+=("${removeErrorMsg}") - fi -} - -searchAndRemove(){ - wiki=$1 - - wikiApiUrl="${WIKI_BASE_URL}/${wiki}/api.php" - - rawSearchResult=$( - curl \ - -s \ - -b "$ckf" \ - -c "$ckf" \ - --data-urlencode "list=search" \ - --data-urlencode "srsearch=intitle:${LUA_DEV_ENV_NAME}" \ - --data-urlencode "srnamespace=828" \ - --data-urlencode "srlimit=5000" \ - --data-urlencode "srprop=" \ - -H "User-Agent: ${userAgent}" \ - -H 'Accept-Encoding: gzip' \ - -X POST "${wikiApiUrl}?format=json&action=query" \ - | gunzip - ) - - sleep 4 - - pages=($(echo "$rawSearchResult" | jq ".query.search[] | .title" -r -c)) - - if [[ -n $pages && ${#pages[@]} -ne 0 ]]; then - for page in ${pages[@]}; do - if [[ ${INCLUDE_SUB_ENVS} == true || "${page}" == *"${LUA_DEV_ENV_NAME}" ]]; then - removePage $page $wiki - fi - done - fi -} - -for wiki in $allWikis; do - if [[ $wiki != "commons" || ${INCLUDE_COMMONS} == true ]]; then - echo "::group::Checking $wiki" - searchAndRemove $wiki - fi -done - -rm -f cookie_* - -if [[ ${#removeErrors[@]} -ne 0 ]]; then - echo "::warning::Could not delete some pages on some wikis" - echo "::warning::Could not delete some pages on some wikis" >> $GITHUB_STEP_SUMMARY - echo "::group::Failed protections" - for value in "${removeErrors[@]}"; do - echo "... ${value}" - done - echo "::endgroup::" - - exit 1 -fi