From 3482b523d07cc13630c96e187c7d90c3f7a29a11 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Tue, 3 Feb 2026 16:20:10 +0900 Subject: [PATCH 01/54] rewrite login_and_get_token --- scripts/login_and_get_token.py | 78 ++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 scripts/login_and_get_token.py diff --git a/scripts/login_and_get_token.py b/scripts/login_and_get_token.py new file mode 100644 index 00000000000..9eda4917780 --- /dev/null +++ b/scripts/login_and_get_token.py @@ -0,0 +1,78 @@ +import asyncio +import os + +import aiohttp + +USER_AGENT = f"GitHub Autodeploy Bot/1.1.0 ({ os.getenv("WIKI_UA_EMAIL") })" +WIKI_BASE_URL = os.getenv("WIKI_BASE_URL") +WIKI_USER = os.getenv("WIKI_USER") +WIKI_PASSWORD = os.getenv("WIKI_PASSWORD") + +loggedin: set[str] = set() +loggedin_lock = asyncio.Lock() + + +async def login(wiki: str): + await loggedin_lock.acquire() + try: + if wiki in loggedin: + return + ckf = f"cookie_{wiki}.ck" + cookie_jar = aiohttp.CookieJar() + if os.path.exists(ckf): + cookie_jar.load(ckf) + print(f"...logging in on { wiki }") + async with aiohttp.ClientSession( + WIKI_BASE_URL + "/", + headers={"User-Agent": USER_AGENT, "Accept-Encoding": "gzip"}, + cookie_jar=cookie_jar, + ) as session: + token_response = await session.post( + "api.php", + params={ + "format": "json", + "action": "query", + "meta": "tokens", + "type": "login", + }, + ) + response = await token_response.json() + await session.post( + "api.php", + data={ + "lgname": WIKI_USER, + "lgpassword": WIKI_PASSWORD, + "lgtoken": response["query"]["tokens"]["logintoken"], + }, + params={"format": "json", "action": "login"}, + ) + loggedin.add(wiki) + cookie_jar.save(ckf) + await asyncio.sleep(4) + + finally: + loggedin_lock.release() + + +async def get_token(wiki: str): + await login(wiki) + + ckf = f"cookie_{wiki}.ck" + cookie_jar = aiohttp.CookieJar() + if os.path.exists(ckf): + cookie_jar.load(ckf) + async with aiohttp.ClientSession( + WIKI_BASE_URL + "/", + headers={"User-Agent": USER_AGENT, "Accept-Encoding": "gzip"}, + cookie_jar=cookie_jar, + ) as session: + token_response = await session.post( + "api.php", + params={ + "format": "json", + "action": "query", + "meta": "tokens", + }, + ) + response = await token_response.json() + return response["query"]["tokens"]["csrftoken"] From a21dc56b2d9d792f887eccb47799830b7b48002f Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Wed, 4 Feb 2026 00:57:27 +0900 Subject: [PATCH 02/54] add deploy script --- scripts/deploy.py | 123 +++++++++++++++++++++++++++++++++ scripts/login_and_get_token.py | 8 ++- 2 files changed, 128 insertions(+), 3 deletions(-) create mode 100644 scripts/deploy.py diff --git a/scripts/deploy.py b/scripts/deploy.py new file mode 100644 index 00000000000..16e6f2d7f7f --- /dev/null +++ b/scripts/deploy.py @@ -0,0 +1,123 @@ +import asyncio +import itertools +import os +import pathlib +import re +import sys +import subprocess + +from typing import Iterable + +import aiohttp + +from login_and_get_token import * + +DEPLOY_TRIGGER = os.getenv("DEPLOY_TRIGGER") +HEADER_PATTERN = re.compile( + r"\A---\n" r"-- @Liquipedia\n" r"-- page=(?P[^\n]*)\n" +) +GITHUB_STEP_SUMMARY_FILE = os.getenv("GITHUB_STEP_SUMMARY") + +all_modules_deployed: bool = True +gh_summary_write_lock = asyncio.Lock() + + +async def write_to_github_summary_file(text: str): + if not GITHUB_STEP_SUMMARY_FILE: + return + await gh_summary_write_lock.acquire() + try: + with open(GITHUB_STEP_SUMMARY_FILE, "a") as summary: + summary.write(f"{text}\n") + finally: + gh_summary_write_lock.release() + + +async def deploy_all_files_for_wiki( + wiki: str, file_paths: Iterable[pathlib.Path], deploy_reason: str +): + token = await get_token(wiki) + ckf = f"cookie_{wiki}.ck" + cookie_jar = aiohttp.CookieJar() + if os.path.exists(ckf): + cookie_jar.load(ckf) + session = aiohttp.ClientSession( + f"{WIKI_BASE_URL}/{wiki}/", + headers={"User-Agent": USER_AGENT, "Accept-Encoding": "gzip"}, + cookie_jar=cookie_jar, + ) + for file_path in file_paths: + output: list[str] = [f"::group::Checking {str(file_path)}"] + with file_path.open("r") as file: + file_content = file.read() + header_match = HEADER_PATTERN.match(file_content) + if not header_match: + output.append("...skipping - no magic comment found") + await write_to_github_summary_file(f"{str(file_path)} skipped") + else: + page = header_match.groupdict()["pageName"] + ( + os.getenv("LUA_DEV_ENV_NAME") or "" + ) + response = await session.post( + "api.php", + params={"format": "json", "action": "edit"}, + data={ + "title": page, + "text": file_content, + "summary": f"Git: {deploy_reason.strip()}", + "bot": "true", + "recreate": "true", + "token": token, + }, + ) + parsed_response = await response.json() + result = parsed_response["edit"]["result"] + if result == "Success": + no_change = parsed_response["edit"]["nochange"] + if len(no_change) == 0 and DEPLOY_TRIGGER == "push": + output.append(f"::notice file={str(file_path)}::No change made") + elif len(no_change) != 0 and DEPLOY_TRIGGER != "push": + output.append(f"::warning file={str(file_path)}::File changed") + output.append("...done") + await write_to_github_summary_file( + f":information_source: {str(file_path)} successfully deployed" + ) + else: + all_modules_deployed = False + output.append(f"::warning file={str(file_path)}::failed to deploy") + await write_to_github_summary_file( + f":warning: {str(file_path)} failed to deploy" + ) + await asyncio.sleep(4) + output.append("::endgroup::") + print("\n".join(output)) + + +async def async_main(): + lua_files: Iterable[pathlib.Path] + git_deploy_reason: str + if len(sys.argv[1:]) == 0: + lua_files = pathlib.Path("./").glob("lua/wikis/**/*.lua") + git_deploy_reason = "Automated Weekly Re-Sync" + else: + lua_files = [pathlib.Path(arg) for arg in sys.argv[1:]] + git_deploy_reason = subprocess.check_output( + ["git", "log", "-1", "--pretty='%h %s'"] + ).decode() + + deploy_tasks: list[asyncio.Task] = list() + + for wiki, files in itertools.groupby(lua_files, lambda path: path.parts[2]): + deploy_tasks.append( + asyncio.Task( + deploy_all_files_for_wiki(wiki, list(files), git_deploy_reason) + ) + ) + + await asyncio.wait(deploy_tasks) + if not all_modules_deployed: + print("::warning::Some modules were not deployed!") + sys.exit(1) + + +asyncio.run(async_main()) diff --git a/scripts/login_and_get_token.py b/scripts/login_and_get_token.py index 9eda4917780..be01c96d0ae 100644 --- a/scripts/login_and_get_token.py +++ b/scripts/login_and_get_token.py @@ -3,6 +3,8 @@ import aiohttp +__all__ = ["USER_AGENT", "WIKI_BASE_URL" "get_token"] + USER_AGENT = f"GitHub Autodeploy Bot/1.1.0 ({ os.getenv("WIKI_UA_EMAIL") })" WIKI_BASE_URL = os.getenv("WIKI_BASE_URL") WIKI_USER = os.getenv("WIKI_USER") @@ -23,7 +25,7 @@ async def login(wiki: str): cookie_jar.load(ckf) print(f"...logging in on { wiki }") async with aiohttp.ClientSession( - WIKI_BASE_URL + "/", + f"{WIKI_BASE_URL}/{wiki}/", headers={"User-Agent": USER_AGENT, "Accept-Encoding": "gzip"}, cookie_jar=cookie_jar, ) as session: @@ -54,7 +56,7 @@ async def login(wiki: str): loggedin_lock.release() -async def get_token(wiki: str): +async def get_token(wiki: str) -> str: await login(wiki) ckf = f"cookie_{wiki}.ck" @@ -62,7 +64,7 @@ async def get_token(wiki: str): if os.path.exists(ckf): cookie_jar.load(ckf) async with aiohttp.ClientSession( - WIKI_BASE_URL + "/", + f"{WIKI_BASE_URL}/{wiki}/", headers={"User-Agent": USER_AGENT, "Accept-Encoding": "gzip"}, cookie_jar=cookie_jar, ) as session: From 54d41dfacf5b599d8e4b5ed0aeb8981fef706c1d Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Wed, 4 Feb 2026 10:53:33 +0900 Subject: [PATCH 03/54] syntax --- scripts/deploy.py | 1 + scripts/login_and_get_token.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/deploy.py b/scripts/deploy.py index 16e6f2d7f7f..5f0357507d1 100644 --- a/scripts/deploy.py +++ b/scripts/deploy.py @@ -91,6 +91,7 @@ async def deploy_all_files_for_wiki( await asyncio.sleep(4) output.append("::endgroup::") print("\n".join(output)) + await session.close() async def async_main(): diff --git a/scripts/login_and_get_token.py b/scripts/login_and_get_token.py index be01c96d0ae..40e74faf8e3 100644 --- a/scripts/login_and_get_token.py +++ b/scripts/login_and_get_token.py @@ -3,7 +3,7 @@ import aiohttp -__all__ = ["USER_AGENT", "WIKI_BASE_URL" "get_token"] +__all__ = ["USER_AGENT", "WIKI_BASE_URL", "get_token"] USER_AGENT = f"GitHub Autodeploy Bot/1.1.0 ({ os.getenv("WIKI_UA_EMAIL") })" WIKI_BASE_URL = os.getenv("WIKI_BASE_URL") From c8455510878ab51f264fefc11ba9ef8efefcb151 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Wed, 4 Feb 2026 12:39:06 +0900 Subject: [PATCH 04/54] disable async deploy --- scripts/deploy.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/scripts/deploy.py b/scripts/deploy.py index 5f0357507d1..e849dd779b7 100644 --- a/scripts/deploy.py +++ b/scripts/deploy.py @@ -106,16 +106,19 @@ async def async_main(): ["git", "log", "-1", "--pretty='%h %s'"] ).decode() - deploy_tasks: list[asyncio.Task] = list() + # # Asynchronous deploy is disabled due to rate limit + # deploy_tasks: list[asyncio.Task] = list() + # for wiki, files in itertools.groupby(lua_files, lambda path: path.parts[2]): + # deploy_tasks.append( + # asyncio.Task( + # deploy_all_files_for_wiki(wiki, list(files), git_deploy_reason) + # ) + # ) + # await asyncio.wait(deploy_tasks) for wiki, files in itertools.groupby(lua_files, lambda path: path.parts[2]): - deploy_tasks.append( - asyncio.Task( - deploy_all_files_for_wiki(wiki, list(files), git_deploy_reason) - ) - ) + await deploy_all_files_for_wiki(wiki, list(files), git_deploy_reason) - await asyncio.wait(deploy_tasks) if not all_modules_deployed: print("::warning::Some modules were not deployed!") sys.exit(1) From 6774361963d02669f42ba7373aa4f5451d36caff Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Wed, 4 Feb 2026 14:40:04 +0900 Subject: [PATCH 05/54] rewrite with requests --- scripts/deploy.py | 102 ++++++++++++++---------------- scripts/login_and_get_token.py | 109 ++++++++++++++++----------------- 2 files changed, 100 insertions(+), 111 deletions(-) diff --git a/scripts/deploy.py b/scripts/deploy.py index e849dd779b7..c7a0b41ed7b 100644 --- a/scripts/deploy.py +++ b/scripts/deploy.py @@ -1,14 +1,15 @@ -import asyncio +import http.cookiejar import itertools import os import pathlib import re import sys import subprocess +import time from typing import Iterable -import aiohttp +import requests from login_and_get_token import * @@ -18,48 +19,52 @@ ) GITHUB_STEP_SUMMARY_FILE = os.getenv("GITHUB_STEP_SUMMARY") + all_modules_deployed: bool = True -gh_summary_write_lock = asyncio.Lock() -async def write_to_github_summary_file(text: str): +def write_to_github_summary_file(text: str): if not GITHUB_STEP_SUMMARY_FILE: return - await gh_summary_write_lock.acquire() - try: - with open(GITHUB_STEP_SUMMARY_FILE, "a") as summary: - summary.write(f"{text}\n") - finally: - gh_summary_write_lock.release() + with open(GITHUB_STEP_SUMMARY_FILE, "a") as summary: + summary.write(f"{text}\n") + + +def read_file_from_path(file_path: pathlib.Path) -> str: + with file_path.open("r") as file: + return file.read() -async def deploy_all_files_for_wiki( +def deploy_all_files_for_wiki( wiki: str, file_paths: Iterable[pathlib.Path], deploy_reason: str ): - token = await get_token(wiki) + token = get_token(wiki) ckf = f"cookie_{wiki}.ck" - cookie_jar = aiohttp.CookieJar() - if os.path.exists(ckf): - cookie_jar.load(ckf) - session = aiohttp.ClientSession( - f"{WIKI_BASE_URL}/{wiki}/", - headers={"User-Agent": USER_AGENT, "Accept-Encoding": "gzip"}, - cookie_jar=cookie_jar, - ) - for file_path in file_paths: - output: list[str] = [f"::group::Checking {str(file_path)}"] - with file_path.open("r") as file: - file_content = file.read() + cookie_jar = http.cookiejar.LWPCookieJar(filename=ckf) + try: + cookie_jar.load(ignore_discard=True) + except: + pass + with requests.Session() as session: + session.cookies = cookie_jar + for file_path in file_paths: + print(f"::group::Checking {str(file_path)}") + file_content = read_file_from_path(file_path) header_match = HEADER_PATTERN.match(file_content) if not header_match: - output.append("...skipping - no magic comment found") - await write_to_github_summary_file(f"{str(file_path)} skipped") + print("...skipping - no magic comment found") + write_to_github_summary_file(f"{str(file_path)} skipped") else: page = header_match.groupdict()["pageName"] + ( os.getenv("LUA_DEV_ENV_NAME") or "" ) - response = await session.post( - "api.php", + response = session.post( + f"{WIKI_BASE_URL}/{wiki}/api.php", + headers={ + "User-Agent": USER_AGENT, + "accept": "application/json", + "Accept-Encoding": "gzip", + }, params={"format": "json", "action": "edit"}, data={ "title": page, @@ -69,32 +74,29 @@ async def deploy_all_files_for_wiki( "recreate": "true", "token": token, }, - ) - parsed_response = await response.json() - result = parsed_response["edit"]["result"] + ).json() + result = response["edit"]["result"] if result == "Success": - no_change = parsed_response["edit"]["nochange"] + no_change = response["edit"]["nochange"] if len(no_change) == 0 and DEPLOY_TRIGGER == "push": - output.append(f"::notice file={str(file_path)}::No change made") + print(f"::notice file={str(file_path)}::No change made") elif len(no_change) != 0 and DEPLOY_TRIGGER != "push": - output.append(f"::warning file={str(file_path)}::File changed") - output.append("...done") - await write_to_github_summary_file( + print(f"::warning file={str(file_path)}::File changed") + print("...done") + write_to_github_summary_file( f":information_source: {str(file_path)} successfully deployed" ) else: all_modules_deployed = False - output.append(f"::warning file={str(file_path)}::failed to deploy") - await write_to_github_summary_file( + print(f"::warning file={str(file_path)}::failed to deploy") + write_to_github_summary_file( f":warning: {str(file_path)} failed to deploy" ) - await asyncio.sleep(4) - output.append("::endgroup::") - print("\n".join(output)) - await session.close() + time.sleep(4) + print("::endgroup::") -async def async_main(): +def main(): lua_files: Iterable[pathlib.Path] git_deploy_reason: str if len(sys.argv[1:]) == 0: @@ -106,22 +108,12 @@ async def async_main(): ["git", "log", "-1", "--pretty='%h %s'"] ).decode() - # # Asynchronous deploy is disabled due to rate limit - # deploy_tasks: list[asyncio.Task] = list() - # for wiki, files in itertools.groupby(lua_files, lambda path: path.parts[2]): - # deploy_tasks.append( - # asyncio.Task( - # deploy_all_files_for_wiki(wiki, list(files), git_deploy_reason) - # ) - # ) - # await asyncio.wait(deploy_tasks) - for wiki, files in itertools.groupby(lua_files, lambda path: path.parts[2]): - await deploy_all_files_for_wiki(wiki, list(files), git_deploy_reason) + deploy_all_files_for_wiki(wiki, list(files), git_deploy_reason) if not all_modules_deployed: print("::warning::Some modules were not deployed!") sys.exit(1) -asyncio.run(async_main()) +main() diff --git a/scripts/login_and_get_token.py b/scripts/login_and_get_token.py index 40e74faf8e3..b7869bde989 100644 --- a/scripts/login_and_get_token.py +++ b/scripts/login_and_get_token.py @@ -1,7 +1,8 @@ -import asyncio +import http.cookiejar import os +import time -import aiohttp +import requests __all__ = ["USER_AGENT", "WIKI_BASE_URL", "get_token"] @@ -11,70 +12,66 @@ WIKI_PASSWORD = os.getenv("WIKI_PASSWORD") loggedin: set[str] = set() -loggedin_lock = asyncio.Lock() -async def login(wiki: str): - await loggedin_lock.acquire() +def login(wiki: str): + if wiki in loggedin: + return + ckf = f"cookie_{wiki}.ck" + cookie_jar = http.cookiejar.LWPCookieJar(filename=ckf) try: - if wiki in loggedin: - return - ckf = f"cookie_{wiki}.ck" - cookie_jar = aiohttp.CookieJar() - if os.path.exists(ckf): - cookie_jar.load(ckf) - print(f"...logging in on { wiki }") - async with aiohttp.ClientSession( - f"{WIKI_BASE_URL}/{wiki}/", + cookie_jar.load(ignore_discard=True) + except: + pass + print(f"...logging in on { wiki }") + with requests.Session() as session: + session.cookies = cookie_jar + token_response = session.post( + f"{WIKI_BASE_URL}/{wiki}/api.php", headers={"User-Agent": USER_AGENT, "Accept-Encoding": "gzip"}, - cookie_jar=cookie_jar, - ) as session: - token_response = await session.post( - "api.php", - params={ - "format": "json", - "action": "query", - "meta": "tokens", - "type": "login", - }, - ) - response = await token_response.json() - await session.post( - "api.php", - data={ - "lgname": WIKI_USER, - "lgpassword": WIKI_PASSWORD, - "lgtoken": response["query"]["tokens"]["logintoken"], - }, - params={"format": "json", "action": "login"}, - ) - loggedin.add(wiki) - cookie_jar.save(ckf) - await asyncio.sleep(4) - - finally: - loggedin_lock.release() + cookies=cookie_jar, + params={ + "format": "json", + "action": "query", + "meta": "tokens", + "type": "login", + }, + ).json() + session.post( + f"{WIKI_BASE_URL}/{wiki}/api.php", + headers={"User-Agent": USER_AGENT, "Accept-Encoding": "gzip"}, + cookies=cookie_jar, + data={ + "lgname": WIKI_USER, + "lgpassword": WIKI_PASSWORD, + "lgtoken": token_response["query"]["tokens"]["logintoken"], + }, + params={"format": "json", "action": "login"}, + ) + loggedin.add(wiki) + cookie_jar.save(ignore_discard=True) + time.sleep(4) -async def get_token(wiki: str) -> str: - await login(wiki) +def get_token(wiki: str) -> str: + login(wiki) ckf = f"cookie_{wiki}.ck" - cookie_jar = aiohttp.CookieJar() - if os.path.exists(ckf): - cookie_jar.load(ckf) - async with aiohttp.ClientSession( - f"{WIKI_BASE_URL}/{wiki}/", - headers={"User-Agent": USER_AGENT, "Accept-Encoding": "gzip"}, - cookie_jar=cookie_jar, - ) as session: - token_response = await session.post( - "api.php", + cookie_jar = http.cookiejar.LWPCookieJar(filename=ckf) + try: + cookie_jar.load(ignore_discard=True) + except: + pass + with requests.Session() as session: + session.cookies = cookie_jar + token_response = session.post( + f"{WIKI_BASE_URL}/{wiki}/api.php", + headers={"User-Agent": USER_AGENT, "Accept-Encoding": "gzip"}, + cookies=cookie_jar, params={ "format": "json", "action": "query", "meta": "tokens", }, - ) - response = await token_response.json() - return response["query"]["tokens"]["csrftoken"] + ).json() + return token_response["query"]["tokens"]["csrftoken"] From c5f9062e19c2d4177f9be7fd46cbf03c61bacb2e Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Wed, 4 Feb 2026 15:07:01 +0900 Subject: [PATCH 06/54] memoize tokens --- scripts/login_and_get_token.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/login_and_get_token.py b/scripts/login_and_get_token.py index b7869bde989..da5dcea5504 100644 --- a/scripts/login_and_get_token.py +++ b/scripts/login_and_get_token.py @@ -1,3 +1,4 @@ +import functools import http.cookiejar import os import time @@ -53,6 +54,7 @@ def login(wiki: str): time.sleep(4) +@functools.cache def get_token(wiki: str) -> str: login(wiki) From 1cecef4bec6fb026e75eaf41a44138412ee04b71 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Wed, 4 Feb 2026 15:13:09 +0900 Subject: [PATCH 07/54] res deploy script to python --- scripts/deploy_res.py | 157 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 scripts/deploy_res.py diff --git a/scripts/deploy_res.py b/scripts/deploy_res.py new file mode 100644 index 00000000000..ad137da3315 --- /dev/null +++ b/scripts/deploy_res.py @@ -0,0 +1,157 @@ +import http.cookiejar +import itertools +import os +import pathlib +import sys +import subprocess +import time + +from typing import Iterable + +import requests + +from login_and_get_token import * + +DEPLOY_TRIGGER = os.getenv("DEPLOY_TRIGGER") +GITHUB_STEP_SUMMARY_FILE = os.getenv("GITHUB_STEP_SUMMARY") + + +all_deployed: bool = True +changes_made: bool = False + + +def write_to_github_summary_file(text: str): + if not GITHUB_STEP_SUMMARY_FILE: + return + with open(GITHUB_STEP_SUMMARY_FILE, "a") as summary: + summary.write(f"{text}\n") + + +def read_file_from_path(file_path: pathlib.Path) -> str: + with file_path.open("r") as file: + return file.read() + + +def deploy_resources( + res_type: str, file_paths: Iterable[pathlib.Path], deploy_reason: str +): + token = get_token("commons") + ckf = f"cookie_commons.ck" + cookie_jar = http.cookiejar.LWPCookieJar(filename=ckf) + try: + cookie_jar.load(ignore_discard=True) + except: + pass + with requests.Session() as session: + session.cookies = cookie_jar + for file_path in file_paths: + print(f"::group::Checking {str(file_path)}") + file_content = read_file_from_path(file_path) + page = ( + f"MediaWiki:Common.{ "js" if res_type == ".js" else "css" }/" + + file_path.name + ) + print(f"...page = { page }") + response = session.post( + f"{WIKI_BASE_URL}/commons/api.php", + headers={ + "User-Agent": USER_AGENT, + "accept": "application/json", + "Accept-Encoding": "gzip", + }, + params={"format": "json", "action": "edit"}, + data={ + "title": page, + "text": file_content, + "summary": f"Git: {deploy_reason.strip()}", + "bot": "true", + "recreate": "true", + "token": token, + }, + ).json() + print(response) + result = response["edit"]["result"] + new_rev_id = response["edit"].get("newrevid") + if result == "Success": + if new_rev_id is not None: + changes_made = True + if DEPLOY_TRIGGER != "push": + print(f"::warning file={str(file_path)}::File changed") + print(f"...{result}") + print("...done") + write_to_github_summary_file( + f":information_source: {str(file_path)} successfully deployed" + ) + + else: + print(f"::warning file={str(file_path)}::failed to deploy") + write_to_github_summary_file( + f":warning: {str(file_path)} failed to deploy" + ) + all_deployed = False + print("::endgroup::") + time.sleep(4) + + +def update_cache(): + ckf = f"cookie_commons.ck" + cookie_jar = http.cookiejar.LWPCookieJar(filename=ckf) + try: + cookie_jar.load(ignore_discard=True) + except: + pass + with requests.Session() as session: + session.cookies = cookie_jar + cache_result = session.post( + f"{WIKI_BASE_URL}/commons/api.php", + headers={ + "User-Agent": USER_AGENT, + "accept": "application/json", + "Accept-Encoding": "gzip", + }, + params={"format": "json", "action": "updatelpmwmessageapi"}, + data={ + "messagename": "Resourceloaderarticles-cacheversion", + "value": subprocess.check_output( + ["git", "log", "-1", "--pretty='%h'"] + ).decode(), + }, + ).json() + if ( + cache_result["updatelpmwmessageapi"].get("message") + == "Successfully changed the message value" + ): + print("Resource cache version updated succesfully!") + else: + print("::error::Resource cache version unable to be updated!") + exit(1) + + +def main(): + resource_files: Iterable[pathlib.Path] + git_deploy_reason: str + if len(sys.argv[1:]) == 0: + resource_files = itertools.chain( + pathlib.Path("./").glob("javascript/**/*.js"), + pathlib.Path("./").glob("stylesheets/**/*.scss"), + ) + git_deploy_reason = "Automated Weekly Re-Sync" + else: + resource_files = [pathlib.Path(arg) for arg in sys.argv[1:]] + git_deploy_reason = subprocess.check_output( + ["git", "log", "-1", "--pretty='%h %s'"] + ).decode() + + for res_type, files in itertools.groupby(resource_files, lambda path: path.suffix): + deploy_resources(res_type, list(files), git_deploy_reason) + + if not all_deployed: + print( + "::error::Some files were not deployed; resource cache version not updated!" + ) + exit(1) + elif changes_made: + update_cache() + + +main() From ac73a24e40c3f22ae711b56732596902b575a60d Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Wed, 4 Feb 2026 15:35:00 +0900 Subject: [PATCH 08/54] refactor --- scripts/deploy.py | 39 ++++------------------ scripts/deploy_res.py | 56 +++++-------------------------- scripts/deploy_util.py | 60 ++++++++++++++++++++++++++++++++++ scripts/login_and_get_token.py | 37 +++++++-------------- 4 files changed, 86 insertions(+), 106 deletions(-) create mode 100644 scripts/deploy_util.py diff --git a/scripts/deploy.py b/scripts/deploy.py index c7a0b41ed7b..930063efef1 100644 --- a/scripts/deploy.py +++ b/scripts/deploy.py @@ -1,52 +1,31 @@ -import http.cookiejar import itertools import os import pathlib import re import sys -import subprocess import time from typing import Iterable import requests +from deploy_util import * from login_and_get_token import * -DEPLOY_TRIGGER = os.getenv("DEPLOY_TRIGGER") HEADER_PATTERN = re.compile( r"\A---\n" r"-- @Liquipedia\n" r"-- page=(?P[^\n]*)\n" ) -GITHUB_STEP_SUMMARY_FILE = os.getenv("GITHUB_STEP_SUMMARY") all_modules_deployed: bool = True -def write_to_github_summary_file(text: str): - if not GITHUB_STEP_SUMMARY_FILE: - return - with open(GITHUB_STEP_SUMMARY_FILE, "a") as summary: - summary.write(f"{text}\n") - - -def read_file_from_path(file_path: pathlib.Path) -> str: - with file_path.open("r") as file: - return file.read() - - def deploy_all_files_for_wiki( wiki: str, file_paths: Iterable[pathlib.Path], deploy_reason: str ): token = get_token(wiki) - ckf = f"cookie_{wiki}.ck" - cookie_jar = http.cookiejar.LWPCookieJar(filename=ckf) - try: - cookie_jar.load(ignore_discard=True) - except: - pass with requests.Session() as session: - session.cookies = cookie_jar + session.cookies = read_cookie_jar(wiki) for file_path in file_paths: print(f"::group::Checking {str(file_path)}") file_content = read_file_from_path(file_path) @@ -59,17 +38,13 @@ def deploy_all_files_for_wiki( os.getenv("LUA_DEV_ENV_NAME") or "" ) response = session.post( - f"{WIKI_BASE_URL}/{wiki}/api.php", - headers={ - "User-Agent": USER_AGENT, - "accept": "application/json", - "Accept-Encoding": "gzip", - }, + get_wiki_api_url(wiki), + headers=HEADER, params={"format": "json", "action": "edit"}, data={ "title": page, "text": file_content, - "summary": f"Git: {deploy_reason.strip()}", + "summary": f"Git: {deploy_reason}", "bot": "true", "recreate": "true", "token": token, @@ -104,9 +79,7 @@ def main(): git_deploy_reason = "Automated Weekly Re-Sync" else: lua_files = [pathlib.Path(arg) for arg in sys.argv[1:]] - git_deploy_reason = subprocess.check_output( - ["git", "log", "-1", "--pretty='%h %s'"] - ).decode() + git_deploy_reason = get_git_deploy_reason() for wiki, files in itertools.groupby(lua_files, lambda path: path.parts[2]): deploy_all_files_for_wiki(wiki, list(files), git_deploy_reason) diff --git a/scripts/deploy_res.py b/scripts/deploy_res.py index ad137da3315..44ce28f5b08 100644 --- a/scripts/deploy_res.py +++ b/scripts/deploy_res.py @@ -1,6 +1,4 @@ -import http.cookiejar import itertools -import os import pathlib import sys import subprocess @@ -10,40 +8,20 @@ import requests +from deploy_util import * from login_and_get_token import * -DEPLOY_TRIGGER = os.getenv("DEPLOY_TRIGGER") -GITHUB_STEP_SUMMARY_FILE = os.getenv("GITHUB_STEP_SUMMARY") - all_deployed: bool = True changes_made: bool = False -def write_to_github_summary_file(text: str): - if not GITHUB_STEP_SUMMARY_FILE: - return - with open(GITHUB_STEP_SUMMARY_FILE, "a") as summary: - summary.write(f"{text}\n") - - -def read_file_from_path(file_path: pathlib.Path) -> str: - with file_path.open("r") as file: - return file.read() - - def deploy_resources( res_type: str, file_paths: Iterable[pathlib.Path], deploy_reason: str ): token = get_token("commons") - ckf = f"cookie_commons.ck" - cookie_jar = http.cookiejar.LWPCookieJar(filename=ckf) - try: - cookie_jar.load(ignore_discard=True) - except: - pass with requests.Session() as session: - session.cookies = cookie_jar + session.cookies = read_cookie_jar("commons") for file_path in file_paths: print(f"::group::Checking {str(file_path)}") file_content = read_file_from_path(file_path) @@ -53,17 +31,13 @@ def deploy_resources( ) print(f"...page = { page }") response = session.post( - f"{WIKI_BASE_URL}/commons/api.php", - headers={ - "User-Agent": USER_AGENT, - "accept": "application/json", - "Accept-Encoding": "gzip", - }, + get_wiki_api_url("commons"), + headers=HEADER, params={"format": "json", "action": "edit"}, data={ "title": page, "text": file_content, - "summary": f"Git: {deploy_reason.strip()}", + "summary": f"Git: {deploy_reason}", "bot": "true", "recreate": "true", "token": token, @@ -94,21 +68,11 @@ def deploy_resources( def update_cache(): - ckf = f"cookie_commons.ck" - cookie_jar = http.cookiejar.LWPCookieJar(filename=ckf) - try: - cookie_jar.load(ignore_discard=True) - except: - pass with requests.Session() as session: - session.cookies = cookie_jar + session.cookies = read_cookie_jar("commons") cache_result = session.post( - f"{WIKI_BASE_URL}/commons/api.php", - headers={ - "User-Agent": USER_AGENT, - "accept": "application/json", - "Accept-Encoding": "gzip", - }, + get_wiki_api_url("commons"), + headers=HEADER, params={"format": "json", "action": "updatelpmwmessageapi"}, data={ "messagename": "Resourceloaderarticles-cacheversion", @@ -138,9 +102,7 @@ def main(): git_deploy_reason = "Automated Weekly Re-Sync" else: resource_files = [pathlib.Path(arg) for arg in sys.argv[1:]] - git_deploy_reason = subprocess.check_output( - ["git", "log", "-1", "--pretty='%h %s'"] - ).decode() + git_deploy_reason = get_git_deploy_reason() for res_type, files in itertools.groupby(resource_files, lambda path: path.suffix): deploy_resources(res_type, list(files), git_deploy_reason) diff --git a/scripts/deploy_util.py b/scripts/deploy_util.py new file mode 100644 index 00000000000..f18fc6dba54 --- /dev/null +++ b/scripts/deploy_util.py @@ -0,0 +1,60 @@ +import functools +import http.cookiejar +import os +import pathlib +import subprocess + +__all__ = [ + "DEPLOY_TRIGGER", + "HEADER", + "get_git_deploy_reason", + "get_wiki_api_url", + "read_cookie_jar", + "read_file_from_path", + "write_to_github_summary_file", +] + +DEPLOY_TRIGGER = os.getenv("DEPLOY_TRIGGER") +GITHUB_STEP_SUMMARY_FILE = os.getenv("GITHUB_STEP_SUMMARY") +USER_AGENT = f"GitHub Autodeploy Bot/1.1.0 ({ os.getenv("WIKI_UA_EMAIL") })" +WIKI_BASE_URL = os.getenv("WIKI_BASE_URL") +HEADER = { + "User-Agent": USER_AGENT, + "accept": "application/json", + "Accept-Encoding": "gzip", +} + + +@functools.cache +def get_wiki_api_url(wiki: str) -> str: + return f"{WIKI_BASE_URL}/{wiki}/api.php" + + +def get_git_deploy_reason(): + return ( + subprocess.check_output(["git", "log", "-1", "--pretty='%h %s'"]) + .decode() + .strip() + ) + + +def read_cookie_jar(wiki: str) -> http.cookiejar.FileCookieJar: + ckf = f"cookie_{wiki}.ck" + cookie_jar = http.cookiejar.LWPCookieJar(filename=ckf) + try: + cookie_jar.load(ignore_discard=True) + except: + pass + return cookie_jar + + +def read_file_from_path(file_path: pathlib.Path) -> str: + with file_path.open("r") as file: + return file.read() + + +def write_to_github_summary_file(text: str): + if not GITHUB_STEP_SUMMARY_FILE: + return + with open(GITHUB_STEP_SUMMARY_FILE, "a") as summary: + summary.write(f"{text}\n") diff --git a/scripts/login_and_get_token.py b/scripts/login_and_get_token.py index da5dcea5504..a6420a86366 100644 --- a/scripts/login_and_get_token.py +++ b/scripts/login_and_get_token.py @@ -1,14 +1,13 @@ import functools -import http.cookiejar import os import time import requests -__all__ = ["USER_AGENT", "WIKI_BASE_URL", "get_token"] +from deploy_util import * + +__all__ = ["get_token"] -USER_AGENT = f"GitHub Autodeploy Bot/1.1.0 ({ os.getenv("WIKI_UA_EMAIL") })" -WIKI_BASE_URL = os.getenv("WIKI_BASE_URL") WIKI_USER = os.getenv("WIKI_USER") WIKI_PASSWORD = os.getenv("WIKI_PASSWORD") @@ -18,19 +17,13 @@ def login(wiki: str): if wiki in loggedin: return - ckf = f"cookie_{wiki}.ck" - cookie_jar = http.cookiejar.LWPCookieJar(filename=ckf) - try: - cookie_jar.load(ignore_discard=True) - except: - pass + cookie_jar = read_cookie_jar(wiki) print(f"...logging in on { wiki }") with requests.Session() as session: session.cookies = cookie_jar token_response = session.post( - f"{WIKI_BASE_URL}/{wiki}/api.php", - headers={"User-Agent": USER_AGENT, "Accept-Encoding": "gzip"}, - cookies=cookie_jar, + get_wiki_api_url(wiki), + headers=HEADER, params={ "format": "json", "action": "query", @@ -39,9 +32,8 @@ def login(wiki: str): }, ).json() session.post( - f"{WIKI_BASE_URL}/{wiki}/api.php", - headers={"User-Agent": USER_AGENT, "Accept-Encoding": "gzip"}, - cookies=cookie_jar, + get_wiki_api_url(wiki), + headers=HEADER, data={ "lgname": WIKI_USER, "lgpassword": WIKI_PASSWORD, @@ -58,18 +50,11 @@ def login(wiki: str): def get_token(wiki: str) -> str: login(wiki) - ckf = f"cookie_{wiki}.ck" - cookie_jar = http.cookiejar.LWPCookieJar(filename=ckf) - try: - cookie_jar.load(ignore_discard=True) - except: - pass with requests.Session() as session: - session.cookies = cookie_jar + session.cookies = read_cookie_jar(wiki) token_response = session.post( - f"{WIKI_BASE_URL}/{wiki}/api.php", - headers={"User-Agent": USER_AGENT, "Accept-Encoding": "gzip"}, - cookies=cookie_jar, + get_wiki_api_url(wiki), + headers=HEADER, params={ "format": "json", "action": "query", From 40b7bff741383beaaf031e40c80033bf268f27a1 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Wed, 4 Feb 2026 16:22:40 +0900 Subject: [PATCH 09/54] protect_page script to python --- scripts/protect_page.py | 91 ++++++++++++++++++++++++++++++++++++ scripts/protect_templates.py | 16 +++++++ 2 files changed, 107 insertions(+) create mode 100644 scripts/protect_page.py create mode 100644 scripts/protect_templates.py diff --git a/scripts/protect_page.py b/scripts/protect_page.py new file mode 100644 index 00000000000..2ca95854824 --- /dev/null +++ b/scripts/protect_page.py @@ -0,0 +1,91 @@ +import time + +from typing import Literal + +import requests + +from deploy_util import * +from login_and_get_token import * + +__all__ = [ + "protect_non_existing_page", + "protect_existing_page", + "handle_protect_errors", +] + +protect_errors = list() + + +def protect_page(page: str, wiki: str, protect_mode: Literal["edit", "create"]): + protect_options: str + if protect_mode == "edit": + protect_options = "edit=allow-only-sysop|move=allow-only-sysop" + elif protect_mode == "create": + protect_options = "create=allow-only-sysop" + else: + raise ValueError(f"invalid protect mode: {protect_mode}") + print(f"...wiki = { wiki }") + print(f"...page = { page }") + token = get_token(wiki) + with requests.Session() as session: + session.cookies = read_cookie_jar(wiki) + response = session.post( + get_wiki_api_url(wiki), + headers=HEADER, + params={"format": "json", "action": "protect"}, + data={ + "title": page, + "protections": protect_options, + "reason": "Git maintained", + "expiry": "infinite", + "bot": "true", + "token": token, + }, + ).json() + + time.sleep(4) + protections = response["protect"]["protections"] + for protection in protections: + if protection[protect_mode] == "allow-only-sysop": + return + print(f"::warning::could not ({protect_mode}) protect {page} on {wiki}") + protect_errors.append(f"{protect_mode}:{wiki}:{page}") + + +def check_if_page_exists(page: str, wiki: str) -> bool: + with requests.Session() as session: + session.cookies = read_cookie_jar(wiki) + + result = session.post( + get_wiki_api_url(wiki), + headers=HEADER, + params={"format": "json", "action": "query"}, + data={"titles": page, "prop": "info"}, + ).text + + time.sleep(4) + return 'missing":"' in result + + +def protect_non_existing_page(page: str, wiki: str): + if check_if_page_exists(page, wiki): + print(f"::warning::{page} already exists on {wiki}") + protect_errors.append(f"create:{wiki}:{page}") + else: + protect_page(page, wiki, "create") + + +def protect_existing_page(page: str, wiki: str): + protect_page(page, wiki, "edit") + + +def handle_protect_errors(): + if len(protect_errors) == 0: + return + print("::warning::Some templates could not be protected") + write_to_github_summary_file(":warning: Some templates could not be protected") + print("::group::Failed protections") + for protect_error in protect_errors: + print(f"... {protect_error}") + print("::endgroup::") + exit(1) diff --git a/scripts/protect_templates.py b/scripts/protect_templates.py new file mode 100644 index 00000000000..bb0f6a756bc --- /dev/null +++ b/scripts/protect_templates.py @@ -0,0 +1,16 @@ +import os + +from protect_page import * + +WIKI_TO_PROTECT = os.getenv("WIKI_TO_PROTECT") + +with open("./templates/templatesToProtect", "r") as templates_to_protect: + for template_name in templates_to_protect.readlines(): + template = "Template:" + template_name + print(f"::group::Checking {WIKI_TO_PROTECT}:{template}") + if WIKI_TO_PROTECT == "commons": + protect_existing_page(template, WIKI_TO_PROTECT) + else: + protect_non_existing_page(template, WIKI_TO_PROTECT) + print("::endgroup::") +handle_protect_errors() From fa44207c41055f0f3cfb062d061ad46fdcda2aac Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Wed, 4 Feb 2026 20:48:57 +0900 Subject: [PATCH 10/54] protect script to python --- scripts/deploy_util.py | 14 +++++++++++ scripts/protect.py | 55 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 scripts/protect.py diff --git a/scripts/deploy_util.py b/scripts/deploy_util.py index f18fc6dba54..556912830fe 100644 --- a/scripts/deploy_util.py +++ b/scripts/deploy_util.py @@ -3,12 +3,16 @@ import os import pathlib import subprocess +import time + +import requests __all__ = [ "DEPLOY_TRIGGER", "HEADER", "get_git_deploy_reason", "get_wiki_api_url", + "get_wikis", "read_cookie_jar", "read_file_from_path", "write_to_github_summary_file", @@ -25,6 +29,16 @@ } +def get_wikis() -> set[str]: + response = requests.get( + "https://liquipedia.net/api.php", + headers=HEADER, + ) + wikis = response.json() + time.sleep(4) + return set(wikis["allwikis"].keys()) + + @functools.cache def get_wiki_api_url(wiki: str) -> str: return f"{WIKI_BASE_URL}/{wiki}/api.php" diff --git a/scripts/protect.py b/scripts/protect.py new file mode 100644 index 00000000000..3e905ada991 --- /dev/null +++ b/scripts/protect.py @@ -0,0 +1,55 @@ +import os +import pathlib +import sys + +from typing import Iterable + +from deploy_util import get_wikis +from protect_page import * + +WIKI_TO_PROTECT = os.getenv("WIKI_TO_PROTECT") + + +def check_for_local_version(module: str, wiki: str): + if wiki == "commons": + return False + return pathlib.Path(f"./lua/wikis/{wiki}/{module}.lua").exists() + + +def protect_if_has_no_local_version(module: str, wiki: str): + page = "Module:" + module + if check_for_local_version(module, wiki): + protect_non_existing_page(page, wiki) + + +def main(): + lua_files: Iterable[pathlib.Path] + if len(sys.argv[1:]) > 0: + lua_files = [pathlib.Path(arg) for arg in sys.argv[1:]] + elif WIKI_TO_PROTECT: + lua_files = pathlib.Path("./").glob("lua/wikis/**/*.lua") + else: + print("Nothing to protect") + exit(0) + + for file_to_protect in lua_files: + print(f"::group::Checking { str(file_to_protect) }") + wiki = file_to_protect.parts[2] + module = "/".join(file_to_protect.parts[3:])[:-4] + page = "Module:" + module + if WIKI_TO_PROTECT: + if wiki == WIKI_TO_PROTECT: + protect_existing_page(page, wiki) + elif wiki == "commons": + protect_if_has_no_local_version(module, WIKI_TO_PROTECT) + elif wiki != "commons": + protect_existing_page(page, wiki) + else: # commons case + protect_existing_page(page, wiki) + for deploy_wiki in get_wikis(): + protect_if_has_no_local_version(module, deploy_wiki) + print("::endgroup::") + handle_protect_errors() + + +main() From 9387e6e4ab89e0c3856892ab7ad69f1171bcd54e Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Wed, 4 Feb 2026 21:15:30 +0900 Subject: [PATCH 11/54] remove dev script to python --- scripts/remove_dev.py | 76 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 scripts/remove_dev.py diff --git a/scripts/remove_dev.py b/scripts/remove_dev.py new file mode 100644 index 00000000000..dfbde67b7e2 --- /dev/null +++ b/scripts/remove_dev.py @@ -0,0 +1,76 @@ +import os +import time + +import requests + +from deploy_util import * +from login_and_get_token import * + + +LUA_DEV_ENV_NAME = os.getenv("LUA_DEV_ENV_NAME") + +remove_errors: list[str] = list() + + +def remove_page(session: requests.Session, page: str, wiki: str): + print(f"deleting {wiki}:{page}") + token = get_token(wiki) + + result = session.post( + get_wiki_api_url(wiki), + headers=HEADER, + params={ + "format": "json", + "action": "delete", + }, + data={ + "title": page, + "reason": f"Remove {LUA_DEV_ENV_NAME}", + "token": token, + }, + ).text + time.sleep(8) + + if '"delete"' not in result: + print(f"::warning::could not delete {page} on {wiki}") + write_to_github_summary_file(f":warning: could not delete {page} on {wiki}") + remove_errors.append(f"{wiki}:{page}") + + +def search_and_remove(wiki: str): + with requests.Session() as session: + session.cookies = read_cookie_jar(wiki) + search_result = session.post( + get_wiki_api_url(wiki), + headers=HEADER, + params={"format": "json", "action": "query"}, + data={ + "list": "search", + "srsearch": f"intitle:{LUA_DEV_ENV_NAME}", + "srnamespace": 828, + "srlimit": 5000, + "srprop": "", + }, + ).json() + time.sleep(4) + pages = search_result["query"]["search"] + + if len(pages) == 0: + return + + for page in pages: + if os.getenv("INCLUDE_SUB_ENVS") == "true" or page["title"].endswith( + LUA_DEV_ENV_NAME + ): + remove_page(session, page["title"], wiki) + + +def main(): + for wiki in get_wikis(): + if wiki == "commons" and os.getenv("INCLUDE_COMMONS") == "true": + continue + search_and_remove(wiki) + + +if __name__ == "__main__": + main() From a393dc8ddcf1f1591b0a1581b1323c52ec50602a Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Wed, 4 Feb 2026 21:16:47 +0900 Subject: [PATCH 12/54] adjust main call --- scripts/deploy.py | 3 ++- scripts/deploy_res.py | 3 ++- scripts/protect.py | 3 ++- scripts/protect_templates.py | 26 ++++++++++++++++---------- 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/scripts/deploy.py b/scripts/deploy.py index 930063efef1..c523b9230a2 100644 --- a/scripts/deploy.py +++ b/scripts/deploy.py @@ -89,4 +89,5 @@ def main(): sys.exit(1) -main() +if __name__ == "__main__": + main() diff --git a/scripts/deploy_res.py b/scripts/deploy_res.py index 44ce28f5b08..6b164544cf4 100644 --- a/scripts/deploy_res.py +++ b/scripts/deploy_res.py @@ -116,4 +116,5 @@ def main(): update_cache() -main() +if __name__ == "__main__": + main() diff --git a/scripts/protect.py b/scripts/protect.py index 3e905ada991..a13d947cf9e 100644 --- a/scripts/protect.py +++ b/scripts/protect.py @@ -52,4 +52,5 @@ def main(): handle_protect_errors() -main() +if __name__ == "__main__": + main() diff --git a/scripts/protect_templates.py b/scripts/protect_templates.py index bb0f6a756bc..f214cac8485 100644 --- a/scripts/protect_templates.py +++ b/scripts/protect_templates.py @@ -4,13 +4,19 @@ WIKI_TO_PROTECT = os.getenv("WIKI_TO_PROTECT") -with open("./templates/templatesToProtect", "r") as templates_to_protect: - for template_name in templates_to_protect.readlines(): - template = "Template:" + template_name - print(f"::group::Checking {WIKI_TO_PROTECT}:{template}") - if WIKI_TO_PROTECT == "commons": - protect_existing_page(template, WIKI_TO_PROTECT) - else: - protect_non_existing_page(template, WIKI_TO_PROTECT) - print("::endgroup::") -handle_protect_errors() + +def main(): + with open("./templates/templatesToProtect", "r") as templates_to_protect: + for template_name in templates_to_protect.readlines(): + template = "Template:" + template_name + print(f"::group::Checking {WIKI_TO_PROTECT}:{template}") + if WIKI_TO_PROTECT == "commons": + protect_existing_page(template, WIKI_TO_PROTECT) + else: + protect_non_existing_page(template, WIKI_TO_PROTECT) + print("::endgroup::") + handle_protect_errors() + + +if __name__ == "__main__": + main() From 363c04739b073df4d7d7170134a716a41d03e7f8 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Wed, 4 Feb 2026 21:17:46 +0900 Subject: [PATCH 13/54] update gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 07f9d297949..4b5b6746381 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,6 @@ /package-lock.json /luacov.report.out /luacov.stats.out +**/__pycache__/ +.env +*.ck From 0bbe26c5a9a9f27802566e5191d06a48ded87c1a Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Wed, 4 Feb 2026 21:27:13 +0900 Subject: [PATCH 14/54] update workflows --- .github/workflows/deploy test.yml | 13 ++++++++++++- .github/workflows/deploy.yml | 17 ++++++++++++++--- .github/workflows/protect new wiki modules.yml | 11 ++++++++++- .../workflows/protect new wiki templates.yml | 11 ++++++++++- .github/workflows/scheduled.yml | 13 +++++++++++-- 5 files changed, 57 insertions(+), 8 deletions(-) diff --git a/.github/workflows/deploy test.yml b/.github/workflows/deploy test.yml index 2ae268f6480..8fc20fe977a 100644 --- a/.github/workflows/deploy test.yml +++ b/.github/workflows/deploy test.yml @@ -27,6 +27,17 @@ jobs: files: | lua/wikis/**/*.lua + - name: Setup Python + if: steps.changed-files.outputs.any_changed == 'true' + uses: actions/setup-python@v6 + with: + python-version: '3.14' + cache: 'pip' + + - name: Install Python Dependency + if: steps.changed-files.outputs.any_changed == 'true' + run: pip install requests + - name: Personal Lua Deploy if: steps.changed-files.outputs.any_changed == 'true' env: @@ -35,4 +46,4 @@ jobs: WIKI_UA_EMAIL: ${{ secrets.LP_UA_EMAIL }} WIKI_BASE_URL: ${{ secrets.LP_BASE_URL }} LUA_DEV_ENV_NAME: "/dev/${{ github.event.inputs.luadevenv }}" - run: bash scripts/deploy.sh "${{ steps.changed-files.outputs.all_changed_files }}" + run: python3 scripts/deploy.py ${{ steps.changed-files.outputs.all_changed_files }} diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index da02bb258de..329f9bf373e 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -29,6 +29,17 @@ jobs: stylesheets/**/*.scss javascript/**/*.js + - name: Setup Python + if: steps.res-changed-files.outputs.any_changed == 'true' || steps.lua-changed-files.outputs.any_changed == 'true' + uses: actions/setup-python@v6 + with: + python-version: '3.14' + cache: 'pip' + + - name: Install Python Dependency + if: steps.res-changed-files.outputs.any_changed == 'true' || steps.lua-changed-files.outputs.any_changed == 'true' + run: pip install requests + - name: Resource Deploy if: steps.res-changed-files.outputs.any_changed == 'true' env: @@ -37,7 +48,7 @@ jobs: WIKI_UA_EMAIL: ${{ secrets.LP_UA_EMAIL }} WIKI_BASE_URL: ${{ secrets.LP_BASE_URL }} DEPLOY_TRIGGER: ${{ github.event_name }} - run: bash scripts/deploy_res.sh "${{ steps.res-changed-files.outputs.all_changed_files }}" + run: python3 scripts/deploy_res.py ${{ steps.res-changed-files.outputs.all_changed_files }} - name: Lua Deploy if: steps.lua-changed-files.outputs.any_changed == 'true' @@ -47,7 +58,7 @@ jobs: WIKI_UA_EMAIL: ${{ secrets.LP_UA_EMAIL }} WIKI_BASE_URL: ${{ secrets.LP_BASE_URL }} DEPLOY_TRIGGER: ${{ github.event_name }} - run: bash scripts/deploy.sh "${{ steps.lua-changed-files.outputs.all_changed_files }}" + run: python3 scripts/deploy.py ${{ steps.lua-changed-files.outputs.all_changed_files }} - name: Lua Protect if: steps.lua-changed-files.outputs.added_files_count != 0 || steps.lua-changed-files.outputs.renamed_files_count != 0 @@ -56,4 +67,4 @@ jobs: WIKI_PASSWORD: ${{ secrets.LP_BOTPASSWORD }} WIKI_UA_EMAIL: ${{ secrets.LP_UA_EMAIL }} WIKI_BASE_URL: ${{ secrets.LP_BASE_URL }} - run: bash scripts/protect.sh "${{ steps.lua-changed-files.outputs.added_files }} ${{ steps.lua-changed-files.outputs.renamed_files }}" + run: python3 scripts/protect.py ${{ steps.lua-changed-files.outputs.added_files }} ${{ steps.lua-changed-files.outputs.renamed_files }} diff --git a/.github/workflows/protect new wiki modules.yml b/.github/workflows/protect new wiki modules.yml index 369ff76f973..8a06a9600c2 100644 --- a/.github/workflows/protect new wiki modules.yml +++ b/.github/workflows/protect new wiki modules.yml @@ -16,6 +16,15 @@ jobs: steps: - uses: actions/checkout@v6 + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: '3.14' + cache: 'pip' + + - name: Install Python Dependency + run: pip install requests + - name: Lua Protect env: WIKI_USER: ${{ secrets.LP_BOTUSER }} @@ -23,4 +32,4 @@ jobs: WIKI_UA_EMAIL: ${{ secrets.LP_UA_EMAIL }} WIKI_BASE_URL: ${{ secrets.LP_BASE_URL }} WIKI_TO_PROTECT: ${{ github.event.inputs.wiki }} - run: bash scripts/protect.sh + run: python3 scripts/protect.py diff --git a/.github/workflows/protect new wiki templates.yml b/.github/workflows/protect new wiki templates.yml index ae18459b5c0..12ed145324c 100644 --- a/.github/workflows/protect new wiki templates.yml +++ b/.github/workflows/protect new wiki templates.yml @@ -16,6 +16,15 @@ jobs: steps: - uses: actions/checkout@v6 + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: '3.14' + cache: 'pip' + + - name: Install Python Dependency + run: pip install requests + - name: Template Protect env: WIKI_USER: ${{ secrets.LP_BOTUSER }} @@ -23,4 +32,4 @@ jobs: WIKI_UA_EMAIL: ${{ secrets.LP_UA_EMAIL }} WIKI_BASE_URL: ${{ secrets.LP_BASE_URL }} WIKI_TO_PROTECT: ${{ github.event.inputs.wiki }} - run: bash scripts/protect_templates.sh + run: python3 scripts/protect_templates.py diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index f90f3c9a3f2..6c5c7c9e887 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -15,6 +15,15 @@ jobs: with: fetch-depth: 2 + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: '3.14' + cache: 'pip' + + - name: Install Python Dependency + run: pip install requests + - name: Resource Deploy env: WIKI_USER: ${{ secrets.LP_BOTUSER }} @@ -22,7 +31,7 @@ jobs: WIKI_UA_EMAIL: ${{ secrets.LP_UA_EMAIL }} WIKI_BASE_URL: ${{ secrets.LP_BASE_URL }} DEPLOY_TRIGGER: ${{ github.event_name }} - run: bash scripts/deploy_res.sh + run: python3 scripts/deploy_res.py - name: Lua Deploy env: @@ -31,4 +40,4 @@ jobs: WIKI_UA_EMAIL: ${{ secrets.LP_UA_EMAIL }} WIKI_BASE_URL: ${{ secrets.LP_BASE_URL }} DEPLOY_TRIGGER: ${{ github.event_name }} - run: bash scripts/deploy.sh + run: python3 scripts/deploy.py From ada387ea60da69240bb047a5a623b05ae161d20c Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Wed, 4 Feb 2026 21:28:56 +0900 Subject: [PATCH 15/54] throw away shellscript versions --- scripts/deploy.sh | 92 -------------------------- scripts/deploy_res.sh | 102 ----------------------------- scripts/login_and_get_token.sh | 68 ------------------- scripts/protect.sh | 116 --------------------------------- scripts/protect_page.sh | 115 -------------------------------- scripts/protect_templates.sh | 29 --------- scripts/remove_dev.sh | 112 ------------------------------- 7 files changed, 634 deletions(-) delete mode 100644 scripts/deploy.sh delete mode 100644 scripts/deploy_res.sh delete mode 100644 scripts/login_and_get_token.sh delete mode 100644 scripts/protect.sh delete mode 100644 scripts/protect_page.sh delete mode 100644 scripts/protect_templates.sh delete mode 100644 scripts/remove_dev.sh diff --git a/scripts/deploy.sh b/scripts/deploy.sh deleted file mode 100644 index c1ec013d29c..00000000000 --- a/scripts/deploy.sh +++ /dev/null @@ -1,92 +0,0 @@ -#!/bin/bash - -userAgent="GitHub Autodeploy Bot/1.1.0 (${WIKI_UA_EMAIL})" -pat='\-\-\-\ -\-\- @Liquipedia\ -\-\- page=([^ -]*)\ -' - -. ./scripts/login_and_get_token.sh - -if [[ -n "$1" ]]; then - luaFiles=$1 - gitDeployReason="\"$(git log -1 --pretty='%h %s')\"" -else - luaFiles=$(find lua/wikis -type f -name '*.lua') - gitDeployReason='Automated Weekly Re-Sync' -fi - -allModulesDeployed=true -for luaFile in $luaFiles; do - if [[ -n "$1" ]]; then - luaFile="./$luaFile" - fi - echo "::group::Checking $luaFile" - fileContents=$(cat "$luaFile") - - [[ $fileContents =~ $pat ]] - - if [[ "${BASH_REMATCH[1]}" == "" ]]; then - echo '...skipping - no magic comment found' - echo "${luaFile} skipped" >> $GITHUB_STEP_SUMMARY - else - # Extract wiki name from path, e.g. ./lua/wikis/commons/Widget/Match/Page/Header.lua -> commons - wiki=$(echo "$luaFile" | awk -F'/' '{for(i=1;i<=NF;i++) if($i=="wikis") print $(i+1)}') - page="${BASH_REMATCH[1]}${LUA_DEV_ENV_NAME}" - - echo '...magic comment found - updating wiki...' - - echo "...wiki = $wiki" - echo "...page = $page" - wikiApiUrl="${WIKI_BASE_URL}/${wiki}/api.php" - ckf="cookie_${wiki}.ck" - - getToken ${wiki} - - # Edit page - rawResult=$( - curl \ - -s \ - -b "$ckf" \ - -c "$ckf" \ - --data-urlencode "title=${page}" \ - --data-urlencode "text=${fileContents}" \ - --data-urlencode "summary=Git: ${gitDeployReason}" \ - --data-urlencode "bot=true" \ - --data-urlencode "recreate=true" \ - --data-urlencode "token=${token}" \ - -H "User-Agent: ${userAgent}" \ - -H 'Accept-Encoding: gzip' \ - -X POST "${wikiApiUrl}?format=json&action=edit" \ - | gunzip - ) - result=$(echo "$rawResult" | jq ".edit.result" -r) - if [[ "${result}" == "Success" ]]; then - nochange=$(echo "$rawResult" | jq ".edit.nochange" -r) - echo "...${result}" - if [[ "${nochange}" == "" ]] && [[ "${DEPLOY_TRIGGER}" == "push" ]]; then - echo "::notice file=${luaFile}::No change made" - elif [[ "${nochange}" != "" ]] && [[ "${DEPLOY_TRIGGER}" != "push" ]]; then - echo "::warning file=${luaFile}::File changed" - fi - echo '...done' - echo ":information_source: ${luaFile} successfully deployed" >> $GITHUB_STEP_SUMMARY - else - echo "::warning file=${luaFile}::failed to deploy" - echo ":warning: ${luaFile} failed to deploy" >> $GITHUB_STEP_SUMMARY - allModulesDeployed=false - fi - - # Don't get rate limited - sleep 4 - fi - echo '::endgroup::' - - if [ "$allModulesDeployed" != true ]; then - echo "::warning::Some modules were not deployed!" - exit 1 - fi -done - -rm -f cookie_* diff --git a/scripts/deploy_res.sh b/scripts/deploy_res.sh deleted file mode 100644 index 8cfcb9a533e..00000000000 --- a/scripts/deploy_res.sh +++ /dev/null @@ -1,102 +0,0 @@ -#!/bin/bash - -userAgent="GitHub Autodeploy Bot/1.1.0 (${WIKI_UA_EMAIL})" - -. ./scripts/login_and_get_token.sh - -if [[ -n "$1" ]]; then - files=$1 - gitDeployReason="\"$(git log -1 --pretty='%h %s')\"" -else - files=$(find stylesheets javascript -type f -name '*.scss' -o -name '*.js') - gitDeployReason='Automated Weekly Re-Sync' -fi - -wikiApiUrl="${WIKI_BASE_URL}/commons/api.php" -ckf="cookie_commons.ck" - -allDeployed=true -changesMade=false -for file in $files; do - if [[ -n "$1" ]]; then - file="./$file" - fi - echo "::group::Checking $file" - fileContents=$(cat "$file") - fileName=$(basename "$file") - - if [[ $file == *.js ]]; then - page="MediaWiki:Common.js/${fileName}" - else - page="MediaWiki:Common.css/${fileName}" - fi - - echo "...page = $page" - - # Edit page - getToken "commons" - rawResult=$( - curl \ - -s \ - -b "$ckf" \ - -c "$ckf" \ - --data-urlencode "title=${page}" \ - --data-urlencode "text=${fileContents}" \ - --data-urlencode "summary=Git: ${gitDeployReason}" \ - --data-urlencode "bot=true" \ - --data-urlencode "recreate=true" \ - --data-urlencode "token=${token}" \ - -H "User-Agent: ${userAgent}" \ - -H 'Accept-Encoding: gzip' \ - -X POST "${wikiApiUrl}?format=json&action=edit" \ - | gunzip - ) - result=$(echo "$rawResult" | jq ".edit.result" -r) - newRevId=$(echo "$rawResult" | jq ".edit.newrevid" -r) - if [[ "${result}" == "Success" ]]; then - if [[ "${newRevId}" != "null" ]]; then - changesMade=true - if [[ "${DEPLOY_TRIGGER}" != "push" ]]; then - echo "::warning file=${file}::File changed" - fi - fi - echo "...${result}" - echo '...done' - echo ":information_source: ${file} successfully deployed" >> $GITHUB_STEP_SUMMARY - else - echo "::warning file=${file}::failed to deploy" - echo ":warning: ${file} failed to deploy" >> $GITHUB_STEP_SUMMARY - allDeployed=false - fi - echo '::endgroup::' - - # Don't get rate limited - sleep 4 -done - -if [ "$allDeployed" != true ]; then - echo "::error::Some files were not deployed; resource cache version not updated!" - exit 1 -elif [ "$changesMade" == true ]; then - cacheResult=$( - curl \ - -s \ - -b "$ckf" \ - -c "$ckf" \ - --data-urlencode "messagename=Resourceloaderarticles-cacheversion" \ - --data-urlencode "value=$(git log -1 --pretty='%h')" \ - -H "User-Agent: ${userAgent}" \ - -H 'Accept-Encoding: gzip' \ - -X POST "${wikiApiUrl}?format=json&action=updatelpmwmessageapi" \ - | gunzip \ - | jq ".updatelpmwmessageapi.message" -r - ) - if [[ "${cacheResult}" == "Successfully changed the message value" ]]; then - echo "Resource cache version updated succesfully!" - else - echo "::error::Resource cache version unable to be updated!" - exit 1 - fi -fi - -rm -f cookie_* diff --git a/scripts/login_and_get_token.sh b/scripts/login_and_get_token.sh deleted file mode 100644 index 7394ff57ea9..00000000000 --- a/scripts/login_and_get_token.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/bin/bash - -userAgent="GitHub Autodeploy Bot/1.1.0 (${WIKI_UA_EMAIL})" - -declare -A loggedin - -# logs into a specified wiki -# $1 -> the wiki to log into -login() { - wiki=$1 - - if [[ ${loggedin[${wiki}]} != 1 ]]; then - ckf="cookie_${wiki}.ck" - wikiApiUrl="${WIKI_BASE_URL}/${wiki}/api.php" - echo "...logging in on \"${wiki}\"" - loginToken=$( - curl \ - -s \ - -b "$ckf" \ - -c "$ckf" \ - -d "format=json&action=query&meta=tokens&type=login" \ - -H "User-Agent: ${userAgent}" \ - -H 'Accept-Encoding: gzip' \ - -X POST "$wikiApiUrl" \ - | gunzip \ - | jq ".query.tokens.logintoken" -r - ) - curl \ - -s \ - -b "$ckf" \ - -c "$ckf" \ - --data-urlencode "lgname=${WIKI_USER}" \ - --data-urlencode "lgpassword=${WIKI_PASSWORD}" \ - --data-urlencode "lgtoken=${loginToken}" \ - -H "User-Agent: ${userAgent}" \ - -H 'Accept-Encoding: gzip' \ - -X POST "${wikiApiUrl}?format=json&action=login" \ - | gunzip \ - > /dev/null - loggedin[$wiki]=1 - # Don't get rate limited - sleep 4 - fi -} - -# gets an edit/protect token for a specified wiki -# if not already logged in on the wiki also logs in on the wiki -# $1 -> the wiki to get the token for -getToken(){ - wiki=$1 - wikiApiUrl="${WIKI_BASE_URL}/${wiki}/api.php" - ckf="cookie_${wiki}.ck" - - login $wiki - - token=$( - curl \ - -s \ - -b "$ckf" \ - -c "$ckf" \ - -d "format=json&action=query&meta=tokens" \ - -H "User-Agent: ${userAgent}" \ - -H 'Accept-Encoding: gzip' \ - -X POST "$wikiApiUrl" \ - | gunzip \ - | jq ".query.tokens.csrftoken" -r - ) -} diff --git a/scripts/protect.sh b/scripts/protect.sh deleted file mode 100644 index b7d68f53335..00000000000 --- a/scripts/protect.sh +++ /dev/null @@ -1,116 +0,0 @@ -#!/bin/bash - -userAgent="GitHub Autodeploy Bot/1.1.0 (${WIKI_UA_EMAIL})" - -. ./scripts/protect_page.sh - -declare -a allWikis -declare -a regexErrors=() - -regex="^\.?/?lua/wikis/([a-z0-9]+)/(.*)\.lua$" - -if [[ -n "$1" ]] then - filesToProtect=$1 -elif [[ -n ${WIKI_TO_PROTECT} ]]; then - filesToProtect=$(find lua/wikis -type f -name '*.lua') -else - echo "Nothing to protect" - exit 0 -fi - -allLuaFiles=$(find lua -type f -name '*.lua') - -allWikis=$( - curl \ - -s \ - -b "$ckf" \ - -c "$ckf" \ - -H "User-Agent: ${userAgent}" \ - -H 'Accept-Encoding: gzip' \ - -X GET "https://liquipedia.net/api.php?action=listwikis" \ - | gunzip \ - | jq '.allwikis | keys[]' -r -) -# Don't get rate limited -sleep 4 - -# checks if for a specified (commons) module there exists a local version of said module on the specified wiki in the git repo -# $1 -> module (without namespace prefix) -# $2 -> wiki -checkForLocalVersion() { - if [[ $2 == "commons" ]]; then - hasNoLocalVersion=false - elif [[ $allLuaFiles == *"lua/wikis/${2}/${1}.lua"* ]] || [[ $filesToProtect == *"lua/wikis/${2}/${1}.lua"* ]]; then - hasNoLocalVersion=false - else - hasNoLocalVersion=true - fi -} - -# protects a module against creation if it has no local version according to git -# $1 -> module (without namespace prefix) -# $2 -> wiki -protectIfHasNoLocalVersion() { - module="${1}" - page="Module:${module}" - wiki="${2}" - checkForLocalVersion $module $wiki - if $hasNoLocalVersion; then - protectNonExistingPage $page $wiki - fi -} - -for fileToProtect in $filesToProtect; do - echo "::group::Checking $fileToProtect" - if [[ $fileToProtect =~ $regex ]]; then - wiki=${BASH_REMATCH[1]} - module=${BASH_REMATCH[2]} - page="Module:${module}" - - if [[ -n ${WIKI_TO_PROTECT} ]]; then - if [[ $wiki == ${WIKI_TO_PROTECT} ]]; then - protectExistingPage $page ${WIKI_TO_PROTECT} - elif [[ $wiki == "commons" ]]; then - protectIfHasNoLocalVersion $module ${WIKI_TO_PROTECT} - fi - elif [[ "commons" != $wiki ]]; then - protectExistingPage $page $wiki - else # commons case - protectExistingPage $page $wiki - - for deployWiki in $allWikis; do - protectIfHasNoLocalVersion $module $deployWiki - done - fi - else - echo '::warning::skipping - regex failed' - regexErrors+=($fileToProtect) - fi - echo '::endgroup::' -done - -if [[ ${#regexErrors[@]} -ne 0 ]]; then - echo "::warning::Some regexes failed" - echo ":warning: Some regexes failed" >> $GITHUB_STEP_SUMMARY - echo "::group::Files the regex failed on" - for value in "${regexErrors[@]}"; do - echo "... ${failedRegex}" - done - echo "::endgroup::" -fi - -if [[ ${#protectErrors[@]} -ne 0 ]]; then - echo "::warning::Some modules could not be protected" - echo ":warning: Some modules could not be protected" >> $GITHUB_STEP_SUMMARY - echo "::group::Failed protections" - for value in "${protectErrors[@]}"; do - echo "... ${value}" - done - echo "::endgroup::" -fi - -rm -f cookie_* - -if [[ ${#protectErrors[@]} -ne 0 ]] || [[ ${#regexErrors[@]} -ne 0 ]]; then - exit 1 -fi diff --git a/scripts/protect_page.sh b/scripts/protect_page.sh deleted file mode 100644 index 3b78f6c6aeb..00000000000 --- a/scripts/protect_page.sh +++ /dev/null @@ -1,115 +0,0 @@ -#!/bin/bash - -userAgent="GitHub Autodeploy Bot/1.1.0 (${WIKI_UA_EMAIL})" - -. ./scripts/login_and_get_token.sh - -declare -a protectErrors=() - -# protects a specified page on a specified wiki with the specified protect mode -# $1 -> page (inkl namespace prefix) -# $2 -> wiki -# $3 -> protect mode ('edit' || 'create') -protectPage() { - page="${1}" - wiki=$2 - protectMode=$3 - - if [[ ${protectMode} == 'edit' ]]; then - protectOptions="edit=allow-only-sysop|move=allow-only-sysop" - elif [[ ${protectMode} == 'create' ]]; then - protectOptions="create=allow-only-sysop" - else - echo "::warning:: invalid protect mode: ${protectMode}" - exit 1 - fi - - echo "...wiki = $wiki" - echo "...page = ${page}" - wikiApiUrl="${WIKI_BASE_URL}/${wiki}/api.php" - ckf="cookie_${wiki}.ck" - - getToken $wiki - - rawProtectResult=$( - curl \ - -s \ - -b "$ckf" \ - -c "$ckf" \ - --data-urlencode "title=${page}" \ - --data-urlencode "protections=${protectOptions}" \ - --data-urlencode "reason=Git maintained" \ - --data-urlencode "expiry=infinite" \ - --data-urlencode "bot=true" \ - --data-urlencode "token=${token}" \ - -H "User-Agent: ${userAgent}" \ - -H 'Accept-Encoding: gzip' \ - -X POST "${wikiApiUrl}?format=json&action=protect" \ - | gunzip - ) - # Don't get rate limited - sleep 4 - - result=$(echo "$rawProtectResult" | jq ".protect.protections.[].${protectMode}" -r) - if [[ $result != *"allow-only-sysop"* ]]; then - echo "::warning::could not (${protectMode}) protect ${page} on ${wiki}" - protectErrorMsg="${protectMode}:${wiki}:${page}" - protectErrors+=("${protectErrorMsg}") - fi -} - -# checks if a specified page on a specified wiki exists -# $1 -> page (inkl namespace prefix) -# $2 -> wiki -checkIfPageExists() { - page="${1}" - wiki="${2}" - wikiApiUrl="${WIKI_BASE_URL}/${wiki}/api.php" - ckf="cookie_${wiki}.ck" - - rawResult=$( - curl \ - -s \ - -b "$ckf" \ - -c "$ckf" \ - --data-urlencode "titles=${page}" \ - --data-urlencode "prop=info" \ - -H "User-Agent: ${userAgent}" \ - -H 'Accept-Encoding: gzip' \ - -X POST "${wikiApiUrl}?format=json&action=query" \ - | gunzip - ) - - # Don't get rate limited - sleep 4 - - if [[ $rawResult == *'missing":"'* ]]; then - pageExists=false - else - pageExists=true - fi -} - -# protects a specified page on a specified wiki against creation -# if the page already exists it will issue a warning -# $1 -> page (inkl namespace prefix) -# $2 -> wiki -protectNonExistingPage() { - page="${1}" - wiki=$2 - - checkIfPageExists "${page}" $wiki - if $pageExists; then - echo "::warning::$page already exists on $wiki" - protectErrors+=("create:${wiki}:${page}") - else - protectPage "${page}" "${wiki}" "create" - fi -} - -# protects a specified page on a specified wiki against editing/moving -# $1 -> page (inkl namespace prefix) -# $2 -> wiki -protectExistingPage() { - protectPage "${1}" "${2}" "edit" -} diff --git a/scripts/protect_templates.sh b/scripts/protect_templates.sh deleted file mode 100644 index 181203e73ab..00000000000 --- a/scripts/protect_templates.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -. ./scripts/protect_page.sh - -readarray filesToProtect < "./templates/templatesToProtect" - -for fileToProtect in "${filesToProtect[@]}"; do - template="Template:${fileToProtect}" - echo "::group::Checking ${WIKI_TO_PROTECT}:${template}" - if [[ "commons" == ${WIKI_TO_PROTECT} ]]; then - protectExistingPage "${template}" ${WIKI_TO_PROTECT} - else - protectNonExistingPage "${template}" ${WIKI_TO_PROTECT} - fi - echo '::endgroup::' -done - -rm -f cookie_* - -if [[ ${#protectErrors[@]} -ne 0 ]]; then - echo "::warning::Some templates could not be protected" - echo ":warning: Some templates could not be protected" >> $GITHUB_STEP_SUMMARY - echo "::group::Failed protections" - for value in "${protectErrors[@]}"; do - echo "... ${value}" - done - echo "::endgroup::" - exit 1 -fi diff --git a/scripts/remove_dev.sh b/scripts/remove_dev.sh deleted file mode 100644 index f1f7536b493..00000000000 --- a/scripts/remove_dev.sh +++ /dev/null @@ -1,112 +0,0 @@ -#!/bin/bash - -userAgent="GitHub Autodeploy Bot/1.1.0 (${WIKI_UA_EMAIL})" - -. ./scripts/login_and_get_token.sh - -ckf="cookie_base.ck" -declare -a removeErrors -declare -a allWikis -allWikis=$( - curl \ - -s \ - -b "$ckf" \ - -c "$ckf" \ - -H "User-Agent: ${userAgent}" \ - -H 'Accept-Encoding: gzip' \ - -X GET "https://liquipedia.net/api.php?action=listwikis" \ - | gunzip \ - | jq '.allwikis | keys[]' -r -) -# Don't get rate limited -sleep 4 - -removePage() { - page="${1}" - wiki=$2 - - echo "deleting ${wiki}:${page}" - - wikiApiUrl="${WIKI_BASE_URL}/${wiki}/api.php" - ckf="cookie_${wiki}.ck" - - getToken $wiki - - rawRemoveResult=$( - curl \ - -s \ - -b "$ckf" \ - -c "$ckf" \ - --data-urlencode "title=${page}" \ - --data-urlencode "reason=Remove ${LUA_DEV_ENV_NAME}" \ - --data-urlencode "token=${token}" \ - -H "User-Agent: ${userAgent}" \ - -H 'Accept-Encoding: gzip' \ - -X POST "${wikiApiUrl}?format=json&action=delete" \ - | gunzip - ) - # Don't get rate limited - sleep 8 - - if [[ $rawRemoveResult != *"delete"* ]]; then - echo "::warning::could not delete ${page} on ${wiki}" - echo "::warning::could not delete ${page} on ${wiki}" >> $GITHUB_STEP_SUMMARY - removeErrorMsg="${wiki}:${page}" - removeErrors+=("${removeErrorMsg}") - fi -} - -searchAndRemove(){ - wiki=$1 - - wikiApiUrl="${WIKI_BASE_URL}/${wiki}/api.php" - - rawSearchResult=$( - curl \ - -s \ - -b "$ckf" \ - -c "$ckf" \ - --data-urlencode "list=search" \ - --data-urlencode "srsearch=intitle:${LUA_DEV_ENV_NAME}" \ - --data-urlencode "srnamespace=828" \ - --data-urlencode "srlimit=5000" \ - --data-urlencode "srprop=" \ - -H "User-Agent: ${userAgent}" \ - -H 'Accept-Encoding: gzip' \ - -X POST "${wikiApiUrl}?format=json&action=query" \ - | gunzip - ) - - sleep 4 - - pages=($(echo "$rawSearchResult" | jq ".query.search[] | .title" -r -c)) - - if [[ -n $pages && ${#pages[@]} -ne 0 ]]; then - for page in ${pages[@]}; do - if [[ ${INCLUDE_SUB_ENVS} == true || "${page}" == *"${LUA_DEV_ENV_NAME}" ]]; then - removePage $page $wiki - fi - done - fi -} - -for wiki in $allWikis; do - if [[ $wiki != "commons" || ${INCLUDE_COMMONS} == true ]]; then - echo "::group::Checking $wiki" - searchAndRemove $wiki - fi -done - -rm -f cookie_* - -if [[ ${#removeErrors[@]} -ne 0 ]]; then - echo "::warning::Could not delete some pages on some wikis" - echo "::warning::Could not delete some pages on some wikis" >> $GITHUB_STEP_SUMMARY - echo "::group::Failed protections" - for value in "${removeErrors[@]}"; do - echo "... ${value}" - done - echo "::endgroup::" - - exit 1 -fi From eb0147418f7119538d9816ecc7c4d8fd53ec1863 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Wed, 4 Feb 2026 21:41:47 +0900 Subject: [PATCH 16/54] disable python cache in GHA --- .github/workflows/deploy test.yml | 2 +- .github/workflows/deploy.yml | 2 +- .github/workflows/protect new wiki modules.yml | 2 +- .github/workflows/protect new wiki templates.yml | 2 +- .github/workflows/scheduled.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy test.yml b/.github/workflows/deploy test.yml index 8fc20fe977a..ae491996e9d 100644 --- a/.github/workflows/deploy test.yml +++ b/.github/workflows/deploy test.yml @@ -32,7 +32,7 @@ jobs: uses: actions/setup-python@v6 with: python-version: '3.14' - cache: 'pip' + - name: Install Python Dependency if: steps.changed-files.outputs.any_changed == 'true' diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 329f9bf373e..404ba2d4bb9 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -34,7 +34,7 @@ jobs: uses: actions/setup-python@v6 with: python-version: '3.14' - cache: 'pip' + - name: Install Python Dependency if: steps.res-changed-files.outputs.any_changed == 'true' || steps.lua-changed-files.outputs.any_changed == 'true' diff --git a/.github/workflows/protect new wiki modules.yml b/.github/workflows/protect new wiki modules.yml index 8a06a9600c2..c751f726f46 100644 --- a/.github/workflows/protect new wiki modules.yml +++ b/.github/workflows/protect new wiki modules.yml @@ -20,7 +20,7 @@ jobs: uses: actions/setup-python@v6 with: python-version: '3.14' - cache: 'pip' + - name: Install Python Dependency run: pip install requests diff --git a/.github/workflows/protect new wiki templates.yml b/.github/workflows/protect new wiki templates.yml index 12ed145324c..81a694f7b5c 100644 --- a/.github/workflows/protect new wiki templates.yml +++ b/.github/workflows/protect new wiki templates.yml @@ -20,7 +20,7 @@ jobs: uses: actions/setup-python@v6 with: python-version: '3.14' - cache: 'pip' + - name: Install Python Dependency run: pip install requests diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index 6c5c7c9e887..b2254a75e53 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -19,7 +19,7 @@ jobs: uses: actions/setup-python@v6 with: python-version: '3.14' - cache: 'pip' + - name: Install Python Dependency run: pip install requests From 29771f6e23a8a3d2f80d1fb3d648c9fc2da0af52 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Wed, 4 Feb 2026 21:48:08 +0900 Subject: [PATCH 17/54] update workflows --- .github/workflows/delete dev env.yml | 2 +- .github/workflows/deploy test.yml | 2 +- .github/workflows/deploy.yml | 6 +++--- .github/workflows/protect new wiki modules.yml | 2 +- .github/workflows/protect new wiki templates.yml | 2 +- .github/workflows/scheduled.yml | 4 ++-- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/delete dev env.yml b/.github/workflows/delete dev env.yml index e9c143ab618..de719b09283 100644 --- a/.github/workflows/delete dev env.yml +++ b/.github/workflows/delete dev env.yml @@ -33,4 +33,4 @@ jobs: LUA_DEV_ENV_NAME: "dev/${{ github.event.inputs.luadevenv }}" INCLUDE_COMMONS: ${{ github.event.inputs.includecommons }} INCLUDE_SUB_ENVS: ${{ github.event.inputs.includesubenvs }} - run: bash scripts/remove_dev.sh + run: python3 ./scripts/remove_dev.py diff --git a/.github/workflows/deploy test.yml b/.github/workflows/deploy test.yml index ae491996e9d..4732df45fa2 100644 --- a/.github/workflows/deploy test.yml +++ b/.github/workflows/deploy test.yml @@ -46,4 +46,4 @@ jobs: WIKI_UA_EMAIL: ${{ secrets.LP_UA_EMAIL }} WIKI_BASE_URL: ${{ secrets.LP_BASE_URL }} LUA_DEV_ENV_NAME: "/dev/${{ github.event.inputs.luadevenv }}" - run: python3 scripts/deploy.py ${{ steps.changed-files.outputs.all_changed_files }} + run: python3 ./scripts/deploy.py ${{ steps.changed-files.outputs.all_changed_files }} diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 404ba2d4bb9..85765fa6319 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -48,7 +48,7 @@ jobs: WIKI_UA_EMAIL: ${{ secrets.LP_UA_EMAIL }} WIKI_BASE_URL: ${{ secrets.LP_BASE_URL }} DEPLOY_TRIGGER: ${{ github.event_name }} - run: python3 scripts/deploy_res.py ${{ steps.res-changed-files.outputs.all_changed_files }} + run: python3 ./scripts/deploy_res.py ${{ steps.res-changed-files.outputs.all_changed_files }} - name: Lua Deploy if: steps.lua-changed-files.outputs.any_changed == 'true' @@ -58,7 +58,7 @@ jobs: WIKI_UA_EMAIL: ${{ secrets.LP_UA_EMAIL }} WIKI_BASE_URL: ${{ secrets.LP_BASE_URL }} DEPLOY_TRIGGER: ${{ github.event_name }} - run: python3 scripts/deploy.py ${{ steps.lua-changed-files.outputs.all_changed_files }} + run: python3 ./scripts/deploy.py ${{ steps.lua-changed-files.outputs.all_changed_files }} - name: Lua Protect if: steps.lua-changed-files.outputs.added_files_count != 0 || steps.lua-changed-files.outputs.renamed_files_count != 0 @@ -67,4 +67,4 @@ jobs: WIKI_PASSWORD: ${{ secrets.LP_BOTPASSWORD }} WIKI_UA_EMAIL: ${{ secrets.LP_UA_EMAIL }} WIKI_BASE_URL: ${{ secrets.LP_BASE_URL }} - run: python3 scripts/protect.py ${{ steps.lua-changed-files.outputs.added_files }} ${{ steps.lua-changed-files.outputs.renamed_files }} + run: python3 ./scripts/protect.py ${{ steps.lua-changed-files.outputs.added_files }} ${{ steps.lua-changed-files.outputs.renamed_files }} diff --git a/.github/workflows/protect new wiki modules.yml b/.github/workflows/protect new wiki modules.yml index c751f726f46..8bfc1916ba3 100644 --- a/.github/workflows/protect new wiki modules.yml +++ b/.github/workflows/protect new wiki modules.yml @@ -32,4 +32,4 @@ jobs: WIKI_UA_EMAIL: ${{ secrets.LP_UA_EMAIL }} WIKI_BASE_URL: ${{ secrets.LP_BASE_URL }} WIKI_TO_PROTECT: ${{ github.event.inputs.wiki }} - run: python3 scripts/protect.py + run: python3 ./scripts/protect.py diff --git a/.github/workflows/protect new wiki templates.yml b/.github/workflows/protect new wiki templates.yml index 81a694f7b5c..16f21bb6d41 100644 --- a/.github/workflows/protect new wiki templates.yml +++ b/.github/workflows/protect new wiki templates.yml @@ -32,4 +32,4 @@ jobs: WIKI_UA_EMAIL: ${{ secrets.LP_UA_EMAIL }} WIKI_BASE_URL: ${{ secrets.LP_BASE_URL }} WIKI_TO_PROTECT: ${{ github.event.inputs.wiki }} - run: python3 scripts/protect_templates.py + run: python3 ./scripts/protect_templates.py diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index b2254a75e53..971460b6b14 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -31,7 +31,7 @@ jobs: WIKI_UA_EMAIL: ${{ secrets.LP_UA_EMAIL }} WIKI_BASE_URL: ${{ secrets.LP_BASE_URL }} DEPLOY_TRIGGER: ${{ github.event_name }} - run: python3 scripts/deploy_res.py + run: python3 ./scripts/deploy_res.py - name: Lua Deploy env: @@ -40,4 +40,4 @@ jobs: WIKI_UA_EMAIL: ${{ secrets.LP_UA_EMAIL }} WIKI_BASE_URL: ${{ secrets.LP_BASE_URL }} DEPLOY_TRIGGER: ${{ github.event_name }} - run: python3 scripts/deploy.py + run: python3 ./scripts/deploy.py From ad62cf0a45999e4d28c66a607d558340188d94dd Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Feb 2026 10:20:33 +0900 Subject: [PATCH 18/54] bump deploy bot version --- scripts/deploy_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/deploy_util.py b/scripts/deploy_util.py index 556912830fe..8c9caff03f5 100644 --- a/scripts/deploy_util.py +++ b/scripts/deploy_util.py @@ -20,7 +20,7 @@ DEPLOY_TRIGGER = os.getenv("DEPLOY_TRIGGER") GITHUB_STEP_SUMMARY_FILE = os.getenv("GITHUB_STEP_SUMMARY") -USER_AGENT = f"GitHub Autodeploy Bot/1.1.0 ({ os.getenv("WIKI_UA_EMAIL") })" +USER_AGENT = f"GitHub Autodeploy Bot/2.0.0 ({ os.getenv("WIKI_UA_EMAIL") })" WIKI_BASE_URL = os.getenv("WIKI_BASE_URL") HEADER = { "User-Agent": USER_AGENT, From be22a1745555d9177510151b35cc506a9916af43 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Feb 2026 10:41:54 +0900 Subject: [PATCH 19/54] kick debugging print --- scripts/deploy_res.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/deploy_res.py b/scripts/deploy_res.py index 6b164544cf4..7d9ac50fc84 100644 --- a/scripts/deploy_res.py +++ b/scripts/deploy_res.py @@ -43,7 +43,6 @@ def deploy_resources( "token": token, }, ).json() - print(response) result = response["edit"]["result"] new_rev_id = response["edit"].get("newrevid") if result == "Success": From c9d63e7a304f5944ec658a10534dcfaf6fe5aaa9 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Feb 2026 10:45:24 +0900 Subject: [PATCH 20/54] suppress python stdout buffering --- .github/workflows/delete dev env.yml | 1 + .github/workflows/deploy test.yml | 1 + .github/workflows/deploy.yml | 3 +++ .github/workflows/protect new wiki modules.yml | 1 + .github/workflows/protect new wiki templates.yml | 1 + .github/workflows/scheduled.yml | 2 ++ 6 files changed, 9 insertions(+) diff --git a/.github/workflows/delete dev env.yml b/.github/workflows/delete dev env.yml index de719b09283..c17d5b196f4 100644 --- a/.github/workflows/delete dev env.yml +++ b/.github/workflows/delete dev env.yml @@ -33,4 +33,5 @@ jobs: LUA_DEV_ENV_NAME: "dev/${{ github.event.inputs.luadevenv }}" INCLUDE_COMMONS: ${{ github.event.inputs.includecommons }} INCLUDE_SUB_ENVS: ${{ github.event.inputs.includesubenvs }} + PYTHONUNBUFFERED: 1 run: python3 ./scripts/remove_dev.py diff --git a/.github/workflows/deploy test.yml b/.github/workflows/deploy test.yml index 4732df45fa2..a459a0f85c9 100644 --- a/.github/workflows/deploy test.yml +++ b/.github/workflows/deploy test.yml @@ -46,4 +46,5 @@ jobs: WIKI_UA_EMAIL: ${{ secrets.LP_UA_EMAIL }} WIKI_BASE_URL: ${{ secrets.LP_BASE_URL }} LUA_DEV_ENV_NAME: "/dev/${{ github.event.inputs.luadevenv }}" + PYTHONUNBUFFERED: 1 run: python3 ./scripts/deploy.py ${{ steps.changed-files.outputs.all_changed_files }} diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 85765fa6319..ebcc434649f 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -48,6 +48,7 @@ jobs: WIKI_UA_EMAIL: ${{ secrets.LP_UA_EMAIL }} WIKI_BASE_URL: ${{ secrets.LP_BASE_URL }} DEPLOY_TRIGGER: ${{ github.event_name }} + PYTHONUNBUFFERED: 1 run: python3 ./scripts/deploy_res.py ${{ steps.res-changed-files.outputs.all_changed_files }} - name: Lua Deploy @@ -58,6 +59,7 @@ jobs: WIKI_UA_EMAIL: ${{ secrets.LP_UA_EMAIL }} WIKI_BASE_URL: ${{ secrets.LP_BASE_URL }} DEPLOY_TRIGGER: ${{ github.event_name }} + PYTHONUNBUFFERED: 1 run: python3 ./scripts/deploy.py ${{ steps.lua-changed-files.outputs.all_changed_files }} - name: Lua Protect @@ -67,4 +69,5 @@ jobs: WIKI_PASSWORD: ${{ secrets.LP_BOTPASSWORD }} WIKI_UA_EMAIL: ${{ secrets.LP_UA_EMAIL }} WIKI_BASE_URL: ${{ secrets.LP_BASE_URL }} + PYTHONUNBUFFERED: 1 run: python3 ./scripts/protect.py ${{ steps.lua-changed-files.outputs.added_files }} ${{ steps.lua-changed-files.outputs.renamed_files }} diff --git a/.github/workflows/protect new wiki modules.yml b/.github/workflows/protect new wiki modules.yml index 8bfc1916ba3..cdf0d86f204 100644 --- a/.github/workflows/protect new wiki modules.yml +++ b/.github/workflows/protect new wiki modules.yml @@ -32,4 +32,5 @@ jobs: WIKI_UA_EMAIL: ${{ secrets.LP_UA_EMAIL }} WIKI_BASE_URL: ${{ secrets.LP_BASE_URL }} WIKI_TO_PROTECT: ${{ github.event.inputs.wiki }} + PYTHONUNBUFFERED: 1 run: python3 ./scripts/protect.py diff --git a/.github/workflows/protect new wiki templates.yml b/.github/workflows/protect new wiki templates.yml index 16f21bb6d41..cdaf06c5ffd 100644 --- a/.github/workflows/protect new wiki templates.yml +++ b/.github/workflows/protect new wiki templates.yml @@ -32,4 +32,5 @@ jobs: WIKI_UA_EMAIL: ${{ secrets.LP_UA_EMAIL }} WIKI_BASE_URL: ${{ secrets.LP_BASE_URL }} WIKI_TO_PROTECT: ${{ github.event.inputs.wiki }} + PYTHONUNBUFFERED: 1 run: python3 ./scripts/protect_templates.py diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index 971460b6b14..c81245ecc55 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -31,6 +31,7 @@ jobs: WIKI_UA_EMAIL: ${{ secrets.LP_UA_EMAIL }} WIKI_BASE_URL: ${{ secrets.LP_BASE_URL }} DEPLOY_TRIGGER: ${{ github.event_name }} + PYTHONUNBUFFERED: 1 run: python3 ./scripts/deploy_res.py - name: Lua Deploy @@ -40,4 +41,5 @@ jobs: WIKI_UA_EMAIL: ${{ secrets.LP_UA_EMAIL }} WIKI_BASE_URL: ${{ secrets.LP_BASE_URL }} DEPLOY_TRIGGER: ${{ github.event_name }} + PYTHONUNBUFFERED: 1 run: python3 ./scripts/deploy.py From e08d40d51e7f3a262ed72a964a9694674e3d6f64 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Feb 2026 11:09:03 +0900 Subject: [PATCH 21/54] use rglob --- scripts/deploy.py | 2 +- scripts/deploy_res.py | 4 ++-- scripts/protect.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/deploy.py b/scripts/deploy.py index c523b9230a2..945e881651b 100644 --- a/scripts/deploy.py +++ b/scripts/deploy.py @@ -75,7 +75,7 @@ def main(): lua_files: Iterable[pathlib.Path] git_deploy_reason: str if len(sys.argv[1:]) == 0: - lua_files = pathlib.Path("./").glob("lua/wikis/**/*.lua") + lua_files = pathlib.Path("./lua/wikis/").rglob("*.lua") git_deploy_reason = "Automated Weekly Re-Sync" else: lua_files = [pathlib.Path(arg) for arg in sys.argv[1:]] diff --git a/scripts/deploy_res.py b/scripts/deploy_res.py index 7d9ac50fc84..8ae8695a7df 100644 --- a/scripts/deploy_res.py +++ b/scripts/deploy_res.py @@ -95,8 +95,8 @@ def main(): git_deploy_reason: str if len(sys.argv[1:]) == 0: resource_files = itertools.chain( - pathlib.Path("./").glob("javascript/**/*.js"), - pathlib.Path("./").glob("stylesheets/**/*.scss"), + pathlib.Path("./javascript/").rglob("*.js"), + pathlib.Path("./stylesheets/").rglob("*.scss"), ) git_deploy_reason = "Automated Weekly Re-Sync" else: diff --git a/scripts/protect.py b/scripts/protect.py index a13d947cf9e..b1cf17e9391 100644 --- a/scripts/protect.py +++ b/scripts/protect.py @@ -27,7 +27,7 @@ def main(): if len(sys.argv[1:]) > 0: lua_files = [pathlib.Path(arg) for arg in sys.argv[1:]] elif WIKI_TO_PROTECT: - lua_files = pathlib.Path("./").glob("lua/wikis/**/*.lua") + lua_files = pathlib.Path("./lua/wikis/").rglob("*.lua") else: print("Nothing to protect") exit(0) From 577f1738c822ba5d18933f8a8982d7076c207745 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Feb 2026 11:15:33 +0900 Subject: [PATCH 22/54] cleanup --- scripts/deploy.py | 12 +++++++----- scripts/deploy_res.py | 15 +++++++++------ 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/scripts/deploy.py b/scripts/deploy.py index 945e881651b..945f45c180a 100644 --- a/scripts/deploy.py +++ b/scripts/deploy.py @@ -17,12 +17,10 @@ ) -all_modules_deployed: bool = True - - def deploy_all_files_for_wiki( wiki: str, file_paths: Iterable[pathlib.Path], deploy_reason: str -): +) -> bool: + all_modules_deployed = True token = get_token(wiki) with requests.Session() as session: session.cookies = read_cookie_jar(wiki) @@ -69,9 +67,11 @@ def deploy_all_files_for_wiki( ) time.sleep(4) print("::endgroup::") + return all_modules_deployed def main(): + all_modules_deployed = True lua_files: Iterable[pathlib.Path] git_deploy_reason: str if len(sys.argv[1:]) == 0: @@ -82,7 +82,9 @@ def main(): git_deploy_reason = get_git_deploy_reason() for wiki, files in itertools.groupby(lua_files, lambda path: path.parts[2]): - deploy_all_files_for_wiki(wiki, list(files), git_deploy_reason) + all_modules_deployed = deploy_all_files_for_wiki( + wiki, list(files), git_deploy_reason + ) if not all_modules_deployed: print("::warning::Some modules were not deployed!") diff --git a/scripts/deploy_res.py b/scripts/deploy_res.py index 8ae8695a7df..341dd270947 100644 --- a/scripts/deploy_res.py +++ b/scripts/deploy_res.py @@ -12,13 +12,11 @@ from login_and_get_token import * -all_deployed: bool = True -changes_made: bool = False - - def deploy_resources( res_type: str, file_paths: Iterable[pathlib.Path], deploy_reason: str -): +) -> tuple[bool, bool]: + all_deployed = True + changes_made = False token = get_token("commons") with requests.Session() as session: session.cookies = read_cookie_jar("commons") @@ -64,6 +62,7 @@ def deploy_resources( all_deployed = False print("::endgroup::") time.sleep(4) + return (all_deployed, changes_made) def update_cache(): @@ -91,6 +90,8 @@ def update_cache(): def main(): + all_deployed: bool = True + changes_made: bool = False resource_files: Iterable[pathlib.Path] git_deploy_reason: str if len(sys.argv[1:]) == 0: @@ -104,7 +105,9 @@ def main(): git_deploy_reason = get_git_deploy_reason() for res_type, files in itertools.groupby(resource_files, lambda path: path.suffix): - deploy_resources(res_type, list(files), git_deploy_reason) + all_deployed, changes_made = deploy_resources( + res_type, list(files), git_deploy_reason + ) if not all_deployed: print( From 87909739debc5a0cbac8c96974a51bd85be8a42d Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Feb 2026 11:20:40 +0900 Subject: [PATCH 23/54] lint --- scripts/deploy.py | 12 ++++++++++-- scripts/deploy_res.py | 12 ++++++++++-- scripts/deploy_util.py | 2 +- scripts/login_and_get_token.py | 2 +- scripts/protect.py | 6 +++++- scripts/protect_page.py | 9 +++++++-- scripts/protect_templates.py | 6 +++++- scripts/remove_dev.py | 10 ++++++++-- 8 files changed, 47 insertions(+), 12 deletions(-) diff --git a/scripts/deploy.py b/scripts/deploy.py index 945f45c180a..60cb909862b 100644 --- a/scripts/deploy.py +++ b/scripts/deploy.py @@ -9,8 +9,16 @@ import requests -from deploy_util import * -from login_and_get_token import * +from deploy_util import ( + DEPLOY_TRIGGER, + HEADER, + get_git_deploy_reason, + get_wiki_api_url, + read_cookie_jar, + read_file_from_path, + write_to_github_summary_file, +) +from login_and_get_token import get_token HEADER_PATTERN = re.compile( r"\A---\n" r"-- @Liquipedia\n" r"-- page=(?P[^\n]*)\n" diff --git a/scripts/deploy_res.py b/scripts/deploy_res.py index 341dd270947..bc5b8cfa98f 100644 --- a/scripts/deploy_res.py +++ b/scripts/deploy_res.py @@ -8,8 +8,16 @@ import requests -from deploy_util import * -from login_and_get_token import * +from deploy_util import ( + DEPLOY_TRIGGER, + HEADER, + get_git_deploy_reason, + get_wiki_api_url, + read_cookie_jar, + read_file_from_path, + write_to_github_summary_file, +) +from login_and_get_token import get_token def deploy_resources( diff --git a/scripts/deploy_util.py b/scripts/deploy_util.py index 8c9caff03f5..6778b534b55 100644 --- a/scripts/deploy_util.py +++ b/scripts/deploy_util.py @@ -57,7 +57,7 @@ def read_cookie_jar(wiki: str) -> http.cookiejar.FileCookieJar: cookie_jar = http.cookiejar.LWPCookieJar(filename=ckf) try: cookie_jar.load(ignore_discard=True) - except: + except OSError: pass return cookie_jar diff --git a/scripts/login_and_get_token.py b/scripts/login_and_get_token.py index a6420a86366..392083f77b3 100644 --- a/scripts/login_and_get_token.py +++ b/scripts/login_and_get_token.py @@ -4,7 +4,7 @@ import requests -from deploy_util import * +from deploy_util import HEADER, get_wiki_api_url, read_cookie_jar __all__ = ["get_token"] diff --git a/scripts/protect.py b/scripts/protect.py index b1cf17e9391..27180fa5840 100644 --- a/scripts/protect.py +++ b/scripts/protect.py @@ -5,7 +5,11 @@ from typing import Iterable from deploy_util import get_wikis -from protect_page import * +from protect_page import ( + protect_non_existing_page, + protect_existing_page, + handle_protect_errors, +) WIKI_TO_PROTECT = os.getenv("WIKI_TO_PROTECT") diff --git a/scripts/protect_page.py b/scripts/protect_page.py index 2ca95854824..77953570109 100644 --- a/scripts/protect_page.py +++ b/scripts/protect_page.py @@ -4,8 +4,13 @@ import requests -from deploy_util import * -from login_and_get_token import * +from deploy_util import ( + HEADER, + get_wiki_api_url, + read_cookie_jar, + write_to_github_summary_file, +) +from login_and_get_token import get_token __all__ = [ "protect_non_existing_page", diff --git a/scripts/protect_templates.py b/scripts/protect_templates.py index f214cac8485..7b34b85110b 100644 --- a/scripts/protect_templates.py +++ b/scripts/protect_templates.py @@ -1,6 +1,10 @@ import os -from protect_page import * +from protect_page import ( + protect_non_existing_page, + protect_existing_page, + handle_protect_errors, +) WIKI_TO_PROTECT = os.getenv("WIKI_TO_PROTECT") diff --git a/scripts/remove_dev.py b/scripts/remove_dev.py index dfbde67b7e2..21b22f5c43d 100644 --- a/scripts/remove_dev.py +++ b/scripts/remove_dev.py @@ -3,8 +3,14 @@ import requests -from deploy_util import * -from login_and_get_token import * +from deploy_util import ( + HEADER, + get_wiki_api_url, + get_wikis, + read_cookie_jar, + write_to_github_summary_file, +) +from login_and_get_token import get_token LUA_DEV_ENV_NAME = os.getenv("LUA_DEV_ENV_NAME") From 7df42c277a5cc8ca20adba3dffec48c81e3cfa2d Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Feb 2026 11:23:50 +0900 Subject: [PATCH 24/54] key safety --- scripts/deploy.py | 4 ++-- scripts/deploy_res.py | 2 +- scripts/protect_page.py | 2 +- scripts/remove_dev.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/deploy.py b/scripts/deploy.py index 60cb909862b..fc283dbae1b 100644 --- a/scripts/deploy.py +++ b/scripts/deploy.py @@ -56,9 +56,9 @@ def deploy_all_files_for_wiki( "token": token, }, ).json() - result = response["edit"]["result"] + result = response["edit"].get("result") if result == "Success": - no_change = response["edit"]["nochange"] + no_change = response["edit"].get("nochange") if len(no_change) == 0 and DEPLOY_TRIGGER == "push": print(f"::notice file={str(file_path)}::No change made") elif len(no_change) != 0 and DEPLOY_TRIGGER != "push": diff --git a/scripts/deploy_res.py b/scripts/deploy_res.py index bc5b8cfa98f..241580777be 100644 --- a/scripts/deploy_res.py +++ b/scripts/deploy_res.py @@ -49,7 +49,7 @@ def deploy_resources( "token": token, }, ).json() - result = response["edit"]["result"] + result = response["edit"].get("result") new_rev_id = response["edit"].get("newrevid") if result == "Success": if new_rev_id is not None: diff --git a/scripts/protect_page.py b/scripts/protect_page.py index 77953570109..dbb22509e29 100644 --- a/scripts/protect_page.py +++ b/scripts/protect_page.py @@ -49,7 +49,7 @@ def protect_page(page: str, wiki: str, protect_mode: Literal["edit", "create"]): ).json() time.sleep(4) - protections = response["protect"]["protections"] + protections = response["protect"].get("protections") for protection in protections: if protection[protect_mode] == "allow-only-sysop": return diff --git a/scripts/remove_dev.py b/scripts/remove_dev.py index 21b22f5c43d..e44a2c8db28 100644 --- a/scripts/remove_dev.py +++ b/scripts/remove_dev.py @@ -59,7 +59,7 @@ def search_and_remove(wiki: str): }, ).json() time.sleep(4) - pages = search_result["query"]["search"] + pages = search_result["query"].get("search") if len(pages) == 0: return From e9e87a640102ba17613db6d3b5d1cdf0cd6419f8 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Feb 2026 11:29:03 +0900 Subject: [PATCH 25/54] add python check workflow --- .github/workflows/pythoncheck.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .github/workflows/pythoncheck.yml diff --git a/.github/workflows/pythoncheck.yml b/.github/workflows/pythoncheck.yml new file mode 100644 index 00000000000..0e2cdbe68a1 --- /dev/null +++ b/.github/workflows/pythoncheck.yml @@ -0,0 +1,19 @@ +name: Python Code Style + +on: [pull_request, workflow_dispatch] + +jobs: + python-code-style: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v6 + - name: Run linter + uses: astral-sh/ruff-action@v3 + with: + src: scripts/*.py + - name: Check styling + uses: psf/black@stable + with: + src: scripts/*.py From 014472fec51985c81a21eecde0dcaf45e2cc48d2 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Feb 2026 11:32:33 +0900 Subject: [PATCH 26/54] adjust workflow --- .github/workflows/pythoncheck.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythoncheck.yml b/.github/workflows/pythoncheck.yml index 0e2cdbe68a1..81574745d2e 100644 --- a/.github/workflows/pythoncheck.yml +++ b/.github/workflows/pythoncheck.yml @@ -12,8 +12,8 @@ jobs: - name: Run linter uses: astral-sh/ruff-action@v3 with: - src: scripts/*.py + src: scripts/ - name: Check styling uses: psf/black@stable with: - src: scripts/*.py + src: scripts/ From 4f8f406da048f35f4a3d663de64e226be1b2a609 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Feb 2026 11:33:46 +0900 Subject: [PATCH 27/54] format --- scripts/remove_dev.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/remove_dev.py b/scripts/remove_dev.py index e44a2c8db28..609e89ad25a 100644 --- a/scripts/remove_dev.py +++ b/scripts/remove_dev.py @@ -12,7 +12,6 @@ ) from login_and_get_token import get_token - LUA_DEV_ENV_NAME = os.getenv("LUA_DEV_ENV_NAME") remove_errors: list[str] = list() From 0be047ef856c8f5f79855e92be90bb7c9f877fc3 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Feb 2026 11:37:21 +0900 Subject: [PATCH 28/54] safer no change detect --- scripts/deploy.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/deploy.py b/scripts/deploy.py index fc283dbae1b..1df27bc9ca9 100644 --- a/scripts/deploy.py +++ b/scripts/deploy.py @@ -58,10 +58,10 @@ def deploy_all_files_for_wiki( ).json() result = response["edit"].get("result") if result == "Success": - no_change = response["edit"].get("nochange") - if len(no_change) == 0 and DEPLOY_TRIGGER == "push": + new_rev_id = response["edit"].get("newrevid") + if new_rev_id is None and DEPLOY_TRIGGER == "push": print(f"::notice file={str(file_path)}::No change made") - elif len(no_change) != 0 and DEPLOY_TRIGGER != "push": + elif new_rev_id is not None and DEPLOY_TRIGGER != "push": print(f"::warning file={str(file_path)}::File changed") print("...done") write_to_github_summary_file( From 9cf7fc271e5bedb091b8aaee5ad87eceb788c940 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Feb 2026 11:40:00 +0900 Subject: [PATCH 29/54] sort paths before looping --- scripts/deploy.py | 2 +- scripts/protect.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/deploy.py b/scripts/deploy.py index 1df27bc9ca9..419d41871d9 100644 --- a/scripts/deploy.py +++ b/scripts/deploy.py @@ -89,7 +89,7 @@ def main(): lua_files = [pathlib.Path(arg) for arg in sys.argv[1:]] git_deploy_reason = get_git_deploy_reason() - for wiki, files in itertools.groupby(lua_files, lambda path: path.parts[2]): + for wiki, files in itertools.groupby(sorted(lua_files), lambda path: path.parts[2]): all_modules_deployed = deploy_all_files_for_wiki( wiki, list(files), git_deploy_reason ) diff --git a/scripts/protect.py b/scripts/protect.py index 27180fa5840..2ab76145714 100644 --- a/scripts/protect.py +++ b/scripts/protect.py @@ -36,7 +36,7 @@ def main(): print("Nothing to protect") exit(0) - for file_to_protect in lua_files: + for file_to_protect in sorted(lua_files): print(f"::group::Checking { str(file_to_protect) }") wiki = file_to_protect.parts[2] module = "/".join(file_to_protect.parts[3:])[:-4] From 614e5eecdf50f7f948252505df5a4035ea887a57 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Feb 2026 11:41:31 +0900 Subject: [PATCH 30/54] sort resources too --- scripts/deploy_res.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/deploy_res.py b/scripts/deploy_res.py index 241580777be..50d3a84d882 100644 --- a/scripts/deploy_res.py +++ b/scripts/deploy_res.py @@ -112,7 +112,9 @@ def main(): resource_files = [pathlib.Path(arg) for arg in sys.argv[1:]] git_deploy_reason = get_git_deploy_reason() - for res_type, files in itertools.groupby(resource_files, lambda path: path.suffix): + for res_type, files in itertools.groupby( + sorted(resource_files), lambda path: path.suffix + ): all_deployed, changes_made = deploy_resources( res_type, list(files), git_deploy_reason ) From 5de7147705e7f63b6c8b22878b2ec76a073677d7 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Feb 2026 15:19:22 +0900 Subject: [PATCH 31/54] adjust workflow --- .github/workflows/delete dev env.yml | 8 ++++++++ .github/workflows/deploy test.yml | 1 - .github/workflows/deploy.yml | 1 - .github/workflows/protect new wiki modules.yml | 1 - .github/workflows/protect new wiki templates.yml | 1 - .github/workflows/scheduled.yml | 1 - 6 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/delete dev env.yml b/.github/workflows/delete dev env.yml index c17d5b196f4..f850adca2e1 100644 --- a/.github/workflows/delete dev env.yml +++ b/.github/workflows/delete dev env.yml @@ -24,6 +24,14 @@ jobs: with: fetch-depth: 0 + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: '3.14' + + - name: Install Python Dependency + run: pip install requests + - name: Remove Lua Dev Env Modules env: WIKI_USER: ${{ secrets.LP_BOTUSER }} diff --git a/.github/workflows/deploy test.yml b/.github/workflows/deploy test.yml index a459a0f85c9..ec3ca33d011 100644 --- a/.github/workflows/deploy test.yml +++ b/.github/workflows/deploy test.yml @@ -33,7 +33,6 @@ jobs: with: python-version: '3.14' - - name: Install Python Dependency if: steps.changed-files.outputs.any_changed == 'true' run: pip install requests diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index ebcc434649f..2abab23cd81 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -35,7 +35,6 @@ jobs: with: python-version: '3.14' - - name: Install Python Dependency if: steps.res-changed-files.outputs.any_changed == 'true' || steps.lua-changed-files.outputs.any_changed == 'true' run: pip install requests diff --git a/.github/workflows/protect new wiki modules.yml b/.github/workflows/protect new wiki modules.yml index cdf0d86f204..6bb1caafa9a 100644 --- a/.github/workflows/protect new wiki modules.yml +++ b/.github/workflows/protect new wiki modules.yml @@ -21,7 +21,6 @@ jobs: with: python-version: '3.14' - - name: Install Python Dependency run: pip install requests diff --git a/.github/workflows/protect new wiki templates.yml b/.github/workflows/protect new wiki templates.yml index cdaf06c5ffd..3b1542f5a2b 100644 --- a/.github/workflows/protect new wiki templates.yml +++ b/.github/workflows/protect new wiki templates.yml @@ -21,7 +21,6 @@ jobs: with: python-version: '3.14' - - name: Install Python Dependency run: pip install requests diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index c81245ecc55..0d2d29c3d8b 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -20,7 +20,6 @@ jobs: with: python-version: '3.14' - - name: Install Python Dependency run: pip install requests From 0ca45cf274c8365516762ca001121ab0e70f18b1 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Feb 2026 18:42:33 +0900 Subject: [PATCH 32/54] extract wiki deploy function --- scripts/deploy.py | 38 +++++------------------------------ scripts/deploy_res.py | 40 ++++++------------------------------- scripts/deploy_util.py | 45 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 67 deletions(-) diff --git a/scripts/deploy.py b/scripts/deploy.py index 419d41871d9..c4653012175 100644 --- a/scripts/deploy.py +++ b/scripts/deploy.py @@ -10,10 +10,8 @@ import requests from deploy_util import ( - DEPLOY_TRIGGER, - HEADER, get_git_deploy_reason, - get_wiki_api_url, + deploy_file_to_wiki, read_cookie_jar, read_file_from_path, write_to_github_summary_file, @@ -43,36 +41,10 @@ def deploy_all_files_for_wiki( page = header_match.groupdict()["pageName"] + ( os.getenv("LUA_DEV_ENV_NAME") or "" ) - response = session.post( - get_wiki_api_url(wiki), - headers=HEADER, - params={"format": "json", "action": "edit"}, - data={ - "title": page, - "text": file_content, - "summary": f"Git: {deploy_reason}", - "bot": "true", - "recreate": "true", - "token": token, - }, - ).json() - result = response["edit"].get("result") - if result == "Success": - new_rev_id = response["edit"].get("newrevid") - if new_rev_id is None and DEPLOY_TRIGGER == "push": - print(f"::notice file={str(file_path)}::No change made") - elif new_rev_id is not None and DEPLOY_TRIGGER != "push": - print(f"::warning file={str(file_path)}::File changed") - print("...done") - write_to_github_summary_file( - f":information_source: {str(file_path)} successfully deployed" - ) - else: - all_modules_deployed = False - print(f"::warning file={str(file_path)}::failed to deploy") - write_to_github_summary_file( - f":warning: {str(file_path)} failed to deploy" - ) + module_deployed, _ = deploy_file_to_wiki( + session, file_path, file_content, wiki, page, token, deploy_reason + ) + all_modules_deployed = all_modules_deployed and module_deployed time.sleep(4) print("::endgroup::") return all_modules_deployed diff --git a/scripts/deploy_res.py b/scripts/deploy_res.py index 50d3a84d882..1fe887ab9f6 100644 --- a/scripts/deploy_res.py +++ b/scripts/deploy_res.py @@ -9,13 +9,12 @@ import requests from deploy_util import ( - DEPLOY_TRIGGER, HEADER, get_git_deploy_reason, get_wiki_api_url, + deploy_file_to_wiki, read_cookie_jar, read_file_from_path, - write_to_github_summary_file, ) from login_and_get_token import get_token @@ -36,38 +35,11 @@ def deploy_resources( + file_path.name ) print(f"...page = { page }") - response = session.post( - get_wiki_api_url("commons"), - headers=HEADER, - params={"format": "json", "action": "edit"}, - data={ - "title": page, - "text": file_content, - "summary": f"Git: {deploy_reason}", - "bot": "true", - "recreate": "true", - "token": token, - }, - ).json() - result = response["edit"].get("result") - new_rev_id = response["edit"].get("newrevid") - if result == "Success": - if new_rev_id is not None: - changes_made = True - if DEPLOY_TRIGGER != "push": - print(f"::warning file={str(file_path)}::File changed") - print(f"...{result}") - print("...done") - write_to_github_summary_file( - f":information_source: {str(file_path)} successfully deployed" - ) - - else: - print(f"::warning file={str(file_path)}::failed to deploy") - write_to_github_summary_file( - f":warning: {str(file_path)} failed to deploy" - ) - all_deployed = False + deploy_result = deploy_file_to_wiki( + session, file_path, file_content, "commons", page, token, deploy_reason + ) + all_deployed = all_deployed and deploy_result[0] + changes_made = changes_made or deploy_result[1] print("::endgroup::") time.sleep(4) return (all_deployed, changes_made) diff --git a/scripts/deploy_util.py b/scripts/deploy_util.py index 6778b534b55..03b04ef4888 100644 --- a/scripts/deploy_util.py +++ b/scripts/deploy_util.py @@ -10,6 +10,7 @@ __all__ = [ "DEPLOY_TRIGGER", "HEADER", + "deploy_file_to_wiki", "get_git_deploy_reason", "get_wiki_api_url", "get_wikis", @@ -52,6 +53,50 @@ def get_git_deploy_reason(): ) +def deploy_file_to_wiki( + session: requests.Session, + file_path: pathlib.Path, + file_content: str, + wiki: str, + target_page: str, + token: str, + deploy_reason: str, +) -> tuple[bool, bool]: + change_made = False + deployed = True + response = session.post( + get_wiki_api_url(wiki), + headers=HEADER, + params={"format": "json", "action": "edit"}, + data={ + "title": target_page, + "text": file_content, + "summary": f"Git: {deploy_reason}", + "bot": "true", + "recreate": "true", + "token": token, + }, + ).json() + result = response["edit"].get("result") + new_rev_id = response["edit"].get("newrevid") + if result == "Success": + if new_rev_id is not None: + change_made = True + if DEPLOY_TRIGGER != "push": + print(f"::warning file={str(file_path)}::File changed") + print(f"...{result}") + print("...done") + write_to_github_summary_file( + f":information_source: {str(file_path)} successfully deployed" + ) + + else: + print(f"::warning file={str(file_path)}::failed to deploy") + write_to_github_summary_file(f":warning: {str(file_path)} failed to deploy") + deployed = False + return deployed, change_made + + def read_cookie_jar(wiki: str) -> http.cookiejar.FileCookieJar: ckf = f"cookie_{wiki}.ck" cookie_jar = http.cookiejar.LWPCookieJar(filename=ckf) From 053418757c0080c3eed0c07fa6e32017cc7225e9 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Feb 2026 18:47:07 +0900 Subject: [PATCH 33/54] extract sleep duration to constant --- scripts/deploy.py | 2 -- scripts/deploy_res.py | 2 -- scripts/deploy_util.py | 5 ++++- scripts/login_and_get_token.py | 4 ++-- scripts/protect_page.py | 5 +++-- scripts/remove_dev.py | 5 +++-- 6 files changed, 12 insertions(+), 11 deletions(-) diff --git a/scripts/deploy.py b/scripts/deploy.py index c4653012175..eaf3b943c6c 100644 --- a/scripts/deploy.py +++ b/scripts/deploy.py @@ -3,7 +3,6 @@ import pathlib import re import sys -import time from typing import Iterable @@ -45,7 +44,6 @@ def deploy_all_files_for_wiki( session, file_path, file_content, wiki, page, token, deploy_reason ) all_modules_deployed = all_modules_deployed and module_deployed - time.sleep(4) print("::endgroup::") return all_modules_deployed diff --git a/scripts/deploy_res.py b/scripts/deploy_res.py index 1fe887ab9f6..cd57e5fb0b0 100644 --- a/scripts/deploy_res.py +++ b/scripts/deploy_res.py @@ -2,7 +2,6 @@ import pathlib import sys import subprocess -import time from typing import Iterable @@ -41,7 +40,6 @@ def deploy_resources( all_deployed = all_deployed and deploy_result[0] changes_made = changes_made or deploy_result[1] print("::endgroup::") - time.sleep(4) return (all_deployed, changes_made) diff --git a/scripts/deploy_util.py b/scripts/deploy_util.py index 03b04ef4888..e3029b2b595 100644 --- a/scripts/deploy_util.py +++ b/scripts/deploy_util.py @@ -10,6 +10,7 @@ __all__ = [ "DEPLOY_TRIGGER", "HEADER", + "SLEEP_DURATION", "deploy_file_to_wiki", "get_git_deploy_reason", "get_wiki_api_url", @@ -28,6 +29,7 @@ "accept": "application/json", "Accept-Encoding": "gzip", } +SLEEP_DURATION = 4 def get_wikis() -> set[str]: @@ -36,7 +38,7 @@ def get_wikis() -> set[str]: headers=HEADER, ) wikis = response.json() - time.sleep(4) + time.sleep(SLEEP_DURATION) return set(wikis["allwikis"].keys()) @@ -94,6 +96,7 @@ def deploy_file_to_wiki( print(f"::warning file={str(file_path)}::failed to deploy") write_to_github_summary_file(f":warning: {str(file_path)} failed to deploy") deployed = False + time.sleep(SLEEP_DURATION) return deployed, change_made diff --git a/scripts/login_and_get_token.py b/scripts/login_and_get_token.py index 392083f77b3..b36bc742280 100644 --- a/scripts/login_and_get_token.py +++ b/scripts/login_and_get_token.py @@ -4,7 +4,7 @@ import requests -from deploy_util import HEADER, get_wiki_api_url, read_cookie_jar +from deploy_util import HEADER, SLEEP_DURATION, get_wiki_api_url, read_cookie_jar __all__ = ["get_token"] @@ -43,7 +43,7 @@ def login(wiki: str): ) loggedin.add(wiki) cookie_jar.save(ignore_discard=True) - time.sleep(4) + time.sleep(SLEEP_DURATION) @functools.cache diff --git a/scripts/protect_page.py b/scripts/protect_page.py index dbb22509e29..f2c4f925f8b 100644 --- a/scripts/protect_page.py +++ b/scripts/protect_page.py @@ -6,6 +6,7 @@ from deploy_util import ( HEADER, + SLEEP_DURATION, get_wiki_api_url, read_cookie_jar, write_to_github_summary_file, @@ -48,7 +49,7 @@ def protect_page(page: str, wiki: str, protect_mode: Literal["edit", "create"]): }, ).json() - time.sleep(4) + time.sleep(SLEEP_DURATION) protections = response["protect"].get("protections") for protection in protections: if protection[protect_mode] == "allow-only-sysop": @@ -68,7 +69,7 @@ def check_if_page_exists(page: str, wiki: str) -> bool: data={"titles": page, "prop": "info"}, ).text - time.sleep(4) + time.sleep(SLEEP_DURATION) return 'missing":"' in result diff --git a/scripts/remove_dev.py b/scripts/remove_dev.py index 609e89ad25a..d64bf9bffe7 100644 --- a/scripts/remove_dev.py +++ b/scripts/remove_dev.py @@ -5,6 +5,7 @@ from deploy_util import ( HEADER, + SLEEP_DURATION, get_wiki_api_url, get_wikis, read_cookie_jar, @@ -34,7 +35,7 @@ def remove_page(session: requests.Session, page: str, wiki: str): "token": token, }, ).text - time.sleep(8) + time.sleep(SLEEP_DURATION) if '"delete"' not in result: print(f"::warning::could not delete {page} on {wiki}") @@ -57,7 +58,7 @@ def search_and_remove(wiki: str): "srprop": "", }, ).json() - time.sleep(4) + time.sleep(SLEEP_DURATION) pages = search_result["query"].get("search") if len(pages) == 0: From b1128e8b9e01be36d1e968a51f207f28abdab33b Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Feb 2026 19:07:09 +0900 Subject: [PATCH 34/54] use ruff for styling too --- .github/workflows/pythoncheck.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/pythoncheck.yml b/.github/workflows/pythoncheck.yml index 81574745d2e..6b5a74f19ae 100644 --- a/.github/workflows/pythoncheck.yml +++ b/.github/workflows/pythoncheck.yml @@ -14,6 +14,4 @@ jobs: with: src: scripts/ - name: Check styling - uses: psf/black@stable - with: - src: scripts/ + run: ruff format scripts/ From 5ede33b353e2d2529e3cb086556f8953a950b13a Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Feb 2026 19:07:16 +0900 Subject: [PATCH 35/54] style --- scripts/deploy_res.py | 4 ++-- scripts/deploy_util.py | 2 +- scripts/login_and_get_token.py | 2 +- scripts/protect.py | 2 +- scripts/protect_page.py | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/deploy_res.py b/scripts/deploy_res.py index cd57e5fb0b0..5f01b0dd5ef 100644 --- a/scripts/deploy_res.py +++ b/scripts/deploy_res.py @@ -30,10 +30,10 @@ def deploy_resources( print(f"::group::Checking {str(file_path)}") file_content = read_file_from_path(file_path) page = ( - f"MediaWiki:Common.{ "js" if res_type == ".js" else "css" }/" + f"MediaWiki:Common.{'js' if res_type == '.js' else 'css'}/" + file_path.name ) - print(f"...page = { page }") + print(f"...page = {page}") deploy_result = deploy_file_to_wiki( session, file_path, file_content, "commons", page, token, deploy_reason ) diff --git a/scripts/deploy_util.py b/scripts/deploy_util.py index e3029b2b595..25ec97a38e7 100644 --- a/scripts/deploy_util.py +++ b/scripts/deploy_util.py @@ -22,7 +22,7 @@ DEPLOY_TRIGGER = os.getenv("DEPLOY_TRIGGER") GITHUB_STEP_SUMMARY_FILE = os.getenv("GITHUB_STEP_SUMMARY") -USER_AGENT = f"GitHub Autodeploy Bot/2.0.0 ({ os.getenv("WIKI_UA_EMAIL") })" +USER_AGENT = f"GitHub Autodeploy Bot/2.0.0 ({os.getenv('WIKI_UA_EMAIL')})" WIKI_BASE_URL = os.getenv("WIKI_BASE_URL") HEADER = { "User-Agent": USER_AGENT, diff --git a/scripts/login_and_get_token.py b/scripts/login_and_get_token.py index b36bc742280..d91b92afd10 100644 --- a/scripts/login_and_get_token.py +++ b/scripts/login_and_get_token.py @@ -18,7 +18,7 @@ def login(wiki: str): if wiki in loggedin: return cookie_jar = read_cookie_jar(wiki) - print(f"...logging in on { wiki }") + print(f"...logging in on {wiki}") with requests.Session() as session: session.cookies = cookie_jar token_response = session.post( diff --git a/scripts/protect.py b/scripts/protect.py index 2ab76145714..cb00b8b0446 100644 --- a/scripts/protect.py +++ b/scripts/protect.py @@ -37,7 +37,7 @@ def main(): exit(0) for file_to_protect in sorted(lua_files): - print(f"::group::Checking { str(file_to_protect) }") + print(f"::group::Checking {str(file_to_protect)}") wiki = file_to_protect.parts[2] module = "/".join(file_to_protect.parts[3:])[:-4] page = "Module:" + module diff --git a/scripts/protect_page.py b/scripts/protect_page.py index f2c4f925f8b..6608014003c 100644 --- a/scripts/protect_page.py +++ b/scripts/protect_page.py @@ -30,8 +30,8 @@ def protect_page(page: str, wiki: str, protect_mode: Literal["edit", "create"]): protect_options = "create=allow-only-sysop" else: raise ValueError(f"invalid protect mode: {protect_mode}") - print(f"...wiki = { wiki }") - print(f"...page = { page }") + print(f"...wiki = {wiki}") + print(f"...page = {page}") token = get_token(wiki) with requests.Session() as session: session.cookies = read_cookie_jar(wiki) From ad1b782c4e3353f7d51c03ebf3771a800ad1a280 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Feb 2026 19:07:49 +0900 Subject: [PATCH 36/54] oops --- .github/workflows/pythoncheck.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythoncheck.yml b/.github/workflows/pythoncheck.yml index 6b5a74f19ae..a181006e108 100644 --- a/.github/workflows/pythoncheck.yml +++ b/.github/workflows/pythoncheck.yml @@ -14,4 +14,4 @@ jobs: with: src: scripts/ - name: Check styling - run: ruff format scripts/ + run: ruff format --check --diff scripts/ From 2076b6dd4c26808715ba0bde5e2973de23a2d879 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Feb 2026 19:32:55 +0900 Subject: [PATCH 37/54] adjust workflow trigger --- .github/workflows/pythoncheck.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pythoncheck.yml b/.github/workflows/pythoncheck.yml index a181006e108..ad756d5a1ec 100644 --- a/.github/workflows/pythoncheck.yml +++ b/.github/workflows/pythoncheck.yml @@ -1,6 +1,11 @@ name: Python Code Style -on: [pull_request, workflow_dispatch] +on: + pull_request: + paths: + - '**.py' + - '.github/workflows/*.yml' + workflow_dispatch: jobs: python-code-style: From 55cf079ad77606f4697130ff4a100937aaa97d5b Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Feb 2026 20:28:11 +0900 Subject: [PATCH 38/54] code conciseness Co-authored-by: SyntacticSalt --- scripts/deploy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/deploy.py b/scripts/deploy.py index eaf3b943c6c..662139e7f3b 100644 --- a/scripts/deploy.py +++ b/scripts/deploy.py @@ -43,7 +43,7 @@ def deploy_all_files_for_wiki( module_deployed, _ = deploy_file_to_wiki( session, file_path, file_content, wiki, page, token, deploy_reason ) - all_modules_deployed = all_modules_deployed and module_deployed + all_modules_deployed &= module_deployed print("::endgroup::") return all_modules_deployed @@ -60,7 +60,7 @@ def main(): git_deploy_reason = get_git_deploy_reason() for wiki, files in itertools.groupby(sorted(lua_files), lambda path: path.parts[2]): - all_modules_deployed = deploy_all_files_for_wiki( + all_modules_deployed &= deploy_all_files_for_wiki( wiki, list(files), git_deploy_reason ) From 3b604f0195177a213459497ea21b4428de0ed876 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Fri, 6 Feb 2026 12:04:55 +0900 Subject: [PATCH 39/54] fix login in dev remove script --- scripts/remove_dev.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/remove_dev.py b/scripts/remove_dev.py index d64bf9bffe7..1f805c27c5b 100644 --- a/scripts/remove_dev.py +++ b/scripts/remove_dev.py @@ -11,7 +11,7 @@ read_cookie_jar, write_to_github_summary_file, ) -from login_and_get_token import get_token +from login_and_get_token import get_token, login LUA_DEV_ENV_NAME = os.getenv("LUA_DEV_ENV_NAME") @@ -34,10 +34,10 @@ def remove_page(session: requests.Session, page: str, wiki: str): "reason": f"Remove {LUA_DEV_ENV_NAME}", "token": token, }, - ).text + ).json() time.sleep(SLEEP_DURATION) - if '"delete"' not in result: + if "delete" not in result.keys(): print(f"::warning::could not delete {page} on {wiki}") write_to_github_summary_file(f":warning: could not delete {page} on {wiki}") remove_errors.append(f"{wiki}:{page}") @@ -45,7 +45,6 @@ def remove_page(session: requests.Session, page: str, wiki: str): def search_and_remove(wiki: str): with requests.Session() as session: - session.cookies = read_cookie_jar(wiki) search_result = session.post( get_wiki_api_url(wiki), headers=HEADER, @@ -64,6 +63,9 @@ def search_and_remove(wiki: str): if len(pages) == 0: return + login(wiki) + session.cookies = read_cookie_jar(wiki) + for page in pages: if os.getenv("INCLUDE_SUB_ENVS") == "true" or page["title"].endswith( LUA_DEV_ENV_NAME From fc35084ae5d2c77da962b12dc269224a982b3295 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Fri, 6 Feb 2026 12:10:41 +0900 Subject: [PATCH 40/54] fix incorrect condition reading --- scripts/remove_dev.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/remove_dev.py b/scripts/remove_dev.py index 1f805c27c5b..8479fc985cd 100644 --- a/scripts/remove_dev.py +++ b/scripts/remove_dev.py @@ -75,7 +75,7 @@ def search_and_remove(wiki: str): def main(): for wiki in get_wikis(): - if wiki == "commons" and os.getenv("INCLUDE_COMMONS") == "true": + if wiki == "commons" and os.getenv("INCLUDE_COMMONS") != "true": continue search_and_remove(wiki) From 630f68bc2c112e0777d54016122dfe37751134aa Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Fri, 27 Feb 2026 16:43:02 +0900 Subject: [PATCH 41/54] error handling --- scripts/protect_page.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/protect_page.py b/scripts/protect_page.py index 6608014003c..c3ab446e8b7 100644 --- a/scripts/protect_page.py +++ b/scripts/protect_page.py @@ -50,6 +50,11 @@ def protect_page(page: str, wiki: str, protect_mode: Literal["edit", "create"]): ).json() time.sleep(SLEEP_DURATION) + if response.get("error"): + print( + f"::warning::could not ({protect_mode}) protect {page} on {wiki}: {response['error']['info']}" + ) + return protections = response["protect"].get("protections") for protection in protections: if protection[protect_mode] == "allow-only-sysop": From 597f85d23b8979ce5ea3efc9307685ffe8724606 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Fri, 27 Feb 2026 16:43:10 +0900 Subject: [PATCH 42/54] pick up env file --- .vscode/settings.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.vscode/settings.json b/.vscode/settings.json index 919b327f401..e367bc086f5 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -12,6 +12,8 @@ "css.validate": false, "less.validate": false, "scss.validate": false, + "python.envFile": "${workspaceFolder}/.env", + "python.terminal.useEnvFile": true, "workbench.editor.customLabels.patterns": { "**/wikis/*/*.lua": "${dirname}: ${filename}", "**/wikis/*/*/*.lua": "${dirname(-4)}: ${dirname}/${filename}", From bd50d467dd91e1fe17f729ad97befd2f0e5feca9 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Fri, 27 Feb 2026 16:52:52 +0900 Subject: [PATCH 43/54] handle empty lines better --- scripts/protect_templates.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/protect_templates.py b/scripts/protect_templates.py index 7b34b85110b..e3ff09e5065 100644 --- a/scripts/protect_templates.py +++ b/scripts/protect_templates.py @@ -12,6 +12,8 @@ def main(): with open("./templates/templatesToProtect", "r") as templates_to_protect: for template_name in templates_to_protect.readlines(): + if len(template_name.strip()) == 0: + continue template = "Template:" + template_name print(f"::group::Checking {WIKI_TO_PROTECT}:{template}") if WIKI_TO_PROTECT == "commons": From ce5f18943a891e80c5159dfc27491c54a9d521f5 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Fri, 27 Feb 2026 16:55:26 +0900 Subject: [PATCH 44/54] better page existence check --- scripts/protect_page.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/protect_page.py b/scripts/protect_page.py index c3ab446e8b7..1060e9143a3 100644 --- a/scripts/protect_page.py +++ b/scripts/protect_page.py @@ -72,10 +72,10 @@ def check_if_page_exists(page: str, wiki: str) -> bool: headers=HEADER, params={"format": "json", "action": "query"}, data={"titles": page, "prop": "info"}, - ).text + ).json() time.sleep(SLEEP_DURATION) - return 'missing":"' in result + return "-1" not in result["query"]["pages"] def protect_non_existing_page(page: str, wiki: str): From 5fd4846e893e5467680be5a9bcbc795b464cfb5d Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Mon, 2 Mar 2026 18:18:20 +0900 Subject: [PATCH 45/54] fix indent --- .vscode/settings.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index e367bc086f5..acda9e63d38 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -13,7 +13,7 @@ "less.validate": false, "scss.validate": false, "python.envFile": "${workspaceFolder}/.env", - "python.terminal.useEnvFile": true, + "python.terminal.useEnvFile": true, "workbench.editor.customLabels.patterns": { "**/wikis/*/*.lua": "${dirname}: ${filename}", "**/wikis/*/*/*.lua": "${dirname(-4)}: ${dirname}/${filename}", From c18176d07aff27ca970ad484c939f90004e5fc46 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Mon, 2 Mar 2026 18:23:05 +0900 Subject: [PATCH 46/54] restore error printing --- scripts/remove_dev.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/remove_dev.py b/scripts/remove_dev.py index 8479fc985cd..79c51e661f4 100644 --- a/scripts/remove_dev.py +++ b/scripts/remove_dev.py @@ -78,6 +78,14 @@ def main(): if wiki == "commons" and os.getenv("INCLUDE_COMMONS") != "true": continue search_and_remove(wiki) + if len(remove_errors) == 0: + return + print("::warning::Could not delete some pages on some wikis") + write_to_github_summary_file("::warning::Could not delete some pages on some wikis") + print("::group::Failed protections") + for remove_error in remove_errors: + print(remove_error) + print("endgroup") if __name__ == "__main__": From d8a26812ed961c2e9511ce412f524ced63d0134c Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Mon, 2 Mar 2026 18:24:52 +0900 Subject: [PATCH 47/54] exit codes --- scripts/remove_dev.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/remove_dev.py b/scripts/remove_dev.py index 79c51e661f4..816e945cb02 100644 --- a/scripts/remove_dev.py +++ b/scripts/remove_dev.py @@ -79,13 +79,14 @@ def main(): continue search_and_remove(wiki) if len(remove_errors) == 0: - return + exit(0) print("::warning::Could not delete some pages on some wikis") write_to_github_summary_file("::warning::Could not delete some pages on some wikis") print("::group::Failed protections") for remove_error in remove_errors: print(remove_error) print("endgroup") + exit(1) if __name__ == "__main__": From d9b89d7f21fda6605a094b20889273485df6eb17 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Mon, 2 Mar 2026 18:27:40 +0900 Subject: [PATCH 48/54] fix condition --- scripts/protect.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/protect.py b/scripts/protect.py index cb00b8b0446..5bc3387e7aa 100644 --- a/scripts/protect.py +++ b/scripts/protect.py @@ -22,7 +22,7 @@ def check_for_local_version(module: str, wiki: str): def protect_if_has_no_local_version(module: str, wiki: str): page = "Module:" + module - if check_for_local_version(module, wiki): + if not check_for_local_version(module, wiki): protect_non_existing_page(page, wiki) From cf3ddb04057e4c550a0524dc299cdd3b5faed46a Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Mon, 2 Mar 2026 18:32:18 +0900 Subject: [PATCH 49/54] use read().splitlines() --- scripts/protect_templates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/protect_templates.py b/scripts/protect_templates.py index e3ff09e5065..488b93be20b 100644 --- a/scripts/protect_templates.py +++ b/scripts/protect_templates.py @@ -11,7 +11,7 @@ def main(): with open("./templates/templatesToProtect", "r") as templates_to_protect: - for template_name in templates_to_protect.readlines(): + for template_name in templates_to_protect.read().splitlines(): if len(template_name.strip()) == 0: continue template = "Template:" + template_name From 310a322b828d4b2ac0d39141c80e2d1e3f7f9fb3 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Mon, 2 Mar 2026 18:36:26 +0900 Subject: [PATCH 50/54] append to protect_errors --- scripts/protect_page.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/protect_page.py b/scripts/protect_page.py index 1060e9143a3..9eeb90e44d0 100644 --- a/scripts/protect_page.py +++ b/scripts/protect_page.py @@ -54,6 +54,7 @@ def protect_page(page: str, wiki: str, protect_mode: Literal["edit", "create"]): print( f"::warning::could not ({protect_mode}) protect {page} on {wiki}: {response['error']['info']}" ) + protect_errors.append(f"{protect_mode}:{wiki}:{page}") return protections = response["protect"].get("protections") for protection in protections: From ba9633b1e7e0bbd4f00f757b184e733feaf9b16d Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Mon, 2 Mar 2026 18:37:52 +0900 Subject: [PATCH 51/54] Update scripts/deploy_res.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- scripts/deploy_res.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/deploy_res.py b/scripts/deploy_res.py index 5f01b0dd5ef..bcb873d6ec8 100644 --- a/scripts/deploy_res.py +++ b/scripts/deploy_res.py @@ -53,8 +53,8 @@ def update_cache(): data={ "messagename": "Resourceloaderarticles-cacheversion", "value": subprocess.check_output( - ["git", "log", "-1", "--pretty='%h'"] - ).decode(), + ["git", "log", "-1", "--pretty=%h"] + ).decode().strip(), }, ).json() if ( From 0b78bf37b2c6227af8b15aa691c213c6e5d246f9 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Mon, 2 Mar 2026 18:39:37 +0900 Subject: [PATCH 52/54] avoid incorrect overwrite of values Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- scripts/deploy_res.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/deploy_res.py b/scripts/deploy_res.py index bcb873d6ec8..b1c00e7a840 100644 --- a/scripts/deploy_res.py +++ b/scripts/deploy_res.py @@ -85,9 +85,11 @@ def main(): for res_type, files in itertools.groupby( sorted(resource_files), lambda path: path.suffix ): - all_deployed, changes_made = deploy_resources( + group_all_deployed, group_changes_made = deploy_resources( res_type, list(files), git_deploy_reason ) + all_deployed = all_deployed and group_all_deployed + changes_made = changes_made or group_changes_made if not all_deployed: print( From 8f2de8de3f9dc2ef052ac2b850a23fce23ca8a6e Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Mon, 2 Mar 2026 18:40:41 +0900 Subject: [PATCH 53/54] lint --- scripts/deploy_res.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/deploy_res.py b/scripts/deploy_res.py index b1c00e7a840..2b30f76a840 100644 --- a/scripts/deploy_res.py +++ b/scripts/deploy_res.py @@ -52,9 +52,9 @@ def update_cache(): params={"format": "json", "action": "updatelpmwmessageapi"}, data={ "messagename": "Resourceloaderarticles-cacheversion", - "value": subprocess.check_output( - ["git", "log", "-1", "--pretty=%h"] - ).decode().strip(), + "value": subprocess.check_output(["git", "log", "-1", "--pretty=%h"]) + .decode() + .strip(), }, ).json() if ( From 6c42863b79258b0c5e8fc1c4da2d13588651d49f Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Mon, 2 Mar 2026 18:50:15 +0900 Subject: [PATCH 54/54] handle mw api error more gracefully Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- scripts/deploy_util.py | 28 ++++++++++++++++++++++++++-- scripts/remove_dev.py | 11 ++++++++++- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/scripts/deploy_util.py b/scripts/deploy_util.py index 25ec97a38e7..32552af2238 100644 --- a/scripts/deploy_util.py +++ b/scripts/deploy_util.py @@ -79,8 +79,32 @@ def deploy_file_to_wiki( "token": token, }, ).json() - result = response["edit"].get("result") - new_rev_id = response["edit"].get("newrevid") + edit_info = response.get("edit") + error_info = response.get("error") + + # Handle API errors or unexpected response structure + if error_info is not None or edit_info is None: + print(f"::warning file={str(file_path)}::failed to deploy (API error)") + details = "" + if isinstance(error_info, dict): + code = error_info.get("code") + info = error_info.get("info") + detail_parts = [] + if code: + detail_parts.append(f"code={code}") + if info: + detail_parts.append(f"info={info}") + if detail_parts: + details = " (" + ", ".join(detail_parts) + ")" + write_to_github_summary_file( + f":warning: {str(file_path)} failed to deploy due to API error{details}" + ) + deployed = False + time.sleep(SLEEP_DURATION) + return deployed, change_made + + result = edit_info.get("result") + new_rev_id = edit_info.get("newrevid") if result == "Success": if new_rev_id is not None: change_made = True diff --git a/scripts/remove_dev.py b/scripts/remove_dev.py index 816e945cb02..c8918c96652 100644 --- a/scripts/remove_dev.py +++ b/scripts/remove_dev.py @@ -58,8 +58,17 @@ def search_and_remove(wiki: str): }, ).json() time.sleep(SLEEP_DURATION) - pages = search_result["query"].get("search") + # Handle API error responses and missing or empty search results safely. + if "error" in search_result: + error_info = search_result.get("error") + print(f"::warning::search API error on {wiki}: {error_info}") + write_to_github_summary_file( + f":warning: search API error on {wiki}: {error_info}" + ) + return + + pages = search_result.get("query", {}).get("search") or [] if len(pages) == 0: return