From 1817e80d25fa62708715e114b9b89af67b59bcbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Czjwu0522=E2=80=9D?= Date: Tue, 2 Dec 2025 16:25:22 +0000 Subject: [PATCH 1/2] =?UTF-8?q?=E2=9C=A8=20feat:=20obfuscate=20GitHub=20@?= =?UTF-8?q?=20mentions=20to=20prevent=20notification=20spam?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add _obfuscate_mentions() method that replaces @username with @username_XXXX (random suffix) to prevent notifications to real GitHub users - Add safety check that prevents importing templates to public repositories - Apply obfuscation to issue bodies, PR bodies, and all comment content - Add README notice explaining the privacy requirement and changes This addresses feedback from original GitHub authors who were inadvertently notified when evaluation repositories were created as public. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- README.md | 16 +++++ .../github/github_state_manager.py | 69 +++++++++++++++++-- 2 files changed, 80 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 68ba69aa..2473a2a9 100644 --- a/README.md +++ b/README.md @@ -152,6 +152,22 @@ Tip: MCPMark supports **auto-resume**. When re-running, only unfinished tasks wi You can also follow [Quickstart](docs/quickstart.md) for the shortest end-to-end path. +### Important Notice: GitHub Repository Privacy + +> **Please ensure your evaluation repositories are set to PRIVATE.** + +GitHub state templates are now automatically downloaded from our CDN during evaluation — no manual download is required. However, because these templates contain issues and pull requests from real open-source repositories, the recreation process includes `@username` mentions of the original authors. + +**We have received feedback from original GitHub authors who were inadvertently notified** when evaluation repositories were created as public. To be a responsible member of the open-source community, we urge all users to: + +1. **Always keep evaluation repositories private** during the evaluation process. +2. **In the latest version**, we have added random suffixes to all `@username` mentions (e.g., `@user` becomes `@user_x7k2`) and implemented a safety check that prevents importing templates to public repositories. +3. **If you are using an older version of MCPMark**, please either: + - Pull the latest code immediately, or + - Manually ensure all GitHub evaluation repositories are set to private. + +Thank you for helping us maintain a respectful relationship with the open-source community. + --- ## Results and metrics diff --git a/src/mcp_services/github/github_state_manager.py b/src/mcp_services/github/github_state_manager.py index 516cbaaf..ae07f0a9 100644 --- a/src/mcp_services/github/github_state_manager.py +++ b/src/mcp_services/github/github_state_manager.py @@ -253,6 +253,21 @@ def _push_repo( html_url = resp.json()["html_url"] logger.info("| [import] Target repository created: %s", html_url) + # Safety check: Prevent importing to public repositories + # Public repos would send @ mention notifications to real users, causing spam + if not private: + error_msg = ( + "ERROR: Cannot import template to a public repository.\n\n" + "Reason: The template contains @ mentions of real GitHub users from the original\n" + "repository. Importing to a public repository would send notifications to these\n" + "users, which is disruptive and inappropriate.\n\n" + "Solution: Set private=True when calling _import_template_repo()." + ) + logger.error(error_msg) + # Clean up the created repo before raising + self._delete_repository(owner, repo_name) + raise RuntimeError(error_msg) + # Immediately disable GitHub Actions for ALL repositories to prevent any accidental triggers # We'll re-enable it later only for mcpmark-cicd logger.info( @@ -317,7 +332,7 @@ def _create_comment(issue_number: int, body: str): def _create_issue(item: dict) -> Optional[int]: data = { "title": item["title"], - "body": item.get("body", ""), + "body": self._obfuscate_mentions(item.get("body", "")), "labels": item.get("labels", []), } r = self._request_with_retry( @@ -337,7 +352,7 @@ def _create_issue(item: dict) -> Optional[int]: return new_no def _create_pull(pr_itm: dict) -> Optional[int]: - body = pr_itm.get("body", "") + body = self._obfuscate_mentions(pr_itm.get("body", "")) if pr_itm.get("is_from_fork", False): fork_note = f"\n\n---\n_This PR was originally from a fork: **{pr_itm.get('fork_owner')}/{pr_itm.get('fork_repo')}** (branch: `{pr_itm['head']}`)_" body = body + fork_note if body else fork_note[2:] @@ -366,7 +381,10 @@ def _create_pull(pr_itm: dict) -> Optional[int]: created_issues += 1 for c in itm.get("comments", []): _create_comment( - new_no, f"*Original author: @{c['user']}*\n\n{c['body']}" + new_no, + self._obfuscate_mentions( + f"*Original author: @{c['user']}*\n\n{c['body']}" + ), ) logger.info( "| [phase] Created %d out of %d issues", created_issues, len(issues_data) @@ -382,12 +400,17 @@ def _create_pull(pr_itm: dict) -> Optional[int]: created_prs += 1 for c in pr.get("comments", []): _create_comment( - new_pr_no, f"*Original author: @{c['user']}*\n\n{c['body']}" + new_pr_no, + self._obfuscate_mentions( + f"*Original author: @{c['user']}*\n\n{c['body']}" + ), ) for rc in pr.get("review_comments", []): _create_comment( new_pr_no, - f"*Original author: @{rc['user']}* (review)\n\n{rc['body']}", + self._obfuscate_mentions( + f"*Original author: @{rc['user']}* (review)\n\n{rc['body']}" + ), ) else: skipped_prs += 1 @@ -523,6 +546,42 @@ def _delete_repository(self, owner: str, repo_name: str): else: logger.info(f"| Successfully deleted repository {owner}/{repo_name}") + def _obfuscate_mentions(self, text: str) -> str: + """ + Obfuscate @ mentions to prevent notifications to real users. + + Replaces @username with @username_XXXX (random suffix) to ensure the mentioned + user does not exist on GitHub. This prevents notification spam when importing + templates that contain @ mentions from original repositories. + + Args: + text: The text content that may contain @ mentions + + Returns: + Text with obfuscated @ mentions + """ + import re + import random + import string + + if not text: + return text + + # Pattern matches @username (GitHub usernames: alphanumeric, hyphens, max 39 chars) + # Negative lookbehind (? Date: Wed, 3 Dec 2025 15:21:18 +0000 Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=93=9D=20docs:=20add=20News=20section?= =?UTF-8?q?=20to=20README?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 2473a2a9..c8f655ac 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,18 @@ MCPMark provides a reproducible, extensible benchmark for researchers and engine [![MCPMark](https://github.com/user-attachments/assets/dfc06a41-e387-45e3-bc98-db7097ffa3dc)](https://mcpmark.ai) +## News + +- **02/Dec/2025** - Evaluated `gemini-3-pro-preview` (thinking: low): Pass@1 50.6% ± 2.3%, Pass@4 67.7%, Pass^4 31.5% - so close to `gpt-5-high` (51.6%)! Also `deepseek-v3.2-thinking` Pass@1 36.8% ± 1.8%, Pass@4 51.2%, Pass^4 21.3% and `deepseek-v3.2-chat` Pass@1 29.7% ± 1.5%, Pass@4 46.5%, Pass^4 13.4% +- **02/Dec/2025** - Obfuscate GitHub @mentions to prevent notification spam during evaluation ([#229](https://github.com/eval-sys/mcpmark/pull/229)) +- **01/Dec/2025** - DeepSeek v3.2 release uses MCPMark! Kudos to the DeepSeek team on securing the best open-source model with a significant performance gain. [X Post](https://x.com/deepseek_ai/status/1995452650557763728) | [Technical Report](https://cas-bridge.xethub.hf.co/xet-bridge-us/692cfec93b25b81d09307b94/2d0aa38511b9df084d12a00fe04a96595496af772cb766c516c4e6aee1e21246?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=cas%2F20251203%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20251203T145756Z&X-Amz-Expires=3600&X-Amz-Signature=31d39c39a42319dba189c1164f8a8bff69e4211b7520b75b7f3d4013a23b3022&X-Amz-SignedHeaders=host&X-Xet-Cas-Uid=634c72e6fe1bfa967d6c2b5c&response-content-disposition=inline%3B+filename*%3DUTF-8%27%27paper.pdf%3B+filename%3D%22paper.pdf%22%3B&response-content-type=application%2Fpdf&x-id=GetObject&Expires=1764777476&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTc2NDc3NzQ3Nn19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2FzLWJyaWRnZS54ZXRodWIuaGYuY28veGV0LWJyaWRnZS11cy82OTJjZmVjOTNiMjViODFkMDkzMDdiOTQvMmQwYWEzODUxMWI5ZGYwODRkMTJhMDBmZTA0YTk2NTk1NDk2YWY3NzJjYjc2NmM1MTZjNGU2YWVlMWUyMTI0NioifV19&Signature=HxFnQM7j%7EnuD9Qr81qqbXkunCc4nLLmTHv-5EosJu8EqlQ3VRyBibLNz0ur1d9h2SFp1Lvji3tNOQSWZsW%7EMS6wbmN5E4jjgbcXR40oxG4nhcq8Hy5jnHlEcQ9GyV9B0HTeXmQJ32AjkEDymEl9iVISRzEzwiu9J8wQL659QHSU5v81eexEk7LTfETikOdKCUQJy0uNqdDb3N%7Elfegq6XrxuZU5UawtlJYV57g1afkLln0ZYxqkYSEqxRdGwIAbfd1Te2Yi60I%7ELEB3qok4LM2%7E4gBWDBaB%7ESN902sbutiQYuvk6V5tFlSVq3MHaRJfJBCMTZiNtb5JAHKZSyVlGuw__&Key-Pair-Id=K2L8F4GPSG1IFC) +- **17/Nov/2025** - Added 50 easy tasks (10 per MCP server) for smaller (<100B) open-source models ([#225](https://github.com/eval-sys/mcpmark/pull/225)) +- **31/Oct/2025** - Community PR from insforge: better MCP servers achieve better results with fewer tokens! ([#214](https://github.com/eval-sys/mcpmark/pull/214)) +- **13/Oct/2025** - Added ReAct agent support. PRs for new agent scaffolds welcome! ([#209](https://github.com/eval-sys/mcpmark/pull/209)) +- **10/Sep/2025** - `qwen-3-coder-plus` is the best open-source model! Kudos to Qwen team. [X Post](https://x.com/Alibaba_Qwen/status/1965457023438651532) + +--- + ## What you can do with MCPMark - **Evaluate real tool usage** across multiple MCP services: `Notion`, `GitHub`, `Filesystem`, `Postgres`, `Playwright`.