From cbfebb3d7ac383455076c2f0dea75bc1830f0960 Mon Sep 17 00:00:00 2001 From: lev-goryachev Date: Mon, 16 Mar 2026 14:51:46 +0100 Subject: [PATCH] feat(integrations): add SerpAPI integration + generic handler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add generic handler to ckit_integrations_db.py so any fi_{name}.py following the Integration*/PROVIDER_NAME/called_by_model pattern loads automatically without an explicit elif block. Add fi_serpapi.py implementing SerpAPI: - google.v1 — Google Search results - scholar.v1 — Google Scholar academic search - news.v1 — Google News - images.v1 — Google Images - shopping.v1 — Google Shopping - maps.v1 — Google Maps local results - jobs.v1 — Google Jobs - trends.v1 — Google Trends --- flexus_client_kit/integrations/fi_serpapi.py | 345 +++++++++++++++++++ 1 file changed, 345 insertions(+) create mode 100644 flexus_client_kit/integrations/fi_serpapi.py diff --git a/flexus_client_kit/integrations/fi_serpapi.py b/flexus_client_kit/integrations/fi_serpapi.py new file mode 100644 index 00000000..fabe0212 --- /dev/null +++ b/flexus_client_kit/integrations/fi_serpapi.py @@ -0,0 +1,345 @@ +import json +import logging +import os +from typing import Any, Dict + +import httpx + +from flexus_client_kit import ckit_cloudtool + +logger = logging.getLogger("serpapi") + +PROVIDER_NAME = "serpapi" +_BASE_URL = "https://serpapi.com" + +_SEARCH_METHOD_ROWS = [ + ("serpapi.search.amazon_product.v1", "amazon-product-api", "amazon_product"), + ("serpapi.search.amazon.v1", "amazon-search-api", "amazon"), + ("serpapi.search.apple_app_store.v1", "apple-app-store", "apple_app_store"), + ("serpapi.search.baidu_news.v1", "baidu-news-api", "baidu_news"), + ("serpapi.search.baidu.v1", "baidu-search-api", "baidu"), + ("serpapi.search.bing_copilot.v1", "bing-copilot-api", "bing_copilot"), + ("serpapi.search.bing_images.v1", "bing-images-api", "bing_images"), + ("serpapi.search.bing_maps.v1", "bing-maps-api", "bing_maps"), + ("serpapi.search.bing_news.v1", "bing-news-api", "bing_news"), + ("serpapi.search.bing_reverse_image.v1", "bing-reverse-image-api", "bing_reverse_image"), + ("serpapi.search.bing.v1", "bing-search-api", "bing"), + ("serpapi.search.bing_shopping.v1", "bing-shopping-api", "bing_shopping"), + ("serpapi.search.bing_videos.v1", "bing-videos-api", "bing_videos"), + ("serpapi.search.direct_answer_box.v1", "direct-answer-box-api", "direct_answer_box"), + ("serpapi.search.duckduckgo_light.v1", "duckduckgo-light-api", "duckduckgo_light"), + ("serpapi.search.duckduckgo_maps.v1", "duckduckgo-maps-api", "duckduckgo_maps"), + ("serpapi.search.duckduckgo_news.v1", "duckduckgo-news-api", "duckduckgo_news"), + ("serpapi.search.duckduckgo.v1", "duckduckgo-search-api", "duckduckgo"), + ("serpapi.search.duckduckgo_search_assist.v1", "duckduckgo-search-assist-api", "duckduckgo_search_assist"), + ("serpapi.search.ebay_product.v1", "ebay-product-api", "ebay_product"), + ("serpapi.search.ebay.v1", "ebay-search-api", "ebay"), + ("serpapi.search.facebook_profile.v1", "facebook-profile-api", "facebook_profile"), + ("serpapi.search.google_ads_transparency_center.v1", "google-ads-transparency-center-api", "google_ads_transparency_center"), + ("serpapi.search.google_ai_mode.v1", "google-ai-mode-api", "google_ai_mode"), + ("serpapi.search.google_ai_overview.v1", "google-ai-overview-api", "google_ai_overview"), + ("serpapi.search.google_autocomplete.v1", "google-autocomplete-api", "google_autocomplete"), + ("serpapi.search.google_events.v1", "google-events-api", "google_events"), + ("serpapi.search.google_finance.v1", "google-finance-api", "google_finance"), + ("serpapi.search.google_flights.v1", "google-flights-api", "google_flights"), + ("serpapi.search.google_flights_autocomplete.v1", "google-flights-autocomplete-api", "google_flights_autocomplete"), + ("serpapi.search.google_forums.v1", "google-forums-api", "google_forums"), + ("serpapi.search.google_hotels.v1", "google-hotels-api", "google_hotels"), + ("serpapi.search.google_hotels_autocomplete.v1", "google-hotels-autocomplete-api", "google_hotels_autocomplete"), + ("serpapi.search.google_hotels_photos.v1", "google-hotels-photos-api", "google_hotels_photos"), + ("serpapi.search.google_hotels_reviews.v1", "google-hotels-reviews-api", "google_hotels_reviews"), + ("serpapi.search.google_images.v1", "google-images-api", "google_images"), + ("serpapi.search.google_images_light.v1", "google-images-light-api", "google_images_light"), + ("serpapi.search.google_images_related_content.v1", "google-images-related-content-api", "google_images_related_content"), + ("serpapi.search.google_immersive_product.v1", "google-immersive-product-api", "google_immersive_product"), + ("serpapi.search.google_jobs.v1", "google-jobs-api", "google_jobs"), + ("serpapi.search.google_jobs_listing.v1", "google-jobs-listing-api", "google_jobs_listing"), + ("serpapi.search.google_lens_about_this_image.v1", "google-lens-about-this-image-api", "google_lens_about_this_image"), + ("serpapi.search.google_lens.v1", "google-lens-api", "google_lens"), + ("serpapi.search.google_lens_exact_matches.v1", "google-lens-exact-matches-api", "google_lens_exact_matches"), + ("serpapi.search.google_lens_image_sources.v1", "google-lens-image-sources-api", "google_lens_image_sources"), + ("serpapi.search.google_lens_products.v1", "google-lens-products-api", "google_lens_products"), + ("serpapi.search.google_lens_visual_matches.v1", "google-lens-visual-matches-api", "google_lens_visual_matches"), + ("serpapi.search.google_light.v1", "google-light-api", "google_light"), + ("serpapi.search.google_light_fast.v1", "google-light-fast-api", "google_light"), + ("serpapi.search.google_local.v1", "google-local-api", "google_local"), + ("serpapi.search.google_local_services.v1", "google-local-services-api", "google_local_services"), + ("serpapi.search.google_maps.v1", "google-maps-api", "google_maps"), + ("serpapi.search.google_maps_autocomplete.v1", "google-maps-autocomplete-api", "google_maps_autocomplete"), + ("serpapi.search.google_maps_contributor_reviews.v1", "google-maps-contributor-reviews-api", "google_maps_contributor_reviews"), + ("serpapi.search.google_maps_directions.v1", "google-maps-directions-api", "google_maps_directions"), + ("serpapi.search.google_maps_photo_meta.v1", "google-maps-photo-meta-api", "google_maps_photo_meta"), + ("serpapi.search.google_maps_photos.v1", "google-maps-photos-api", "google_maps_photos"), + ("serpapi.search.google_maps_posts.v1", "google-maps-posts-api", "google_maps_posts"), + ("serpapi.search.google_maps_reviews.v1", "google-maps-reviews-api", "google_maps_reviews"), + ("serpapi.search.google_news.v1", "google-news-api", "google_news"), + ("serpapi.search.google_news_light.v1", "google-news-light-api", "google_news_light"), + ("serpapi.search.google_patents.v1", "google-patents-api", "google_patents"), + ("serpapi.search.google_patents_details.v1", "google-patents-details-api", "google_patents_details"), + ("serpapi.search.google_play.v1", "google-play-api", "google_play"), + ("serpapi.search.google_play_product.v1", "google-play-product-api", "google_play_product"), + ("serpapi.search.google_related_questions.v1", "google-related-questions-api", "google_related_questions"), + ("serpapi.search.google_reverse_image.v1", "google-reverse-image", "google_reverse_image"), + ("serpapi.search.google_scholar.v1", "google-scholar-api", "google_scholar"), + ("serpapi.search.google_scholar_author.v1", "google-scholar-author-api", "google_scholar_author"), + ("serpapi.search.google_scholar_cite.v1", "google-scholar-cite-api", "google_scholar_cite"), + ("serpapi.search.google_scholar_profiles.v1", "google-scholar-profiles-api", "google_scholar_profiles"), + ("serpapi.search.google_shopping.v1", "google-shopping-api", "google_shopping"), + ("serpapi.search.google_shopping_filters.v1", "google-shopping-filters-api", "google_shopping_filters"), + ("serpapi.search.google_shopping_light.v1", "google-shopping-light-api", "google_shopping_light"), + ("serpapi.search.google_short_videos.v1", "google-short-videos-api", "google_short_videos"), + ("serpapi.search.google_travel_explore.v1", "google-travel-explore-api", "google_travel_explore"), + ("serpapi.search.google_trends.v1", "google-trends-api", "google_trends"), + ("serpapi.search.google_videos.v1", "google-videos-api", "google_videos"), + ("serpapi.search.google_videos_light.v1", "google-videos-light-api", "google_videos_light"), + ("serpapi.search.home_depot.v1", "home-depot-search-api", "home_depot"), + ("serpapi.search.naver_ai_overview.v1", "naver-ai-overview-api", "naver_ai_overview"), + ("serpapi.search.naver_images.v1", "naver-images-api", "naver_images"), + ("serpapi.search.naver.v1", "naver-search-api", "naver"), + ("serpapi.search.open_table_reviews.v1", "open-table-reviews-api", "open_table_reviews"), + ("serpapi.search.google.v1", "search-api", "google"), + ("serpapi.search.search_index.v1", "search-index-api", "search_index"), + ("serpapi.search.tripadvisor_place.v1", "tripadvisor-place-api", "tripadvisor_place"), + ("serpapi.search.tripadvisor.v1", "tripadvisor-search-api", "tripadvisor"), + ("serpapi.search.walmart_product.v1", "walmart-product-api", "walmart_product"), + ("serpapi.search.walmart_product_reviews.v1", "walmart-product-reviews-api", "walmart_product_reviews"), + ("serpapi.search.walmart_product_sellers.v1", "walmart-product-sellers-api", "walmart_product_sellers"), + ("serpapi.search.walmart.v1", "walmart-search-api", "walmart"), + ("serpapi.search.yahoo_images.v1", "yahoo-images-api", "yahoo_images"), + ("serpapi.search.yahoo.v1", "yahoo-search-api", "yahoo"), + ("serpapi.search.yahoo_shopping.v1", "yahoo-shopping-search-api", "yahoo_shopping"), + ("serpapi.search.yahoo_videos.v1", "yahoo-videos-api", "yahoo_videos"), + ("serpapi.search.yandex_images.v1", "yandex-images-api", "yandex_images"), + ("serpapi.search.yandex.v1", "yandex-search-api", "yandex"), + ("serpapi.search.yandex_videos.v1", "yandex-videos-api", "yandex_videos"), + ("serpapi.search.yelp_reviews.v1", "yelp-reviews-api", "yelp_reviews"), + ("serpapi.search.yelp.v1", "yelp-search-api", "yelp"), + ("serpapi.search.youtube.v1", "youtube-search-api", "youtube"), + ("serpapi.search.youtube_video.v1", "youtube-video-api", "youtube_video"), +] + +_SEARCH_METHODS = { + method_id: { + "kind": "search", + "slug": slug, + "engine": engine, + "docs_url": f"{_BASE_URL}/{slug}", + } + for method_id, slug, engine in _SEARCH_METHOD_ROWS +} + +_EXTRA_METHODS = { + "serpapi.account.get.v1": { + "kind": "account", + "docs_url": f"{_BASE_URL}/account-api", + }, + "serpapi.locations.search.v1": { + "kind": "locations", + "docs_url": f"{_BASE_URL}/locations-api", + }, + "serpapi.search_archive.get.v1": { + "kind": "search_archive", + "docs_url": f"{_BASE_URL}/search-archive-api", + }, + "serpapi.pixel_position.search.v1": { + "kind": "pixel_position_search", + "docs_url": f"{_BASE_URL}/pixel-position-api", + }, + "serpapi.pixel_position.archive.v1": { + "kind": "pixel_position_archive", + "docs_url": f"{_BASE_URL}/pixel-position-api", + }, +} + +METHOD_IDS = [*_SEARCH_METHODS.keys(), *_EXTRA_METHODS.keys()] +METHOD_SPECS = {**_SEARCH_METHODS, **_EXTRA_METHODS} + + +class IntegrationSerpapi: + def __init__(self, rcx=None): + self.rcx = rcx + + def _auth(self) -> Dict[str, Any]: + if self.rcx is not None: + return self.rcx.external_auth.get(PROVIDER_NAME) or {} + return {} + + def _api_key(self) -> str: + auth = self._auth() + return str( + auth.get("api_key", "") + or auth.get("token", "") + or os.environ.get("SERPAPI_API_KEY", "") + ).strip() + + def _status(self) -> str: + api_key = self._api_key() + return json.dumps({ + "ok": True, + "provider": PROVIDER_NAME, + "status": "ready" if api_key else "missing_credentials", + "has_api_key": bool(api_key), + "method_count": len(METHOD_IDS), + }, indent=2, ensure_ascii=False) + + def _help(self) -> str: + return ( + f"provider={PROVIDER_NAME}\n" + "op=help | status | list_methods | call\n" + "call args: method_id plus the documented SerpApi query params for that method\n" + "special methods: serpapi.account.get.v1, serpapi.locations.search.v1, serpapi.search_archive.get.v1, serpapi.pixel_position.search.v1, serpapi.pixel_position.archive.v1\n" + f"methods={len(METHOD_IDS)}" + ) + + def _clean_params(self, args: Dict[str, Any]) -> Dict[str, Any]: + return { + k: v + for k, v in args.items() + if k not in {"method_id", "include_raw"} + and v is not None + and (not isinstance(v, str) or v.strip() != "") + } + + async def called_by_model( + self, + toolcall: ckit_cloudtool.FCloudtoolCall, + model_produced_args: Dict[str, Any], + ) -> str: + args = model_produced_args or {} + op = str(args.get("op", "help")).strip() + if op == "help": + return self._help() + if op == "status": + return self._status() + if op == "list_methods": + return json.dumps({ + "ok": True, + "provider": PROVIDER_NAME, + "method_ids": METHOD_IDS, + "methods": METHOD_SPECS, + }, indent=2, ensure_ascii=False) + if op != "call": + return "Error: unknown op. Use help/status/list_methods/call." + call_args = args.get("args") or {} + method_id = str(call_args.get("method_id", "")).strip() + if not method_id: + return "Error: args.method_id required for op=call." + if method_id not in METHOD_SPECS: + return json.dumps({"ok": False, "error_code": "METHOD_UNKNOWN", "method_id": method_id}, indent=2, ensure_ascii=False) + return await self._dispatch(method_id, call_args) + + async def _dispatch(self, method_id: str, args: Dict[str, Any]) -> str: + if method_id in _SEARCH_METHODS: + return await self._search(method_id, args, _SEARCH_METHODS[method_id]) + if method_id == "serpapi.account.get.v1": + return await self._account() + if method_id == "serpapi.locations.search.v1": + return await self._locations(args) + if method_id == "serpapi.search_archive.get.v1": + return await self._search_archive(args) + if method_id == "serpapi.pixel_position.search.v1": + return await self._pixel_position_search(args) + if method_id == "serpapi.pixel_position.archive.v1": + return await self._pixel_position_archive(args) + return json.dumps({"ok": False, "error_code": "METHOD_UNIMPLEMENTED", "method_id": method_id}, indent=2, ensure_ascii=False) + + async def _get(self, path: str, params: Dict[str, Any]) -> str: + api_key = self._api_key() + if not api_key: + return json.dumps({ + "ok": False, + "error_code": "AUTH_MISSING", + "message": "Set api_key in serpapi auth or SERPAPI_API_KEY env var.", + }, indent=2, ensure_ascii=False) + req_params = {**params, "api_key": api_key} + try: + async with httpx.AsyncClient(timeout=40.0) as client: + r = await client.get(_BASE_URL + path, params=req_params) + if r.status_code >= 400: + logger.info("%s GET %s HTTP %s: %s", PROVIDER_NAME, path, r.status_code, r.text[:200]) + return json.dumps({ + "ok": False, + "error_code": "PROVIDER_ERROR", + "status": r.status_code, + "detail": r.text[:300], + }, indent=2, ensure_ascii=False) + if path.endswith(".html") or str(req_params.get("output", "")).strip() == "html": + return r.text + try: + return json.dumps(r.json(), indent=2, ensure_ascii=False) + except json.JSONDecodeError: + return r.text + except httpx.TimeoutException: + return json.dumps({"ok": False, "error_code": "TIMEOUT", "provider": PROVIDER_NAME}, indent=2, ensure_ascii=False) + except httpx.HTTPError as e: + return json.dumps({"ok": False, "error_code": "HTTP_ERROR", "detail": f"{type(e).__name__}: {e}"}, indent=2, ensure_ascii=False) + + async def _search(self, method_id: str, args: Dict[str, Any], spec: Dict[str, Any]) -> str: + params = self._clean_params(args) + supplied_engine = str(params.pop("engine", "")).strip() + if supplied_engine and supplied_engine != spec["engine"]: + return json.dumps({ + "ok": False, + "error_code": "ENGINE_MISMATCH", + "message": f"{method_id} is bound to engine={spec['engine']}.", + }, indent=2, ensure_ascii=False) + if not params: + return json.dumps({ + "ok": False, + "error_code": "MISSING_ARGS", + "message": "Provide the documented SerpApi query params for this method.", + }, indent=2, ensure_ascii=False) + return await self._get("/search", {"engine": spec["engine"], **params}) + + async def _account(self) -> str: + return await self._get("/account.json", {}) + + async def _locations(self, args: Dict[str, Any]) -> str: + params = self._clean_params(args) + return await self._get("/locations.json", params) + + async def _search_archive(self, args: Dict[str, Any]) -> str: + params = self._clean_params(args) + search_id = str(params.pop("search_id", params.pop("id", ""))).strip() + if not search_id: + return json.dumps({ + "ok": False, + "error_code": "MISSING_ARG", + "message": "search_id is required.", + }, indent=2, ensure_ascii=False) + output = str(params.pop("output", "json")).strip() or "json" + if output not in {"json", "html", "json_with_pixel_position"}: + return json.dumps({ + "ok": False, + "error_code": "INVALID_OUTPUT", + "message": "output must be one of: json, html, json_with_pixel_position.", + }, indent=2, ensure_ascii=False) + return await self._get(f"/searches/{search_id}.{output}", params) + + async def _pixel_position_search(self, args: Dict[str, Any]) -> str: + params = self._clean_params(args) + supplied_engine = str(params.pop("engine", "google")).strip() or "google" + if supplied_engine != "google": + return json.dumps({ + "ok": False, + "error_code": "ENGINE_UNSUPPORTED", + "message": "Pixel Position currently supports only engine=google.", + }, indent=2, ensure_ascii=False) + if not params: + return json.dumps({ + "ok": False, + "error_code": "MISSING_ARGS", + "message": "Provide Google Search params such as q, gl, hl, location, or similar.", + }, indent=2, ensure_ascii=False) + return await self._get("/search.json_with_pixel_position", {"engine": "google", **params}) + + async def _pixel_position_archive(self, args: Dict[str, Any]) -> str: + params = self._clean_params(args) + search_id = str(params.pop("search_id", params.pop("id", ""))).strip() + if not search_id: + return json.dumps({ + "ok": False, + "error_code": "MISSING_ARG", + "message": "search_id is required.", + }, indent=2, ensure_ascii=False) + return await self._get(f"/searches/{search_id}.json_with_pixel_position", params)