From a56b424cbdab71331c39fbfd70bdce43f4c74bad Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Wed, 25 Mar 2026 13:33:20 +0100 Subject: [PATCH 1/5] Add llms.txt and llms-full.txt generation --- .github/workflows/main.yml | 5 +++ .pre-commit-config.yaml | 5 +++ .readthedocs.yml | 18 ++++------ docs/Makefile | 21 ------------ docs/conf.py | 8 +---- docs/make.bat | 36 -------------------- docs/quickstart.rst | 2 +- docs/requirements.in | 2 ++ docs/requirements.txt | 70 ++++++++++++++++++++++++++++++++++++-- pyproject.toml | 3 ++ tox.ini | 10 ++---- 11 files changed, 94 insertions(+), 86 deletions(-) create mode 100644 .pre-commit-config.yaml delete mode 100644 docs/Makefile delete mode 100644 docs/make.bat create mode 100644 docs/requirements.in create mode 100644 pyproject.toml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f4f2a6c4..c4b86e6b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -43,6 +43,11 @@ jobs: - python-version: "3.13" env: TOXENV: "json" + # Keep in sync with pyproject.toml's + # tool.sphinx-scrapy.python-version. + - python-version: "3.14" + env: + TOXENV: "docs" # https://github.com/kevin1024/vcrpy/issues/885 # - python-version: "pypy3.11-v7.3.20" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..d41921eb --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,5 @@ +repos: +- repo: https://github.com/scrapy/sphinx-scrapy + rev: 999017c29b8bc213843e86e0e350a85c47493e34 + hooks: + - id: sphinx-scrapy diff --git a/.readthedocs.yml b/.readthedocs.yml index f9a5741e..3ea6996b 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,15 +1,9 @@ version: 2 -formats: all -sphinx: - configuration: docs/conf.py - fail_on_warning: true build: - os: ubuntu-22.04 + os: ubuntu-24.04 tools: - # For available versions, see: - # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python - python: "3.10" # Keep in sync with .github/workflows/main.yml -python: - install: - - requirements: docs/requirements.txt - - path: . + python: "3.14" + commands: + - pip install tox + - tox -e docs + - cp -a docs/_build/all/. $READTHEDOCS_OUTPUT/html/ diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index c44acee0..00000000 --- a/docs/Makefile +++ /dev/null @@ -1,21 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -SPHINXPROJ = python-scrapinghub -SPHINXAPIDOCS = sphinx-apidoc -SOURCEDIR = . -BUILDDIR = _build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/conf.py b/docs/conf.py index c3b9b3f6..61181de9 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -39,7 +39,7 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['sphinx.ext.autodoc'] +extensions = ['sphinx_scrapy'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -160,9 +160,3 @@ author, 'python-scrapinghub', 'One line description of project.', 'Miscellaneous'), ] - -# Set Sphinx Read The Docs theme -import sphinx_rtd_theme - -html_theme = 'sphinx_rtd_theme' -html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] diff --git a/docs/make.bat b/docs/make.bat deleted file mode 100644 index f7dfb3e2..00000000 --- a/docs/make.bat +++ /dev/null @@ -1,36 +0,0 @@ -@ECHO OFF - -pushd %~dp0 - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set SOURCEDIR=. -set BUILDDIR=_build -set SPHINXPROJ=python-scrapinghub - -if "%1" == "" goto help - -%SPHINXBUILD% >NUL 2>NUL -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - exit /b 1 -) - -%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% -goto end - -:help -%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% - -:end -popd diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 45f9807e..8512bc20 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -4,7 +4,7 @@ Quickstart Requirements ------------ -* Python 3.8+ +* Python 3.9+ Installation diff --git a/docs/requirements.in b/docs/requirements.in new file mode 100644 index 00000000..fd365b3c --- /dev/null +++ b/docs/requirements.in @@ -0,0 +1,2 @@ +sphinx-rtd-theme +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@999017c29b8bc213843e86e0e350a85c47493e34 diff --git a/docs/requirements.txt b/docs/requirements.txt index d5476d84..600984e2 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,68 @@ -sphinx==7.2.6 -sphinx-rtd-theme==2.0.0 +# This file was autogenerated by uv via the following command: +# uv pip compile requirements.in +alabaster==1.0.0 + # via sphinx +babel==2.18.0 + # via sphinx +certifi==2026.2.25 + # via requests +charset-normalizer==3.4.6 + # via requests +docutils==0.22.4 + # via + # sphinx + # sphinx-markdown-builder + # sphinx-rtd-theme +idna==3.11 + # via requests +imagesize==2.0.0 + # via sphinx +jinja2==3.1.6 + # via sphinx +markupsafe==3.0.3 + # via jinja2 +packaging==26.0 + # via + # sphinx + # sphinx-scrapy +pygments==2.19.2 + # via sphinx +requests==2.32.5 + # via sphinx +roman-numerals==4.1.0 + # via sphinx +snowballstemmer==3.0.1 + # via sphinx +sphinx==9.1.0 + # via + # sphinx-llms-txt + # sphinx-markdown-builder + # sphinx-rtd-theme + # sphinx-scrapy + # sphinxcontrib-jquery +sphinx-llms-txt==0.7.1 + # via sphinx-scrapy +sphinx-markdown-builder @ git+https://github.com/AdrianAtZyte/sphinx-markdown-builder.git@2050ffaed43fcc523e03cc7a677cbcb5f5f6d30b + # via sphinx-scrapy +sphinx-rtd-theme==3.1.0 + # via -r requirements.in +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@999017c29b8bc213843e86e0e350a85c47493e34 + # via -r requirements.in +sphinxcontrib-applehelp==2.0.0 + # via sphinx +sphinxcontrib-devhelp==2.0.0 + # via sphinx +sphinxcontrib-htmlhelp==2.1.0 + # via sphinx +sphinxcontrib-jquery==4.1 + # via sphinx-rtd-theme +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==2.0.0 + # via sphinx +sphinxcontrib-serializinghtml==2.0.0 + # via sphinx +tabulate==0.10.0 + # via sphinx-markdown-builder +urllib3==2.6.3 + # via requests diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..e9210475 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[tool.sphinx-scrapy] +# Keep in sync with .github/workflows/main.yml +python-version = "3.14" diff --git a/tox.ini b/tox.ini index 6498dd88..c3232e66 100644 --- a/tox.ini +++ b/tox.ini @@ -4,8 +4,10 @@ # and then run "tox" from this directory. [tox] +requires = + sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@999017c29b8bc213843e86e0e350a85c47493e34 # pypy3 excluded due to https://github.com/kevin1024/vcrpy/issues/885 -envlist = py{39,310,311,312}-{json,msgpack} +envlist = py{39,310,311,312}-{json,msgpack},docs [testenv] deps = @@ -13,9 +15,3 @@ deps = msgpack: -r tests/requirements-msgpack.txt commands = py.test --cov=scrapinghub --cov-report=xml {posargs: scrapinghub tests} -[testenv:docs] -changedir = docs -deps = - -r docs/requirements.txt -commands = - sphinx-build -W -b html . {envtmpdir}/html From 4629565ff1db0ea53fde9fe1e24912043d8fc1d5 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Wed, 25 Mar 2026 13:38:00 +0100 Subject: [PATCH 2/5] Drop Python 3.9 support --- .github/workflows/main.yml | 6 ------ README.rst | 2 +- docs/quickstart.rst | 2 +- setup.py | 3 +-- tox.ini | 2 +- 5 files changed, 4 insertions(+), 11 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c4b86e6b..5bd464b0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,12 +13,6 @@ jobs: fail-fast: false matrix: include: - - python-version: "3.9" - env: - TOXENV: "msgpack" - - python-version: "3.9" - env: - TOXENV: "json" - python-version: "3.10" env: TOXENV: "msgpack" diff --git a/README.rst b/README.rst index c2d58795..92c4f12c 100644 --- a/README.rst +++ b/README.rst @@ -14,7 +14,7 @@ Client interface for Scrapinghub API .. image:: https://codecov.io/gh/scrapinghub/python-scrapinghub/branch/master/graph/badge.svg :target: https://app.codecov.io/gh/scrapinghub/python-scrapinghub -The ``scrapinghub`` is a Python 3.9+ library for communicating with the +The ``scrapinghub`` is a Python 3.10+ library for communicating with the `Scrapinghub API`_. diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 8512bc20..278161e9 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -4,7 +4,7 @@ Quickstart Requirements ------------ -* Python 3.9+ +* Python 3.10+ Installation diff --git a/setup.py b/setup.py index 51ee111b..8f73b841 100644 --- a/setup.py +++ b/setup.py @@ -28,13 +28,12 @@ package_data={'scrapinghub': ['VERSION']}, install_requires=['requests>=1.0', 'retrying>=1.3.3', 'six>=1.10.0'], extras_require={'msgpack': mpack_required}, - python_requires='>=3.9', + python_requires='>=3.10', classifiers=[ 'Development Status :: 5 - Production/Stable', 'License :: OSI Approved :: BSD License', 'Operating System :: OS Independent', 'Programming Language :: Python', - 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: 3.12', diff --git a/tox.ini b/tox.ini index c3232e66..d68666c8 100644 --- a/tox.ini +++ b/tox.ini @@ -7,7 +7,7 @@ requires = sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@999017c29b8bc213843e86e0e350a85c47493e34 # pypy3 excluded due to https://github.com/kevin1024/vcrpy/issues/885 -envlist = py{39,310,311,312}-{json,msgpack},docs +envlist = py{310,311,312}-{json,msgpack},docs [testenv] deps = From 37ddd5746cc4becb873761724b3cef50b812238e Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Mon, 6 Apr 2026 11:45:50 +0200 Subject: [PATCH 3/5] Use sphinx-scrapy 0.7.0 --- docs/requirements.in | 2 +- docs/requirements.txt | 11 +++++++---- tox.ini | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/requirements.in b/docs/requirements.in index fd365b3c..9bb18eef 100644 --- a/docs/requirements.in +++ b/docs/requirements.in @@ -1,2 +1,2 @@ sphinx-rtd-theme -sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@999017c29b8bc213843e86e0e350a85c47493e34 +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.7.0 diff --git a/docs/requirements.txt b/docs/requirements.txt index 600984e2..015da52c 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile requirements.in +# uv pip compile requirements.in -o requirements.txt alabaster==1.0.0 # via sphinx babel==2.18.0 @@ -35,18 +35,21 @@ snowballstemmer==3.0.1 # via sphinx sphinx==9.1.0 # via + # sphinx-copybutton # sphinx-llms-txt # sphinx-markdown-builder # sphinx-rtd-theme # sphinx-scrapy # sphinxcontrib-jquery -sphinx-llms-txt==0.7.1 +sphinx-copybutton==0.5.2 # via sphinx-scrapy -sphinx-markdown-builder @ git+https://github.com/AdrianAtZyte/sphinx-markdown-builder.git@2050ffaed43fcc523e03cc7a677cbcb5f5f6d30b +sphinx-llms-txt @ git+https://github.com/zytedata/sphinx-llms-txt.git@5e8866cb0cc249aa2017ad9050b3b83a7ca16f69 + # via sphinx-scrapy +sphinx-markdown-builder @ git+https://github.com/zytedata/sphinx-markdown-builder.git@ac9f8babfe622e4300099ab44b96d9d9228e742e # via sphinx-scrapy sphinx-rtd-theme==3.1.0 # via -r requirements.in -sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@999017c29b8bc213843e86e0e350a85c47493e34 +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@46c52fc3a6d5ee632a8ef6bccf4daa1ff53e96fd # via -r requirements.in sphinxcontrib-applehelp==2.0.0 # via sphinx diff --git a/tox.ini b/tox.ini index d68666c8..c184e860 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,7 @@ [tox] requires = - sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@999017c29b8bc213843e86e0e350a85c47493e34 + sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.7.0 # pypy3 excluded due to https://github.com/kevin1024/vcrpy/issues/885 envlist = py{310,311,312}-{json,msgpack},docs From 1112055661e4920c1445a915b397046aed8cc119 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Mon, 6 Apr 2026 11:47:21 +0200 Subject: [PATCH 4/5] Complete sphinx-scrapy 0.7.0 upgrade --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d41921eb..1f4ae527 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,5 @@ repos: - repo: https://github.com/scrapy/sphinx-scrapy - rev: 999017c29b8bc213843e86e0e350a85c47493e34 + rev: 0.7.0 hooks: - id: sphinx-scrapy From 15b8ea1b21d21dab0b5ce9371e10e39f063e0c27 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Mon, 6 Apr 2026 12:34:40 +0200 Subject: [PATCH 5/5] Use sphinx-scrapy 0.7.1 --- .pre-commit-config.yaml | 2 +- .readthedocs.yml | 1 + docs/requirements.in | 2 +- docs/requirements.txt | 2 +- tox.ini | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1f4ae527..53802d64 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,5 @@ repos: - repo: https://github.com/scrapy/sphinx-scrapy - rev: 0.7.0 + rev: 0.7.1 hooks: - id: sphinx-scrapy diff --git a/.readthedocs.yml b/.readthedocs.yml index 3ea6996b..a2773dcf 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -6,4 +6,5 @@ build: commands: - pip install tox - tox -e docs + - mkdir -p $READTHEDOCS_OUTPUT/html - cp -a docs/_build/all/. $READTHEDOCS_OUTPUT/html/ diff --git a/docs/requirements.in b/docs/requirements.in index 9bb18eef..a97eaea5 100644 --- a/docs/requirements.in +++ b/docs/requirements.in @@ -1,2 +1,2 @@ sphinx-rtd-theme -sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.7.0 +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.7.1 diff --git a/docs/requirements.txt b/docs/requirements.txt index 015da52c..8dda1b27 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -49,7 +49,7 @@ sphinx-markdown-builder @ git+https://github.com/zytedata/sphinx-markdown-builde # via sphinx-scrapy sphinx-rtd-theme==3.1.0 # via -r requirements.in -sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@46c52fc3a6d5ee632a8ef6bccf4daa1ff53e96fd +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@2b5f6c7de64c8317cb771fdeb2e5020d1c9c9dcf # via -r requirements.in sphinxcontrib-applehelp==2.0.0 # via sphinx diff --git a/tox.ini b/tox.ini index c184e860..6f0dbb36 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,7 @@ [tox] requires = - sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.7.0 + sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.7.1 # pypy3 excluded due to https://github.com/kevin1024/vcrpy/issues/885 envlist = py{310,311,312}-{json,msgpack},docs