setup.py

import shutil
import subprocess
import sys

from setuptools import find_packages, setup
from setuptools.command.install import install


class PostInstallCommand(install):
    def run(self):
        install.run(self)

        pip = ["uv", "pip"] if shutil.which("uv") else ["pip"]

        # do "python -m playwright install"
        try:
            subprocess.check_call([sys.executable, "-m", "playwright", "install"])
        except Exception as err:
            print(f"Error when installing playwright: '{err}'")

        # do pip install --user -U --pre yt-dlp
        try:
            subprocess.check_call(
                [
                    sys.executable,
                    "-m",
                ]
                + pip
                + [
                    "install",
                    "--user",
                    "-U",
                    "--pre",
                    "yt-dlp",
                ]
            )
        except Exception as err:
            print(f"Error when installing yt-dlp pre-release: '{err}'")

        # do "python -m pip install -U git+https://github.com/ahupp/python-magic/
        # see https://github.com/ahupp/python-magic/issues/261
        try:
            subprocess.check_call(
                [
                    sys.executable,
                    "-m",
                ]
                + pip
                + [
                    "install",
                    "-U",
                    "git+https://github.com/ahupp/python-magic/",
                ],
            )
        except Exception as err:
            print(f"Error when pip updating python-magic from git: '{err}'")

        # Install audioop-lts only for Python 3.13+
        # audioop was removed in Python 3.13, and pydub needs it
        # See https://github.com/jiaaro/pydub/issues/815
        if sys.version_info >= (3, 13):
            try:
                subprocess.check_call(
                    [
                        sys.executable,
                        "-m",
                    ]
                    + pip
                    + [
                        "install",
                        "-U",
                        "audioop-lts>=0.2.2",
                    ]
                )
            except Exception as err:
                print(f"Error when installing audioop-lts for Python 3.13+: '{err}'")

        # do "openparse-download"
        try:
            subprocess.check_call(
                ["openparse-download"],
            )
        except Exception as err:
            print(
                "Error when trying to run 'openparse-download' to download"
                f" weights for deep learning based table detection : '{err}'"
                "\nBy default wdoc still uses pymupdf via openparse so it "
                "shouldn't matter too much.\n"
                "For more: see https://github.com/Filimoa/open-parse/"
            )

        # do "import nltk ; nltk.download('punkt_tab')"
        try:
            import nltk

            nltk.download("punkt_tab")
        except Exception as err:
            print(f"Error when downloading nltk punkt_tab: '{err}'")


with open("README.md", "r") as readme:
    long_description = readme.read()

    # Convert icon HTML to markdown
    assert (
        '<p align="center"><img src="https://github.com/thiswillbeyourgithub/wdoc/blob/main/images/icon.png?raw=true" width="512" style="background-color: transparent !important"></p>'
        in long_description
    ), "Unexpected HTML for the icon"
    long_description = long_description.replace(
        '<p align="center"><img src="https://github.com/thiswillbeyourgithub/wdoc/blob/main/images/icon.png?raw=true" width="512" style="background-color: transparent !important"></p>',
        "![icon](https://github.com/thiswillbeyourgithub/wdoc/blob/main/images/icon.png?raw=true)",
    )

    # Convert query diagram HTML to markdown
    assert (
        '<img src="https://github.com/thiswillbeyourgithub/wdoc/blob/main/images/diagram_query.png?raw=true" alt="Query task workflow diagram showing the flow from user inputs through Raphael the Rephraser, VectorStore, Eve the Evaluator, Anna the Answerer, and recursive combining to final output" height="400">'
        in long_description
    ), "Unexpected HTML for query diagram"
    long_description = long_description.replace(
        '<img src="https://github.com/thiswillbeyourgithub/wdoc/blob/main/images/diagram_query.png?raw=true" alt="Query task workflow diagram showing the flow from user inputs through Raphael the Rephraser, VectorStore, Eve the Evaluator, Anna the Answerer, and recursive combining to final output" height="400">',
        "![Query task workflow diagram showing the flow from user inputs through Raphael the Rephraser, VectorStore, Eve the Evaluator, Anna the Answerer, and recursive combining to final output](https://github.com/thiswillbeyourgithub/wdoc/blob/main/images/diagram_query.png?raw=true)",
    )

    # Convert summary diagram HTML to markdown
    assert (
        '<img src="https://github.com/thiswillbeyourgithub/wdoc/blob/main/images/diagram_summary.png?raw=true" alt="Summary task workflow diagram showing the flow from user inputs through loading & chunking, Sam the Summarizer, concatenation to wdocSummary output" height="400">'
        in long_description
    ), "Unexpected HTML for summary diagram"
    long_description = long_description.replace(
        '<img src="https://github.com/thiswillbeyourgithub/wdoc/blob/main/images/diagram_summary.png?raw=true" alt="Summary task workflow diagram showing the flow from user inputs through loading & chunking, Sam the Summarizer, concatenation to wdocSummary output" height="400">',
        "![Summary task workflow diagram showing the flow from user inputs through loading & chunking, Sam the Summarizer, concatenation to wdocSummary output](https://github.com/thiswillbeyourgithub/wdoc/blob/main/images/diagram_summary.png?raw=true)",
    )

    # Convert search diagram HTML to markdown
    assert (
        '<img src="https://github.com/thiswillbeyourgithub/wdoc/blob/main/images/diagram_search.png?raw=true" alt="Search task workflow diagram showing the flow from user inputs through Raphael the Rephraser, VectorStore, Eve the Evaluator to search output" height="400">'
        in long_description
    ), "Unexpected HTML for search diagram"
    long_description = long_description.replace(
        '<img src="https://github.com/thiswillbeyourgithub/wdoc/blob/main/images/diagram_search.png?raw=true" alt="Search task workflow diagram showing the flow from user inputs through Raphael the Rephraser, VectorStore, Eve the Evaluator to search output" height="400">',
        "![Search task workflow diagram showing the flow from user inputs through Raphael the Rephraser, VectorStore, Eve the Evaluator to search output](https://github.com/thiswillbeyourgithub/wdoc/blob/main/images/diagram_search.png?raw=true)",
    )

    assert 'align="center"' not in long_description

setup(
    name="wdoc",
    version="5.0.0",
    description="A perfect AI powered RAG for document query and summary. Supports ~all LLM and ~all filetypes (url, pdf, epub, youtube (incl playlist), audio, anki, md, docx, pptx, or any combination!)",
    long_description=long_description,
    long_description_content_type="text/markdown",
    url="https://github.com/thiswillbeyourgithub/wdoc/",
    packages=find_packages(),
    include_package_data=True,
    classifiers=[
        "Programming Language :: Python :: 3",
        "Operating System :: OS Independent",
    ],
    license="AGPLv3",
    keywords=[
        "RAG",
        "search",
        "summary",
        "summarize",
        "pdf",
        "documents",
        "doc",
        "docx",
        "youtube",
        "mp3",
        "embeddings",
        "AI",
        "LLM",
        "openai",
        "logseq",
        "doctools",
        "doctoolsllm",
        "winston_doc",
    ],
    entry_points={
        "console_scripts": [
            "wdoc=wdoc.__main__:cli_launcher",
        ],
    },
    python_requires=">=3.11",
    install_requires=[
        "sqlalchemy>=2.0.32",
        "beautifulsoup4>=4.12.3",
        "fire>=0.6.0",
        "ftfy>=6.2.0",
        "joblib>=1.4.2",
        "langchain>=1.2.0",
        "langchain-classic>=1.0.0",
        "langchain-community>=0.3.30",
        "langchain-openai>=0.3.34",
        "langchain-litellm>=0.3.5",
        "langfuse>=3.6.1",  # for observability
        "litellm>=v1.78.2",
        "nest_asyncio>=1.6.0",  # needed to fix ollama 'event loop closed' error thanks to https://github.com/BerriAI/litellm/pull/7625/files
        "chonkie[all]>=1.4.0",  # chonkie is for the semantic embeddings
        "chonkie[semantic]>=1.4.0",
        "prompt-toolkit>=3.0.47",
        "tqdm>=4.66.4",
        "faiss-cpu>=1.8.0",
        "rich>=13.8.1",
        "beartype >= 0.22.2",
        "platformdirs >= 4.2.2",
        "dill >= 0.3.8",
        "pyfiglet >= 1.0.2",  # banner
        "rtoml >= 0.11.0",
        "loguru >= 0.7.2",
        "grandalf >= 0.8",  # to print ascii graph
        "lazy-import >= 0.2.2",
        "py_ankiconnect >= 1.1.2",  # DIY wrapper to tell anki to sync just in case
        "scikit-learn >= 1.5.1",  # for semantic reordering
        "scipy >= 1.13.1",  # for semantic reordering
        # 'python-magic >= 0.4.27',  # for detecting file type  # made optional as it can help infer the filetype, and 0.4.28 is necessary for the pipe feature.
        "uuid6 >= 2025.0.1",  # for time sortable timestamp
        "PersistDict >= 0.2.14",  # by me, like a dict but an LMDB database, to fix langchain's caches
        "nltk>=3.9.2",  # needed for punkt_tab download in post-install
        "blake3>=1.0.8",  # faster than sha256
        "pandas >= 2.3.3",
        # some loaders are included by default:
        "playwright >= 1.45.0",  # for online_media and urls
        "openparse[ml] >= 0.5.7",  # pdf with table support
        # youtube
        "yt-dlp >= 2025.09.26",  # we actually need to install yt-dlp here otherwise readthedocs crashes. Note that in the postinstall script above it will be reinstalled using the master branch
        "youtube-transcript-api >= 0.6.2",
        # "pytube >= 15.0.0",
        # url
        "tldextract>=5.1.2",
        "goose3 >= 3.1.20",
        # online search via 'filetype=web'
        "ddgs >= 9.6.0",
        "duckduckgo-search >= 8.1.1",
        # audio/video transcription
        "deepgram-sdk >= 3.2.7",
        "httpx >= 0.27.0",  # to increase deepgram timeout
        "pydub >= 0.25.1",  # extracting audio from local video
        "ffmpeg-python >= 0.2.0",  # extracting audio from local video
        "torchaudio >= 2.8.0",  # silence removal from audio
        "trio >= 0.31.0",  # for some reason older versions of trio, when present are used and cause issues on python 3.11: https://github.com/python-trio/trio/issues/2317
        # many file formats
        "unstructured[all-docs]>=0.18.15",
    ],
    extras_require={
        "full": [
            # Loaders:
            # pdf
            "pdfminer.six >= 20231228",
            "pillow_heif >= 0.16.0",
            "pypdfium2 >= 4.30.0",
            "pymupdf >= 1.24.5",
            "pdfplumber >= 0.11.1",
            "pdf2image >= 1.17.0",
            # word documents
            "docx2txt >= 0.8",
            # epub
            "pandoc >= 2.4",
            # anki
            "ankipandas>=0.3.15",
            # logseq files (I'm the dev behind it)
            "LogseqMarkdownParser >= 3.3",
        ],
        "fasttext": [
            # buggy in windows so optional: https://github.com/zafercavdar/fasttext-langdetect/issues/14
            "fasttext-langdetect >= 1.0.5",
            "langdetect >= 1.0.9",
        ],
        "pdftotext": [
            # sudo apt install build-essential libpoppler-cpp-dev pkg-config python3-dev
            "pdftotext >= 2.2.2",
        ],
        "dev": [
            "ruff >= 0.14.1",
            # "isort >= 6.0.0",
            "pre-commit >= 4.1.0",
            "pytest >= 8.3.4",
            "pytest-xdist >= 3.6.1",
            "build >= 1.2.2.post1",
            "twine >= 6.1.0",
            "bumpver >= 2025.1131",
        ],
    },
    cmdclass={
        "install": PostInstallCommand,
    },
)