From 87b2b92922687460174c9483582828a8e23a0419 Mon Sep 17 00:00:00 2001 From: Qiusheng Wu Date: Sat, 31 Jan 2026 01:44:20 -0500 Subject: [PATCH] Fix subprocess ResourceWarning, modernize Python code, fix broken links Fixes: - Fix ResourceWarning from unclosed subprocess (fixes #84): Use Popen as context manager via new _run_process() and _run_process_output() helpers. All 17 bare Popen calls now properly use 'with' statements. - Extract common Popen kwargs into _popen_kwargs() to eliminate duplicated Windows startupinfo logic across every method. Python modernization: - Remove 'from __future__ import print_function' (Python 2 compat, unnecessary) - Replace 'class WhiteboxTools(object)' with 'class WhiteboxTools' (new-style) - Convert .format() to f-strings in core methods - Replace '== True' comparisons with idiomatic boolean checks - Fix mutable default argument: list_tools(keywords=[]) -> keywords=None - Fix incorrect docstring on set_max_procs (was copy of set_compress_rasters) README fixes: - Replace 5 broken gishub.org short links (all returning 404) with direct URLs - Update pepy.tech badge to current format setup.py: - Update Python version classifiers from 3.4-3.6 to 3.9-3.13 - Update development status from Pre-Alpha to Beta --- README.rst | 19 +- setup.py | 10 +- whitebox/whitebox_tools.py | 471 +++++++++---------------------------- 3 files changed, 130 insertions(+), 370 deletions(-) diff --git a/README.rst b/README.rst index 8b18b45..5f27e57 100644 --- a/README.rst +++ b/README.rst @@ -5,13 +5,13 @@ whitebox-python .. image:: https://colab.research.google.com/assets/colab-badge.svg :target: https://colab.research.google.com/github/opengeos/whitebox-python/blob/master/examples/whitebox.ipynb -.. image:: https://mybinder.org/badge_logo.svg - :target: https://gishub.org/whitebox-cloud +.. image:: https://mybinder.org/badge_logo.svg + :target: https://mybinder.org/v2/gh/opengeos/whitebox-python/HEAD?labpath=examples%2Fwhitebox.ipynb .. image:: https://img.shields.io/pypi/v/whitebox.svg :target: https://pypi.python.org/pypi/whitebox -.. image:: https://pepy.tech/badge/whitebox +.. image:: https://img.shields.io/pepy/dt/whitebox :target: https://pepy.tech/project/whitebox .. image:: https://anaconda.org/conda-forge/whitebox/badges/version.svg @@ -44,7 +44,7 @@ This repository is related to the WhiteboxTools Python Frontend only. You can re * PyPI: https://pypi.org/project/whitebox/ * conda-forge: https://anaconda.org/conda-forge/whitebox * Documentation: https://whitebox.readthedocs.io -* Binder: https://gishub.org/whitebox-cloud +* Binder: https://mybinder.org/v2/gh/opengeos/whitebox-python/HEAD?labpath=examples%2Fwhitebox.ipynb * Free software: `MIT license`_ @@ -112,8 +112,8 @@ whitebox Tutorials Launch the whitebox tutorial notebook directly with **mybinder.org** now: -.. image:: https://mybinder.org/badge_logo.svg - :target: https://gishub.org/whitebox-cloud +.. image:: https://mybinder.org/badge_logo.svg + :target: https://mybinder.org/v2/gh/opengeos/whitebox-python/HEAD?labpath=examples%2Fwhitebox.ipynb Quick Example ============= @@ -145,11 +145,10 @@ A Jupyter Notebook Tutorial for whitebox This tutorial can be accessed in three ways: -- HTML version: https://gishub.org/whitebox-html -- Viewable Notebook: https://gishub.org/whitebox-notebook -- Interactive Notebook: https://gishub.org/whitebox-cloud +- Viewable Notebook: https://github.com/opengeos/whitebox-python/blob/master/examples/whitebox.ipynb +- Interactive Notebook: https://mybinder.org/v2/gh/opengeos/whitebox-python/HEAD?labpath=examples%2Fwhitebox.ipynb -Launch this tutorial as an interactive Jupyter Notebook on the cloud - https://gishub.org/whitebox-cloud. +Launch this tutorial as an interactive Jupyter Notebook on the cloud - https://mybinder.org/v2/gh/opengeos/whitebox-python/HEAD?labpath=examples%2Fwhitebox.ipynb .. image:: https://i.imgur.com/LF4UE1j.gif diff --git a/setup.py b/setup.py index 2421a7d..000715e 100644 --- a/setup.py +++ b/setup.py @@ -23,14 +23,16 @@ author="Qiusheng Wu", author_email="giswqs@gmail.com", classifiers=[ - "Development Status :: 2 - Pre-Alpha", + "Development Status :: 4 - Beta", "Intended Audience :: Developers", "License :: OSI Approved :: MIT License", "Natural Language :: English", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.4", - "Programming Language :: Python :: 3.5", - "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", ], description="An advanced geospatial data analysis platform ", entry_points={ diff --git a/whitebox/whitebox_tools.py b/whitebox/whitebox_tools.py index a48a0d4..9111066 100644 --- a/whitebox/whitebox_tools.py +++ b/whitebox/whitebox_tools.py @@ -9,7 +9,6 @@ # Last Modified: 09/12/2019 # License: MIT -from __future__ import print_function import urllib.request import zipfile import shutil @@ -20,7 +19,6 @@ import re import json -# import shutil from subprocess import CalledProcessError, Popen, PIPE, STDOUT running_windows = platform.system() == "Windows" @@ -291,7 +289,7 @@ def to_snakecase(name): return re.sub("([a-z0-9])([A-Z])", r"\1_\2", s1).lower() -class WhiteboxTools(object): +class WhiteboxTools: """ An object for interfacing with the WhiteboxTools executable. """ @@ -301,7 +299,7 @@ def __init__(self): self.ext = ".exe" else: self.ext = "" - self.exe_name = "whitebox_tools{}".format(self.ext) + self.exe_name = f"whitebox_tools{self.ext}" # self.exe_path = os.path.dirname(shutil.which( # self.exe_name) or path.dirname(path.abspath(__file__))) # self.exe_path = os.path.dirname(os.path.join(os.path.realpath(__file__))) @@ -329,6 +327,60 @@ def __init__(self): self.default_callback = default_callback self.start_minimized = False + def _popen_kwargs(self): + """Returns common keyword arguments for Popen calls.""" + kwargs = { + "shell": False, + "stdout": PIPE, + "stderr": STDOUT, + "bufsize": 1, + "universal_newlines": True, + } + if running_windows and self.start_minimized: + si = STARTUPINFO() + si.dwFlags = STARTF_USESHOWWINDOW + si.wShowWindow = 7 # Set window minimized and not activated + kwargs["startupinfo"] = si + return kwargs + + def _run_process(self, args, callback): + """ + Runs a subprocess with the given args and streams output to callback. + + Uses Popen as a context manager to avoid ResourceWarning (see #84). + Returns 0 on success, 1 on error, 2 if cancelled by user. + """ + with Popen(args, **self._popen_kwargs()) as proc: + while True: + line = proc.stdout.readline() + if line != "": + if not self.cancel_op: + if callback: + callback(line.strip()) + else: + self.cancel_op = False + proc.terminate() + return 2 + else: + break + return 0 + + def _run_process_output(self, args): + """ + Runs a subprocess and returns the full output as a string. + + Uses Popen as a context manager to avoid ResourceWarning (see #84). + """ + with Popen(args, **self._popen_kwargs()) as proc: + ret = "" + while True: + line = proc.stdout.readline() + if line != "": + ret += line + else: + break + return ret + def set_whitebox_dir(self, path_str): """ Sets the directory to the WhiteboxTools executable file. @@ -367,54 +419,14 @@ def set_verbose_mode(self, val=True): work_dir = os.getcwd() os.chdir(self.exe_path) - args2 = [] - args2.append("." + path.sep + self.exe_name) + args2 = [f".{path.sep}{self.exe_name}"] if self.verbose: args2.append("-v") else: args2.append("-v=false") - proc = None - - if running_windows and self.start_minimized == True: - si = STARTUPINFO() - si.dwFlags = STARTF_USESHOWWINDOW - si.wShowWindow = 7 # Set window minimized and not activated - proc = Popen( - args2, - shell=False, - stdout=PIPE, - stderr=STDOUT, - bufsize=1, - universal_newlines=True, - startupinfo=si, - ) - else: - proc = Popen( - args2, - shell=False, - stdout=PIPE, - stderr=STDOUT, - bufsize=1, - universal_newlines=True, - ) - - while proc is not None: - line = proc.stdout.readline() - sys.stdout.flush() - if line != "": - if not self.cancel_op: - callback(line.strip()) - else: - self.cancel_op = False - proc.terminate() - return 2 - - else: - break - - return 0 + return self._run_process(args2, callback) except (OSError, ValueError, CalledProcessError) as err: callback(str(err)) return 1 @@ -438,54 +450,14 @@ def set_compress_rasters(self, val=True): work_dir = os.getcwd() os.chdir(self.exe_path) - args2 = [] - args2.append("." + path.sep + self.exe_name) + args2 = [f".{path.sep}{self.exe_name}"] if self.__compress_rasters: args2.append("--compress_rasters=true") else: args2.append("--compress_rasters=false") - proc = None - - if running_windows and self.start_minimized == True: - si = STARTUPINFO() - si.dwFlags = STARTF_USESHOWWINDOW - si.wShowWindow = 7 # Set window minimized and not activated - proc = Popen( - args2, - shell=False, - stdout=PIPE, - stderr=STDOUT, - bufsize=1, - universal_newlines=True, - startupinfo=si, - ) - else: - proc = Popen( - args2, - shell=False, - stdout=PIPE, - stderr=STDOUT, - bufsize=1, - universal_newlines=True, - ) - - while proc is not None: - line = proc.stdout.readline() - sys.stdout.flush() - if line != "": - if not self.cancel_op: - callback(line.strip()) - else: - self.cancel_op = False - proc.terminate() - return 2 - - else: - break - - return 0 + return self._run_process(args2, callback) except (OSError, ValueError, CalledProcessError) as err: callback(str(err)) return 1 @@ -497,7 +469,7 @@ def get_compress_rasters(self): def set_max_procs(self, val=-1): """ - Sets the flag used by WhiteboxTools to determine whether to use compression for output rasters. + Sets the maximum number of processes for WhiteboxTools. """ self.__max_procs = val @@ -506,51 +478,9 @@ def set_max_procs(self, val=-1): work_dir = os.getcwd() os.chdir(self.exe_path) - args2 = [] - args2.append("." + path.sep + self.exe_name) - - args2.append(f"--max_procs={val}") - - proc = None - - if running_windows and self.start_minimized == True: - si = STARTUPINFO() - si.dwFlags = STARTF_USESHOWWINDOW - si.wShowWindow = 7 # Set window minimized and not activated - proc = Popen( - args2, - shell=False, - stdout=PIPE, - stderr=STDOUT, - bufsize=1, - universal_newlines=True, - startupinfo=si, - ) - else: - proc = Popen( - args2, - shell=False, - stdout=PIPE, - stderr=STDOUT, - bufsize=1, - universal_newlines=True, - ) + args2 = [f".{path.sep}{self.exe_name}", f"--max_procs={val}"] - while proc is not None: - line = proc.stdout.readline() - sys.stdout.flush() - if line != "": - if not self.cancel_op: - callback(line.strip()) - else: - self.cancel_op = False - proc.terminate() - return 2 - - else: - break - - return 0 + return self._run_process(args2, callback) except (OSError, ValueError, CalledProcessError) as err: callback(str(err)) return 1 @@ -573,18 +503,15 @@ def run_tool(self, tool_name, args, callback=None): work_dir = os.getcwd() os.chdir(self.exe_path) - args2 = [] - args2.append("." + path.sep + self.exe_name) - args2.append('--run="{}"'.format(to_camelcase(tool_name))) + args2 = [ + f".{path.sep}{self.exe_name}", + f'--run="{to_camelcase(tool_name)}"', + ] if self.work_dir.strip() != "": - args2.append('--wd="{}"'.format(self.work_dir)) - - for arg in args: - args2.append(arg) + args2.append(f'--wd="{self.work_dir}"') - # args_str = args_str[:-1] - # a.append("--args=\"{}\"".format(args_str)) + args2.extend(args) if self.verbose: args2.append("-v") @@ -600,46 +527,9 @@ def run_tool(self, tool_name, args, callback=None): cl = " ".join(args2) callback(cl.strip() + "\n") - proc = None - - if running_windows and self.start_minimized == True: - si = STARTUPINFO() - si.dwFlags = STARTF_USESHOWWINDOW - si.wShowWindow = 7 # Set window minimized and not activated - proc = Popen( - args2, - shell=False, - stdout=PIPE, - stderr=STDOUT, - bufsize=1, - universal_newlines=True, - startupinfo=si, - ) - else: - proc = Popen( - args2, - shell=False, - stdout=PIPE, - stderr=STDOUT, - bufsize=1, - universal_newlines=True, - ) - - while proc is not None: - line = proc.stdout.readline() - sys.stdout.flush() - if line != "": - if not self.cancel_op: - if self.verbose: - callback(line.strip()) - else: - self.cancel_op = False - proc.terminate() - return 2 - else: - break - - return 0 + return self._run_process( + args2, callback if self.verbose else None + ) except (OSError, ValueError, CalledProcessError) as err: callback(str(err)) return 1 @@ -653,27 +543,8 @@ def help(self): try: work_dir = os.getcwd() os.chdir(self.exe_path) - args = [] - args.append("." + os.path.sep + self.exe_name) - args.append("-h") - - proc = Popen( - args, - shell=False, - stdout=PIPE, - stderr=STDOUT, - bufsize=1, - universal_newlines=True, - ) - ret = "" - while True: - line = proc.stdout.readline() - if line != "": - ret += line - else: - break - - return ret + args = [f".{os.path.sep}{self.exe_name}", "-h"] + return self._run_process_output(args) except (OSError, ValueError, CalledProcessError) as err: return err finally: @@ -686,29 +557,10 @@ def license(self, toolname=None): try: work_dir = os.getcwd() os.chdir(self.exe_path) - args = [] - args.append("." + os.path.sep + self.exe_name) - args.append("--license") + args = [f".{os.path.sep}{self.exe_name}", "--license"] if toolname is not None: args.append(f"={toolname}") - - proc = Popen( - args, - shell=False, - stdout=PIPE, - stderr=STDOUT, - bufsize=1, - universal_newlines=True, - ) - ret = "" - while True: - line = proc.stdout.readline() - if line != "": - ret += line - else: - break - - return ret + return self._run_process_output(args) except (OSError, ValueError, CalledProcessError) as err: return err finally: @@ -721,27 +573,8 @@ def version(self): try: work_dir = os.getcwd() os.chdir(self.exe_path) - args = [] - args.append("." + os.path.sep + self.exe_name) - args.append("--version") - - proc = Popen( - args, - shell=False, - stdout=PIPE, - stderr=STDOUT, - bufsize=1, - universal_newlines=True, - ) - ret = "" - while True: - line = proc.stdout.readline() - if line != "": - ret += line - else: - break - - return ret + args = [f".{os.path.sep}{self.exe_name}", "--version"] + return self._run_process_output(args) except (OSError, ValueError, CalledProcessError) as err: return err finally: @@ -754,27 +587,11 @@ def tool_help(self, tool_name=""): try: work_dir = os.getcwd() os.chdir(self.exe_path) - args = [] - args.append("." + os.path.sep + self.exe_name) - args.append("--toolhelp={}".format(to_camelcase(tool_name))) - - proc = Popen( - args, - shell=False, - stdout=PIPE, - stderr=STDOUT, - bufsize=1, - universal_newlines=True, - ) - ret = "" - while True: - line = proc.stdout.readline() - if line != "": - ret += line - else: - break - - return ret + args = [ + f".{os.path.sep}{self.exe_name}", + f"--toolhelp={to_camelcase(tool_name)}", + ] + return self._run_process_output(args) except (OSError, ValueError, CalledProcessError) as err: return err finally: @@ -787,27 +604,11 @@ def tool_parameters(self, tool_name): try: work_dir = os.getcwd() os.chdir(self.exe_path) - args = [] - args.append("." + os.path.sep + self.exe_name) - args.append("--toolparameters={}".format(to_camelcase(tool_name))) - - proc = Popen( - args, - shell=False, - stdout=PIPE, - stderr=STDOUT, - bufsize=1, - universal_newlines=True, - ) - ret = "" - while True: - line = proc.stdout.readline() - if line != "": - ret += line - else: - break - - return ret + args = [ + f".{os.path.sep}{self.exe_name}", + f"--toolparameters={to_camelcase(tool_name)}", + ] + return self._run_process_output(args) except (OSError, ValueError, CalledProcessError) as err: return err finally: @@ -820,27 +621,11 @@ def toolbox(self, tool_name=""): try: work_dir = os.getcwd() os.chdir(self.exe_path) - args = [] - args.append("." + os.path.sep + self.exe_name) - args.append("--toolbox={}".format(to_camelcase(tool_name))) - - proc = Popen( - args, - shell=False, - stdout=PIPE, - stderr=STDOUT, - bufsize=1, - universal_newlines=True, - ) - ret = "" - while True: - line = proc.stdout.readline() - if line != "": - ret += line - else: - break - - return ret + args = [ + f".{os.path.sep}{self.exe_name}", + f"--toolbox={to_camelcase(tool_name)}", + ] + return self._run_process_output(args) except (OSError, ValueError, CalledProcessError) as err: return err finally: @@ -854,65 +639,39 @@ def view_code(self, tool_name): try: work_dir = os.getcwd() os.chdir(self.exe_path) - args = [] - args.append("." + os.path.sep + self.exe_name) - args.append("--viewcode={}".format(to_camelcase(tool_name))) - - proc = Popen( - args, - shell=False, - stdout=PIPE, - stderr=STDOUT, - bufsize=1, - universal_newlines=True, - ) - ret = "" - while True: - line = proc.stdout.readline() - if line != "": - ret += line - else: - break - - return ret + args = [ + f".{os.path.sep}{self.exe_name}", + f"--viewcode={to_camelcase(tool_name)}", + ] + return self._run_process_output(args) except (OSError, ValueError, CalledProcessError) as err: return err finally: os.chdir(work_dir) - def list_tools(self, keywords=[]): + def list_tools(self, keywords=None): """ Lists all available tools in WhiteboxTools. """ + if keywords is None: + keywords = [] try: work_dir = os.getcwd() os.chdir(self.exe_path) - args = [] - args.append("." + os.path.sep + self.exe_name) - args.append("--listtools") - if len(keywords) > 0: - for kw in keywords: - args.append(kw) - - proc = Popen( - args, - shell=False, - stdout=PIPE, - stderr=STDOUT, - bufsize=1, - universal_newlines=True, - ) - ret = {} - line = proc.stdout.readline() # skip number of available tools header - while True: - line = proc.stdout.readline() - if line != "": - if line.strip() != "": - name, descr = line.split(":") - ret[to_snakecase(name.strip())] = descr.strip() - else: - break - + args = [f".{os.path.sep}{self.exe_name}", "--listtools"] + args.extend(keywords) + + with Popen(args, **self._popen_kwargs()) as proc: + ret = {} + proc.stdout.readline() # skip number of available tools header + while True: + line = proc.stdout.readline() + if line != "": + if line.strip() != "": + name, descr = line.split(":") + ret[to_snakecase(name.strip())] = descr.strip() + else: + break return ret except (OSError, ValueError, CalledProcessError) as err: return err