diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7e970fac7..77a5347e0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -21,9 +21,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout source - uses: actions/checkout@v4 + uses: actions/checkout@v4.1.1 - name: Set up Python 3.12 for linting - uses: actions/setup-python@v4.7.1 + uses: actions/setup-python@v5.0.0 with: python-version: '3.12' - name: Install dependencies @@ -48,9 +48,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout source - uses: actions/checkout@v4 + uses: actions/checkout@v4.1.1 - name: Set up Python 3.12 - uses: actions/setup-python@v4.7.1 + uses: actions/setup-python@v5.0.0 with: python-version: '3.12' - name: Upgrade pip @@ -66,7 +66,7 @@ jobs: python -m build --sdist --outdir wheelhouse - name: Install sdist run: |- - ls -al ./wheelhouse + ls -al wheelhouse pip install --prefer-binary wheelhouse/ubelt*.tar.gz -v - name: Test minimal loose sdist run: |- @@ -80,7 +80,7 @@ jobs: # Get path to installed package MOD_DPATH=$(python -c "import ubelt, os; print(os.path.dirname(ubelt.__file__))") echo "MOD_DPATH = $MOD_DPATH" - python -m pytest --verbose --cov={self.mod_name} $MOD_DPATH ../tests + python -m pytest --verbose --cov=ubelt $MOD_DPATH ../tests cd .. - name: Test full loose sdist run: |- @@ -95,13 +95,13 @@ jobs: # Get path to installed package MOD_DPATH=$(python -c "import ubelt, os; print(os.path.dirname(ubelt.__file__))") echo "MOD_DPATH = $MOD_DPATH" - python -m pytest --verbose --cov={self.mod_name} $MOD_DPATH ../tests + python -m pytest --verbose --cov=ubelt $MOD_DPATH ../tests cd .. - - name: Upload sdist artifact - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v3.1.3 + name: Upload sdist artifact with: - name: wheels - path: ./wheelhouse/*.tar.gz + name: sdist_wheels + path: wheelhouse/*.tar.gz build_purepy_wheels: ## # Download and test the pure-python wheels that were build in the @@ -110,6 +110,7 @@ jobs: name: ${{ matrix.python-version }} on ${{ matrix.os }}, arch=${{ matrix.arch }} with ${{ matrix.install-extras }} runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: os: - ubuntu-latest @@ -119,14 +120,14 @@ jobs: - auto steps: - name: Checkout source - uses: actions/checkout@v4 + uses: actions/checkout@v4.1.1 - name: Set up QEMU uses: docker/setup-qemu-action@v3 if: runner.os == 'Linux' && matrix.arch != 'auto' with: platforms: all - name: Setup Python - uses: actions/setup-python@v4.7.1 + uses: actions/setup-python@v5.0.0 with: python-version: ${{ matrix.python-version }} - name: Build pure wheel @@ -138,7 +139,7 @@ jobs: - name: Show built files shell: bash run: ls -la wheelhouse - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v3.1.3 name: Upload wheels artifact with: name: wheels @@ -149,6 +150,7 @@ jobs: needs: - build_purepy_wheels strategy: + fail-fast: false matrix: # Xcookie generates an explicit list of environments that will be used # for testing instead of using the more concise matrix notation. @@ -295,7 +297,7 @@ jobs: arch: auto steps: - name: Checkout source - uses: actions/checkout@v4 + uses: actions/checkout@v4.1.1 - name: Enable MSVC 64bit uses: ilammy/msvc-dev-cmd@v1 if: matrix.os == 'windows-latest' @@ -305,10 +307,10 @@ jobs: with: platforms: all - name: Setup Python - uses: actions/setup-python@v4.7.1 + uses: actions/setup-python@v5.0.0 with: python-version: ${{ matrix.python-version }} - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v2.1.1 name: Download wheels with: name: wheels @@ -342,6 +344,7 @@ jobs: ls -altr # Get the path to the installed package and run the tests export MOD_DPATH=$(python -c "import ubelt, os; print(os.path.dirname(ubelt.__file__))") + export MOD_NAME=ubelt echo " --- MOD_DPATH = $MOD_DPATH @@ -349,7 +352,7 @@ jobs: running the pytest command inside the workspace --- " - python -m pytest --verbose -p pytester -p no:doctest --xdoctest --cov-config ../pyproject.toml --cov-report term --cov="ubelt" "$MOD_DPATH" ../tests + python -m pytest --verbose -p pytester -p no:doctest --xdoctest --cov-config ../pyproject.toml --cov-report term --durations=100 --cov="$MOD_NAME" "$MOD_DPATH" ../tests echo "pytest command finished, moving the coverage file to the repo root" ls -al # Move coverage file to a new name @@ -372,10 +375,11 @@ jobs: echo '### The cwd should now have a coverage.xml' ls -altr pwd - - uses: codecov/codecov-action@v3 + - uses: codecov/codecov-action@v4.0.1 name: Codecov Upload with: file: ./coverage.xml + token: ${{ secrets.CODECOV_TOKEN }} test_deploy: name: Uploading Test to PyPi runs-on: ubuntu-latest @@ -386,12 +390,17 @@ jobs: - test_purepy_wheels steps: - name: Checkout source - uses: actions/checkout@v4 - - uses: actions/download-artifact@v3 - name: Download wheels and sdist + uses: actions/checkout@v4.1.1 + - uses: actions/download-artifact@v2.1.1 + name: Download wheels with: name: wheels path: wheelhouse + - uses: actions/download-artifact@v2.1.1 + name: Download sdist + with: + name: sdist_wheels + path: wheelhouse - name: Show files to upload shell: bash run: ls -la wheelhouse @@ -419,7 +428,33 @@ jobs: pip install urllib3 requests[security] twine GPG_KEYID=$(cat dev/public_gpg_key) echo "GPG_KEYID = '$GPG_KEYID'" - DO_GPG=True GPG_KEYID=$GPG_KEYID TWINE_REPOSITORY_URL=${TWINE_REPOSITORY_URL} TWINE_PASSWORD=$TWINE_PASSWORD TWINE_USERNAME=$TWINE_USERNAME GPG_EXECUTABLE=$GPG_EXECUTABLE DO_UPLOAD=True DO_TAG=False ./publish.sh + GPG_SIGN_CMD="$GPG_EXECUTABLE --batch --yes --detach-sign --armor --local-user $GPG_KEYID" + WHEEL_PATHS=(wheelhouse/*.whl wheelhouse/*.tar.gz) + WHEEL_PATHS_STR=$(printf '"%s" ' "${WHEEL_PATHS[@]}") + echo "$WHEEL_PATHS_STR" + for WHEEL_PATH in "${WHEEL_PATHS[@]}" + do + echo "------" + echo "WHEEL_PATH = $WHEEL_PATH" + $GPG_SIGN_CMD --output $WHEEL_PATH.asc $WHEEL_PATH + $GPG_EXECUTABLE --verify $WHEEL_PATH.asc $WHEEL_PATH || echo "hack, the first run of gpg very fails" + $GPG_EXECUTABLE --verify $WHEEL_PATH.asc $WHEEL_PATH + done + ls -la wheelhouse + pip install opentimestamps-client + ots stamp wheelhouse/*.whl wheelhouse/*.tar.gz wheelhouse/*.asc + ls -la wheelhouse + twine upload --username __token__ --password "$TWINE_PASSWORD" --repository-url "$TWINE_REPOSITORY_URL" wheelhouse/*.whl wheelhouse/*.tar.gz --skip-existing --verbose || { echo "failed to twine upload" ; exit 1; } + - uses: actions/upload-artifact@v3.1.3 + name: Upload deploy artifacts + with: + name: deploy_artifacts + path: |- + wheelhouse/*.whl + wheelhouse/*.zip + wheelhouse/*.tar.gz + wheelhouse/*.asc + wheelhouse/*.ots live_deploy: name: Uploading Live to PyPi runs-on: ubuntu-latest @@ -430,12 +465,17 @@ jobs: - test_purepy_wheels steps: - name: Checkout source - uses: actions/checkout@v4 - - uses: actions/download-artifact@v3 - name: Download wheels and sdist + uses: actions/checkout@v4.1.1 + - uses: actions/download-artifact@v2.1.1 + name: Download wheels with: name: wheels path: wheelhouse + - uses: actions/download-artifact@v2.1.1 + name: Download sdist + with: + name: sdist_wheels + path: wheelhouse - name: Show files to upload shell: bash run: ls -la wheelhouse @@ -463,7 +503,71 @@ jobs: pip install urllib3 requests[security] twine GPG_KEYID=$(cat dev/public_gpg_key) echo "GPG_KEYID = '$GPG_KEYID'" - DO_GPG=True GPG_KEYID=$GPG_KEYID TWINE_REPOSITORY_URL=${TWINE_REPOSITORY_URL} TWINE_PASSWORD=$TWINE_PASSWORD TWINE_USERNAME=$TWINE_USERNAME GPG_EXECUTABLE=$GPG_EXECUTABLE DO_UPLOAD=True DO_TAG=False ./publish.sh + GPG_SIGN_CMD="$GPG_EXECUTABLE --batch --yes --detach-sign --armor --local-user $GPG_KEYID" + WHEEL_PATHS=(wheelhouse/*.whl wheelhouse/*.tar.gz) + WHEEL_PATHS_STR=$(printf '"%s" ' "${WHEEL_PATHS[@]}") + echo "$WHEEL_PATHS_STR" + for WHEEL_PATH in "${WHEEL_PATHS[@]}" + do + echo "------" + echo "WHEEL_PATH = $WHEEL_PATH" + $GPG_SIGN_CMD --output $WHEEL_PATH.asc $WHEEL_PATH + $GPG_EXECUTABLE --verify $WHEEL_PATH.asc $WHEEL_PATH || echo "hack, the first run of gpg very fails" + $GPG_EXECUTABLE --verify $WHEEL_PATH.asc $WHEEL_PATH + done + ls -la wheelhouse + pip install opentimestamps-client + ots stamp wheelhouse/*.whl wheelhouse/*.tar.gz wheelhouse/*.asc + ls -la wheelhouse + twine upload --username __token__ --password "$TWINE_PASSWORD" --repository-url "$TWINE_REPOSITORY_URL" wheelhouse/*.whl wheelhouse/*.tar.gz --skip-existing --verbose || { echo "failed to twine upload" ; exit 1; } + - uses: actions/upload-artifact@v3.1.3 + name: Upload deploy artifacts + with: + name: deploy_artifacts + path: |- + wheelhouse/*.whl + wheelhouse/*.zip + wheelhouse/*.tar.gz + wheelhouse/*.asc + wheelhouse/*.ots + release: + name: Create Github Release + if: github.event_name == 'push' && (startsWith(github.event.ref, 'refs/tags') || startsWith(github.event.ref, 'refs/heads/release')) + runs-on: ubuntu-latest + permissions: + contents: write + needs: + - live_deploy + steps: + - name: Checkout source + uses: actions/checkout@v4.1.1 + - uses: actions/download-artifact@v2.1.1 + name: Download artifacts + with: + name: deploy_artifacts + path: wheelhouse + - name: Show files to release + shell: bash + run: ls -la wheelhouse + - run: 'echo "Automatic Release Notes. TODO: improve" > ${{ github.workspace }}-CHANGELOG.txt' + - uses: softprops/action-gh-release@v1 + name: Create Release + id: create_release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + body_path: ${{ github.workspace }}-CHANGELOG.txt + tag_name: ${{ github.ref }} + release_name: Release ${{ github.ref }} + body: Automatic Release + draft: true + prerelease: false + files: |- + wheelhouse/*.whl + wheelhouse/*.asc + wheelhouse/*.ots + wheelhouse/*.zip + wheelhouse/*.tar.gz ### diff --git a/CHANGELOG.md b/CHANGELOG.md index c25c66626..82123e467 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,24 @@ We are currently working on porting this changelog to the specifications in [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). This project (loosely) adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## Version 1.3.4 - +## Version 1.3.5 - + +### Added: +* New wrapper around `pathlib.Path.chmod` in `ubelt.Path.chmod`. Can now + specify string codes like "u+x" or "+rw". Old stat logic works as it + previously did. + + +### Changed: +* Allow the argument to `ubelt.cmd` to be a `PathLike` object, which we will + expect to be an executable. + +### Fixed +* `ub.modname_to_modpath` now handles cases where editable packages have modules where the name is different than the package. +* Fixed deprecated usage of `ast.Num` + + +## Version 1.3.4 - 2023-10-27 ### Added * Add backend option to `highlight_code` which can be "pygments" or "rich". diff --git a/README.rst b/README.rst index 3e3e1e4cb..494d58b9f 100644 --- a/README.rst +++ b/README.rst @@ -128,8 +128,8 @@ Ubelt is small. Its top-level API is defined using roughly 40 lines: from ubelt.util_indexable import (IndexableWalker, indexable_allclose,) from ubelt.util_memoize import (memoize, memoize_method, memoize_property,) from ubelt.util_mixins import (NiceRepr,) - from ubelt.util_path import (Path, TempDir, augpath, ensuredir, expandpath, - shrinkuser, userhome,) + from ubelt.util_path import (ChDir, Path, TempDir, augpath, ensuredir, + expandpath, shrinkuser, userhome,) from ubelt.util_platform import (DARWIN, LINUX, POSIX, WIN32, find_exe, find_path, platform_cache_dir, platform_config_dir, platform_data_dir,) @@ -141,6 +141,7 @@ Ubelt is small. Its top-level API is defined using roughly 40 lines: from ubelt.progiter import (ProgIter,) + Installation: ============= @@ -187,102 +188,105 @@ project. Note: this measure is biased towards older functions. ================================================================================================================================================ ================ Function name Usefulness ================================================================================================================================================ ================ -`ubelt.urepr `__ 2893 -`ubelt.Path `__ 992 -`ubelt.ProgIter `__ 544 -`ubelt.paragraph `__ 482 -`ubelt.take `__ 387 -`ubelt.codeblock `__ 358 -`ubelt.expandpath `__ 331 -`ubelt.cmd `__ 302 -`ubelt.udict `__ 271 -`ubelt.ensuredir `__ 256 -`ubelt.odict `__ 253 -`ubelt.iterable `__ 252 -`ubelt.ddict `__ 238 -`ubelt.NiceRepr `__ 221 -`ubelt.NoParam `__ 216 -`ubelt.map_vals `__ 215 -`ubelt.flatten `__ 214 -`ubelt.dzip `__ 200 -`ubelt.oset `__ 198 -`ubelt.peek `__ 196 -`ubelt.argflag `__ 177 -`ubelt.group_items `__ 171 -`ubelt.hash_data `__ 165 -`ubelt.grabdata `__ 131 -`ubelt.argval `__ 125 -`ubelt.Timer `__ 120 -`ubelt.dict_isect `__ 113 -`ubelt.dict_hist `__ 111 -`ubelt.augpath `__ 106 -`ubelt.identity `__ 106 -`ubelt.ensure_app_cache_dir `__ 105 -`ubelt.allsame `__ 102 -`ubelt.memoize `__ 99 -`ubelt.color_text `__ 98 -`ubelt.dict_diff `__ 95 -`ubelt.delete `__ 89 -`ubelt.hzcat `__ 88 -`ubelt.schedule_deprecation `__ 87 -`ubelt.named_product `__ 85 -`ubelt.compress `__ 85 -`ubelt.IndexableWalker `__ 74 -`ubelt.indent `__ 68 -`ubelt.JobPool `__ 67 -`ubelt.unique `__ 63 -`ubelt.dict_union `__ 57 -`ubelt.map_keys `__ 49 -`ubelt.invert_dict `__ 48 -`ubelt.iter_window `__ 46 -`ubelt.timestamp `__ 46 -`ubelt.argsort `__ 44 -`ubelt.Cacher `__ 43 -`ubelt.find_exe `__ 41 -`ubelt.symlink `__ 41 -`ubelt.dict_subset `__ 41 -`ubelt.writeto `__ 40 -`ubelt.find_duplicates `__ 39 -`ubelt.chunks `__ 38 -`ubelt.hash_file `__ 37 -`ubelt.modname_to_modpath `__ 37 -`ubelt.ensure_unicode `__ 33 -`ubelt.memoize_property `__ 33 -`ubelt.highlight_code `__ 33 -`ubelt.sorted_vals `__ 32 -`ubelt.CacheStamp `__ 30 -`ubelt.WIN32 `__ 28 -`ubelt.import_module_from_name `__ 27 -`ubelt.argmax `__ 27 +`ubelt.urepr `__ 4327 +`ubelt.Path `__ 2125 +`ubelt.paragraph `__ 1349 +`ubelt.ProgIter `__ 747 +`ubelt.cmd `__ 657 +`ubelt.codeblock `__ 611 +`ubelt.udict `__ 603 +`ubelt.expandpath `__ 508 +`ubelt.take `__ 462 +`ubelt.oset `__ 342 +`ubelt.ddict `__ 341 +`ubelt.iterable `__ 313 +`ubelt.flatten `__ 303 +`ubelt.group_items `__ 287 +`ubelt.NiceRepr `__ 270 +`ubelt.ensuredir `__ 267 +`ubelt.map_vals `__ 265 +`ubelt.peek `__ 262 +`ubelt.NoParam `__ 248 +`ubelt.dzip `__ 239 +`ubelt.odict `__ 236 +`ubelt.hash_data `__ 200 +`ubelt.argflag `__ 184 +`ubelt.grabdata `__ 161 +`ubelt.dict_hist `__ 156 +`ubelt.identity `__ 156 +`ubelt.dict_isect `__ 152 +`ubelt.Timer `__ 145 +`ubelt.memoize `__ 142 +`ubelt.argval `__ 134 +`ubelt.allsame `__ 133 +`ubelt.color_text `__ 129 +`ubelt.schedule_deprecation `__ 123 +`ubelt.augpath `__ 120 +`ubelt.dict_diff `__ 117 +`ubelt.IndexableWalker `__ 116 +`ubelt.compress `__ 116 +`ubelt.JobPool `__ 107 +`ubelt.named_product `__ 104 +`ubelt.hzcat `__ 90 +`ubelt.delete `__ 88 +`ubelt.unique `__ 84 +`ubelt.WIN32 `__ 78 +`ubelt.dict_union `__ 76 +`ubelt.symlink `__ 76 +`ubelt.indent `__ 69 +`ubelt.ensure_app_cache_dir `__ 67 +`ubelt.iter_window `__ 62 +`ubelt.invert_dict `__ 58 +`ubelt.memoize_property `__ 57 +`ubelt.import_module_from_name `__ 56 +`ubelt.argsort `__ 55 +`ubelt.timestamp `__ 54 +`ubelt.modname_to_modpath `__ 53 +`ubelt.find_duplicates `__ 53 +`ubelt.hash_file `__ 51 +`ubelt.find_exe `__ 50 +`ubelt.map_keys `__ 50 +`ubelt.dict_subset `__ 50 +`ubelt.Cacher `__ 49 +`ubelt.chunks `__ 47 +`ubelt.sorted_vals `__ 40 +`ubelt.CacheStamp `__ 38 +`ubelt.highlight_code `__ 37 +`ubelt.argmax `__ 36 +`ubelt.writeto `__ 36 +`ubelt.ensure_unicode `__ 32 +`ubelt.sorted_keys `__ 30 +`ubelt.memoize_method `__ 29 +`ubelt.compatible `__ 24 +`ubelt.import_module_from_path `__ 24 +`ubelt.Executor `__ 23 `ubelt.readfrom `__ 23 -`ubelt.import_module_from_path `__ 22 -`ubelt.compatible `__ 17 +`ubelt.modpath_to_modname `__ 17 +`ubelt.AutoDict `__ 17 `ubelt.touch `__ 17 -`ubelt.Executor `__ 16 -`ubelt.memoize_method `__ 16 -`ubelt.sorted_keys `__ 14 -`ubelt.AutoDict `__ 11 +`ubelt.inject_method `__ 14 +`ubelt.timeparse `__ 13 +`ubelt.ChDir `__ 11 `ubelt.shrinkuser `__ 11 -`ubelt.inject_method `__ 10 +`ubelt.argmin `__ 10 `ubelt.varied_values `__ 9 `ubelt.split_modpath `__ 8 -`ubelt.modpath_to_modname `__ 8 -`ubelt.get_app_cache_dir `__ 8 -`ubelt.zopen `__ 7 -`ubelt.LINUX `__ 7 +`ubelt.LINUX `__ 8 +`ubelt.download `__ 7 +`ubelt.NO_COLOR `__ 7 +`ubelt.OrderedSet `__ 6 +`ubelt.zopen `__ 6 `ubelt.CaptureStdout `__ 6 -`ubelt.download `__ 5 -`ubelt.timeparse `__ 5 `ubelt.DARWIN `__ 5 -`ubelt.argmin `__ 5 +`ubelt.boolmask `__ 4 `ubelt.find_path `__ 4 -`ubelt.indexable_allclose `__ 4 -`ubelt.boolmask `__ 3 -`ubelt.map_values `__ 2 +`ubelt.get_app_cache_dir `__ 4 +`ubelt.indexable_allclose `__ 3 +`ubelt.UDict `__ 3 +`ubelt.SetDict `__ 2 `ubelt.AutoOrderedDict `__ 2 `ubelt.argunique `__ 2 -`ubelt.NO_COLOR `__ 2 -`ubelt.UDict `__ 1 +`ubelt.map_values `__ 1 `ubelt.unique_flags `__ 1 `ubelt.userhome `__ 0 `ubelt.split_archive `__ 0 @@ -297,15 +301,14 @@ project. Note: this measure is biased towards older functions. `ubelt.ensure_app_config_dir `__ 0 `ubelt.TempDir `__ 0 `ubelt.TeeStringIO `__ 0 -`ubelt.SetDict `__ 0 `ubelt.ReprExtensions `__ 0 `ubelt.POSIX `__ 0 -`ubelt.OrderedSet `__ 0 `ubelt.DownloadManager `__ 0 `ubelt.CaptureStream `__ 0 ================================================================================================================================================ ================ + Examples ======== diff --git a/dev/maintain/count_usage_freq.py b/dev/maintain/count_usage_freq.py index 0ead06369..dcac76202 100755 --- a/dev/maintain/count_usage_freq.py +++ b/dev/maintain/count_usage_freq.py @@ -1,114 +1,119 @@ - - -import scriptconfig as scfg - - -class UsageConfig(scfg.Config): - default = { - 'print_packages': False, - 'remove_zeros': False, - 'hardcoded_ubelt_hack': True, - 'extra_modnames': [], - } - - -def count_ubelt_usage(): - config = UsageConfig(cmdline=True) - - import ubelt as ub - import glob - from os.path import join - names = [ - 'xdoctest', 'netharn', 'xdev', 'xinspect', 'ndsampler', - 'kwarray', 'kwimage', 'kwplot', 'kwcoco', - 'scriptconfig', 'vimtk', - 'mkinit', 'futures_actors', 'graphid', - - 'ibeis', 'plottool_ibeis', 'guitool_ibeis', 'utool', 'dtool_ibeis', - 'vtool_ibeis', 'hesaff', 'torch_liberator', 'liberator', - ] + config['extra_modnames'] - - code_repos = [ub.Path('~/code').expand() / name for name in names] - repo_dpaths = code_repos + [ - # ub.Path('~/local').expand(), - ub.Path('~/misc').expand(), - ] - all_fpaths = [] - for repo_dpath in repo_dpaths: - name = repo_dpath.stem - fpaths = glob.glob(join(repo_dpath, '**', '*.py'), recursive=True) - for fpath in fpaths: - all_fpaths.append((name, fpath)) - - import re - pat = re.compile(r'\bub\.(?P[a-zA-Z_][A-Za-z_0-9]*)\b') - - import ubelt as ub - - pkg_to_hist = ub.ddict(lambda: ub.ddict(int)) - for name, fpath in ub.ProgIter(all_fpaths): - with open(fpath, 'r') as file: - text = file.read() - for match in pat.finditer(text): - attr = match.groupdict()['attr'] - if attr in ub.__all__: - pkg_to_hist[name][attr] += 1 - - hist_iter = iter(pkg_to_hist.values()) - usage = next(hist_iter).copy() - for other in hist_iter: - for k, v in other.items(): - usage[k] += v - for attr in ub.__all__: - usage[attr] += 0 - - for name in pkg_to_hist.keys(): - pkg_to_hist[name] = ub.odict(sorted(pkg_to_hist[name].items(), key=lambda t: t[1])[::-1]) - - usage = ub.odict(sorted(usage.items(), key=lambda t: t[1])[::-1]) - - if config['print_packages']: - print(ub.repr2(pkg_to_hist, nl=2)) - - if config['remove_zeros']: - for k, v in list(usage.items()): - if v == 0: - usage.pop(k) - - if config['hardcoded_ubelt_hack']: - blocklist = [ - 'progiter', 'timerit', 'orderedset', - ] - for k in list(usage): - if k in blocklist: - usage.pop(k, None) - elif k.startswith('util_'): - usage.pop(k, None) - elif k.startswith('_util_'): - usage.pop(k, None) - # ub._util_deprecated - # from ubelt import _util_deprecated - # if k in dir(_util_deprecated): - # usage.pop(k, None) - - if 1: - # Renamed Aliases - usage['urepr'] += usage.pop('repr2') - usage['ReprExtensions'] += usage.pop('FormatterExtensions') - - usage = ub.udict(usage).sorted_values(reverse=True) - - print(ub.repr2(usage, nl=1)) - return usage - - -if __name__ == '__main__': - """ - For Me: - ~/internal/dev/pkg_usage_stats_update.sh - - CommandLine: - python ~/code/ubelt/dev/maintain/count_usage_freq.py --help - python ~/code/ubelt/dev/maintain/count_usage_freq.py --remove_zeros=False --print_packages=True - """ - count_ubelt_usage() +#!/usr/bin/env python + +# # REMOVE ME + +# import scriptconfig as scfg + + +# class UsageConfig(scfg.Config): +# default = { +# 'print_packages': False, +# 'remove_zeros': False, +# 'hardcoded_ubelt_hack': True, +# 'extra_modnames': [], +# } + + +# def count_package_usage(pkgname='ubelt'): +# config = UsageConfig(cmdline=True) + +# import ubelt as ub +# import glob +# from os.path import join +# names = [ +# 'xdoctest', 'netharn', 'xdev', 'xinspect', 'xcookie', 'ndsampler', +# 'kwarray', 'kwimage', 'kwplot', 'kwcoco', +# 'scriptconfig', 'vimtk', +# 'mkinit', 'futures_actors', 'graphid', + +# 'kwutil', 'git_well', 'line_profiler', 'delayed_image', 'simple_dvc', +# 'pypogo', + +# 'ibeis', 'plottool_ibeis', 'guitool_ibeis', 'utool', 'dtool_ibeis', +# 'vtool_ibeis', 'hesaff', 'torch_liberator', 'liberator', +# ] + config['extra_modnames'] + +# code_repos = [ub.Path('~/code').expand() / name for name in names] +# repo_dpaths = code_repos + [ +# # ub.Path('~/local').expand(), +# ub.Path('~/misc').expand(), +# ] +# all_fpaths = [] +# for repo_dpath in repo_dpaths: +# name = repo_dpath.stem +# fpaths = glob.glob(join(repo_dpath, '**', '*.py'), recursive=True) +# for fpath in fpaths: +# all_fpaths.append((name, fpath)) + +# import re +# pat = re.compile(r'\bub\.(?P[a-zA-Z_][A-Za-z_0-9]*)\b') + +# import ubelt as ub + +# pkg_to_hist = ub.ddict(lambda: ub.ddict(int)) +# for name, fpath in ub.ProgIter(all_fpaths): +# with open(fpath, 'r') as file: +# text = file.read() +# for match in pat.finditer(text): +# attr = match.groupdict()['attr'] +# if attr in ub.__all__: +# pkg_to_hist[name][attr] += 1 + +# hist_iter = iter(pkg_to_hist.values()) +# usage = next(hist_iter).copy() +# for other in hist_iter: +# for k, v in other.items(): +# usage[k] += v +# for attr in ub.__all__: +# usage[attr] += 0 + +# for name in pkg_to_hist.keys(): +# pkg_to_hist[name] = ub.odict(sorted(pkg_to_hist[name].items(), key=lambda t: t[1])[::-1]) + +# usage = ub.odict(sorted(usage.items(), key=lambda t: t[1])[::-1]) + +# if config['print_packages']: +# print(ub.repr2(pkg_to_hist, nl=2)) + +# if config['remove_zeros']: +# for k, v in list(usage.items()): +# if v == 0: +# usage.pop(k) + +# if config['hardcoded_ubelt_hack']: +# blocklist = [ +# 'progiter', 'timerit', 'orderedset', +# ] +# for k in list(usage): +# if k in blocklist: +# usage.pop(k, None) +# elif k.startswith('util_'): +# usage.pop(k, None) +# elif k.startswith('_util_'): +# usage.pop(k, None) +# # ub._util_deprecated +# # from ubelt import _util_deprecated +# # if k in dir(_util_deprecated): +# # usage.pop(k, None) + +# if 1: +# # Renamed Aliases +# usage['urepr'] += usage.pop('repr2') +# usage['ReprExtensions'] += usage.pop('FormatterExtensions') + +# usage = ub.udict(usage).sorted_values(reverse=True) + +# print(ub.repr2(usage, nl=1)) +# return usage + + +# if __name__ == '__main__': +# """ +# For Me: +# ~/internal/dev/pkg_usage_stats_update.sh + +# CommandLine: +# python ~/code/ubelt/dev/maintain/count_usage_freq.py --help +# python ~/code/ubelt/dev/maintain/count_usage_freq.py --remove_zeros=False --print_packages=True +# """ +# count_package_usage() diff --git a/dev/maintain/gen_api_for_docs.py b/dev/maintain/gen_api_for_docs.py old mode 100644 new mode 100755 index 5dd382cc6..de04899b9 --- a/dev/maintain/gen_api_for_docs.py +++ b/dev/maintain/gen_api_for_docs.py @@ -1,20 +1,141 @@ +#!/usr/bin/env python -def count_ubelt_usage(): +import scriptconfig as scfg + + +class UsageConfig(scfg.Config): + default = { + 'print_packages': False, + 'remove_zeros': False, + 'hardcoded_ubelt_hack': True, + 'extra_modnames': [], + } + + +def count_package_usage(modname): + import ubelt as ub + import glob + from os.path import join + import re + config = UsageConfig(cmdline=True) + + names = [ + 'xdoctest', 'netharn', 'xdev', 'xinspect', 'xcookie', 'ndsampler', + 'kwarray', 'kwimage', 'kwplot', 'kwcoco', + 'scriptconfig', 'vimtk', + 'mkinit', 'futures_actors', 'graphid', + + 'kwutil', 'git_well', 'line_profiler', 'delayed_image', 'simple_dvc', + 'pypogo', + + 'ibeis', 'plottool_ibeis', 'guitool_ibeis', 'utool', 'dtool_ibeis', + 'vtool_ibeis', 'hesaff', 'torch_liberator', 'liberator', + ] + config['extra_modnames'] + + code_repos = [ub.Path('~/code').expand() / name for name in names] + repo_dpaths = code_repos + [ + # ub.Path('~/local').expand(), + ub.Path('~/misc').expand(), + ] + all_fpaths = [] + for repo_dpath in repo_dpaths: + name = repo_dpath.stem + fpaths = glob.glob(join(repo_dpath, '**', '*.py'), recursive=True) + for fpath in fpaths: + all_fpaths.append((name, fpath)) + + pat = re.compile(r'\bub\.(?P[a-zA-Z_][A-Za-z_0-9]*)\b') + + modname = modname + module = ub.import_module_from_name(modname) + package_name = module.__name__ + package_allvar = module.__all__ + + pat = re.compile(r'\b' + package_name + r'\.(?P[a-zA-Z_][A-Za-z_0-9]*)\b') + + pkg_to_hist = ub.ddict(lambda: ub.ddict(int)) + for name, fpath in ub.ProgIter(all_fpaths): + with open(fpath, 'r') as file: + text = file.read() + for match in pat.finditer(text): + attr = match.groupdict()['attr'] + if attr in package_allvar: + pkg_to_hist[name][attr] += 1 + + hist_iter = iter(pkg_to_hist.values()) + usage = next(hist_iter).copy() + for other in hist_iter: + for k, v in other.items(): + usage[k] += v + for attr in package_allvar: + usage[attr] += 0 + + for name in pkg_to_hist.keys(): + pkg_to_hist[name] = ub.odict(sorted(pkg_to_hist[name].items(), key=lambda t: t[1])[::-1]) + + usage = ub.odict(sorted(usage.items(), key=lambda t: t[1])[::-1]) + + if config['print_packages']: + print(ub.repr2(pkg_to_hist, nl=2)) + + if config['remove_zeros']: + for k, v in list(usage.items()): + if v == 0: + usage.pop(k) + + if config['hardcoded_ubelt_hack']: + blocklist = [ + 'progiter', 'timerit', 'orderedset', + ] + for k in list(usage): + if k in blocklist: + usage.pop(k, None) + elif k.startswith('util_'): + usage.pop(k, None) + elif k.startswith('_util_'): + usage.pop(k, None) + # ub._util_deprecated + # from ubelt import _util_deprecated + # if k in dir(_util_deprecated): + # usage.pop(k, None) + + if 1: + # Renamed Aliases + try: + usage['urepr'] += usage.pop('repr2') + usage['ReprExtensions'] += usage.pop('FormatterExtensions') + except Exception: + ... + + usage = ub.udict(usage).sorted_values(reverse=True) + + print(ub.repr2(usage, nl=1)) + return usage + + +def gen_api_for_docs(modname): """ import sys, ubelt sys.path.append(ubelt.expandpath('~/code/ubelt/dev/maintain')) from gen_api_for_docs import * # NOQA """ - from count_usage_freq import count_ubelt_usage import ubelt as ub - usage = count_ubelt_usage() + usage = count_package_usage(modname) + + module = ub.import_module_from_name(modname) + attrnames = module.__all__ + if hasattr(module, '__protected__'): + # Hack for lazy imports + for subattr in module.__protected__: + submod = ub.import_module_from_name(modname + '.' + subattr) + setattr(module, subattr, submod) + attrnames += module.__protected__ # Reorgnaize data to contain more information rows = [] unseen = usage.copy() - import ubelt as ub - for attrname in ub.__all__: - member = getattr(ub, attrname) + for attrname in attrnames: + member = getattr(module, attrname) submembers = getattr(member, '__all__', None) if attrname.startswith('util_'): if not submembers: @@ -22,10 +143,10 @@ def count_ubelt_usage(): submembers = _extract_attributes(member.__file__) if submembers: for subname in submembers: - parent_module = 'ubelt.{}'.format(attrname) - short_name = 'ubelt.{subname}'.format(**locals()) + parent_module = f'{modname}.{attrname}' + short_name = '{modname}.{subname}'.format(**locals()) full_name = '{parent_module}.{subname}'.format(**locals()) - url = 'https://ubelt.readthedocs.io/en/latest/{parent_module}.html#{full_name}'.format(**locals()) + url = 'https://{modname}.readthedocs.io/en/latest/{parent_module}.html#{full_name}'.format(**locals()) rst_ref = ':func:`{short_name}<{full_name}>`'.format(**locals()) url_ref = '`{short_name} <{url}>`__'.format(**locals()) rows.append({ @@ -66,7 +187,7 @@ def count_ubelt_usage(): for key, value in usage.items(): infos = attr_to_infos[key] if len(infos) == 0: - print(column_fmt.format(':func:`ubelt.' + key + '`', value)) + print(column_fmt.format(f':func:`{modname}.' + key + '`', value)) else: if len(infos) != 1: print('infos = {}'.format(ub.urepr(infos, nl=1))) @@ -81,18 +202,21 @@ def count_ubelt_usage(): print(ub.indent('usage stats = ' + ub.repr2(kwarray.stats_dict( raw_scores, median=True, sum=True), nl=1))) - for attrname in ub.__all__: - member = getattr(ub, attrname) + for attrname in attrnames: + member = getattr(module, attrname) submembers = getattr(member, '__all__', None) - if attrname.startswith('util_'): - if not submembers: - from mkinit.static_mkinit import _extract_attributes + # if attrname.startswith('util_'): + if not submembers: + from mkinit.static_mkinit import _extract_attributes + try: submembers = _extract_attributes(member.__file__) + except AttributeError: + pass if submembers: - parent_module = 'ubelt.{}'.format(attrname) + parent_module = f'{modname}.{attrname}' title = ':mod:`{}`'.format(parent_module) print('\n' + title) @@ -100,8 +224,8 @@ def count_ubelt_usage(): for subname in submembers: if not subname.startswith('_'): rst_ref = ( - ':func:`<{parent_module}.{subname}>`' - ).format(subname=subname, parent_module=parent_module) + f':func:`<{modname}.{subname}><{parent_module}.{subname}>`' + ) print(rst_ref) submembers = dir(member) @@ -127,4 +251,4 @@ def count_ubelt_usage(): # Then edit: TODO make less manual ~/code/ubelt/docs/source/function_usefulness.rst """ - count_ubelt_usage() + gen_api_for_docs('ubelt') diff --git a/dev/setup_secrets.sh b/dev/setup_secrets.sh index 7b994cae0..6321e5ac5 100644 --- a/dev/setup_secrets.sh +++ b/dev/setup_secrets.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash __doc__=' ============================ SETUP CI SECRET INSTRUCTIONS diff --git a/docs/source/modules.rst b/docs/source/auto/modules.rst similarity index 100% rename from docs/source/modules.rst rename to docs/source/auto/modules.rst diff --git a/docs/source/auto/ubelt.__main__.rst b/docs/source/auto/ubelt.__main__.rst new file mode 100644 index 000000000..8307f8ead --- /dev/null +++ b/docs/source/auto/ubelt.__main__.rst @@ -0,0 +1,8 @@ +ubelt.\_\_main\_\_ module +========================= + +.. automodule:: ubelt.__main__ + :members: + :undoc-members: + :show-inheritance: + :private-members: diff --git a/docs/source/auto/ubelt._win32_links.rst b/docs/source/auto/ubelt._win32_links.rst new file mode 100644 index 000000000..cfe0f6169 --- /dev/null +++ b/docs/source/auto/ubelt._win32_links.rst @@ -0,0 +1,8 @@ +ubelt.\_win32\_links module +=========================== + +.. automodule:: ubelt._win32_links + :members: + :undoc-members: + :show-inheritance: + :private-members: diff --git a/docs/source/ubelt.orderedset.rst b/docs/source/auto/ubelt.orderedset.rst similarity index 86% rename from docs/source/ubelt.orderedset.rst rename to docs/source/auto/ubelt.orderedset.rst index 03c04fda3..f6c4bb50b 100644 --- a/docs/source/ubelt.orderedset.rst +++ b/docs/source/auto/ubelt.orderedset.rst @@ -5,3 +5,4 @@ ubelt.orderedset module :members: :undoc-members: :show-inheritance: + :private-members: diff --git a/docs/source/ubelt.progiter.rst b/docs/source/auto/ubelt.progiter.rst similarity index 86% rename from docs/source/ubelt.progiter.rst rename to docs/source/auto/ubelt.progiter.rst index 6eaa0b320..383f8fb21 100644 --- a/docs/source/ubelt.progiter.rst +++ b/docs/source/auto/ubelt.progiter.rst @@ -5,3 +5,4 @@ ubelt.progiter module :members: :undoc-members: :show-inheritance: + :private-members: diff --git a/docs/source/ubelt.rst b/docs/source/auto/ubelt.rst similarity index 92% rename from docs/source/ubelt.rst rename to docs/source/auto/ubelt.rst index 6b3bed769..f3443b7bc 100644 --- a/docs/source/ubelt.rst +++ b/docs/source/auto/ubelt.rst @@ -7,6 +7,8 @@ Submodules .. toctree:: :maxdepth: 4 + ubelt.__main__ + ubelt._win32_links ubelt.orderedset ubelt.progiter ubelt.util_arg @@ -44,3 +46,4 @@ Module contents :members: :undoc-members: :show-inheritance: + :private-members: diff --git a/docs/source/ubelt.util_arg.rst b/docs/source/auto/ubelt.util_arg.rst similarity index 86% rename from docs/source/ubelt.util_arg.rst rename to docs/source/auto/ubelt.util_arg.rst index ff92597ad..bd5a6c169 100644 --- a/docs/source/ubelt.util_arg.rst +++ b/docs/source/auto/ubelt.util_arg.rst @@ -5,3 +5,4 @@ ubelt.util\_arg module :members: :undoc-members: :show-inheritance: + :private-members: diff --git a/docs/source/ubelt.util_cache.rst b/docs/source/auto/ubelt.util_cache.rst similarity index 86% rename from docs/source/ubelt.util_cache.rst rename to docs/source/auto/ubelt.util_cache.rst index 8af993c72..f803857f5 100644 --- a/docs/source/ubelt.util_cache.rst +++ b/docs/source/auto/ubelt.util_cache.rst @@ -5,3 +5,4 @@ ubelt.util\_cache module :members: :undoc-members: :show-inheritance: + :private-members: diff --git a/docs/source/ubelt.util_cmd.rst b/docs/source/auto/ubelt.util_cmd.rst similarity index 86% rename from docs/source/ubelt.util_cmd.rst rename to docs/source/auto/ubelt.util_cmd.rst index 8c23b24cb..468a5e680 100644 --- a/docs/source/ubelt.util_cmd.rst +++ b/docs/source/auto/ubelt.util_cmd.rst @@ -5,3 +5,4 @@ ubelt.util\_cmd module :members: :undoc-members: :show-inheritance: + :private-members: diff --git a/docs/source/ubelt.util_colors.rst b/docs/source/auto/ubelt.util_colors.rst similarity index 87% rename from docs/source/ubelt.util_colors.rst rename to docs/source/auto/ubelt.util_colors.rst index c2441e7d7..d99d21f63 100644 --- a/docs/source/ubelt.util_colors.rst +++ b/docs/source/auto/ubelt.util_colors.rst @@ -5,3 +5,4 @@ ubelt.util\_colors module :members: :undoc-members: :show-inheritance: + :private-members: diff --git a/docs/source/ubelt.util_const.rst b/docs/source/auto/ubelt.util_const.rst similarity index 86% rename from docs/source/ubelt.util_const.rst rename to docs/source/auto/ubelt.util_const.rst index 8fa4793c7..3cad40cea 100644 --- a/docs/source/ubelt.util_const.rst +++ b/docs/source/auto/ubelt.util_const.rst @@ -5,3 +5,4 @@ ubelt.util\_const module :members: :undoc-members: :show-inheritance: + :private-members: diff --git a/docs/source/ubelt.util_deprecate.rst b/docs/source/auto/ubelt.util_deprecate.rst similarity index 87% rename from docs/source/ubelt.util_deprecate.rst rename to docs/source/auto/ubelt.util_deprecate.rst index 851be9634..2c99ca229 100644 --- a/docs/source/ubelt.util_deprecate.rst +++ b/docs/source/auto/ubelt.util_deprecate.rst @@ -5,3 +5,4 @@ ubelt.util\_deprecate module :members: :undoc-members: :show-inheritance: + :private-members: diff --git a/docs/source/ubelt.util_dict.rst b/docs/source/auto/ubelt.util_dict.rst similarity index 86% rename from docs/source/ubelt.util_dict.rst rename to docs/source/auto/ubelt.util_dict.rst index 016e9eba8..8b481eb8b 100644 --- a/docs/source/ubelt.util_dict.rst +++ b/docs/source/auto/ubelt.util_dict.rst @@ -5,3 +5,4 @@ ubelt.util\_dict module :members: :undoc-members: :show-inheritance: + :private-members: diff --git a/docs/source/ubelt.util_download.rst b/docs/source/auto/ubelt.util_download.rst similarity index 87% rename from docs/source/ubelt.util_download.rst rename to docs/source/auto/ubelt.util_download.rst index b353e4f5b..7afc2aa53 100644 --- a/docs/source/ubelt.util_download.rst +++ b/docs/source/auto/ubelt.util_download.rst @@ -5,3 +5,4 @@ ubelt.util\_download module :members: :undoc-members: :show-inheritance: + :private-members: diff --git a/docs/source/ubelt.util_download_manager.rst b/docs/source/auto/ubelt.util_download_manager.rst similarity index 89% rename from docs/source/ubelt.util_download_manager.rst rename to docs/source/auto/ubelt.util_download_manager.rst index 8306edb1b..52675955d 100644 --- a/docs/source/ubelt.util_download_manager.rst +++ b/docs/source/auto/ubelt.util_download_manager.rst @@ -5,3 +5,4 @@ ubelt.util\_download\_manager module :members: :undoc-members: :show-inheritance: + :private-members: diff --git a/docs/source/ubelt.util_format.rst b/docs/source/auto/ubelt.util_format.rst similarity index 87% rename from docs/source/ubelt.util_format.rst rename to docs/source/auto/ubelt.util_format.rst index 9f3441d91..a8fb72b54 100644 --- a/docs/source/ubelt.util_format.rst +++ b/docs/source/auto/ubelt.util_format.rst @@ -5,3 +5,4 @@ ubelt.util\_format module :members: :undoc-members: :show-inheritance: + :private-members: diff --git a/docs/source/ubelt.util_func.rst b/docs/source/auto/ubelt.util_func.rst similarity index 86% rename from docs/source/ubelt.util_func.rst rename to docs/source/auto/ubelt.util_func.rst index b30efdc05..bd7077ba9 100644 --- a/docs/source/ubelt.util_func.rst +++ b/docs/source/auto/ubelt.util_func.rst @@ -5,3 +5,4 @@ ubelt.util\_func module :members: :undoc-members: :show-inheritance: + :private-members: diff --git a/docs/source/ubelt.util_futures.rst b/docs/source/auto/ubelt.util_futures.rst similarity index 87% rename from docs/source/ubelt.util_futures.rst rename to docs/source/auto/ubelt.util_futures.rst index a7b8d81fa..3938f248c 100644 --- a/docs/source/ubelt.util_futures.rst +++ b/docs/source/auto/ubelt.util_futures.rst @@ -5,3 +5,4 @@ ubelt.util\_futures module :members: :undoc-members: :show-inheritance: + :private-members: diff --git a/docs/source/ubelt.util_hash.rst b/docs/source/auto/ubelt.util_hash.rst similarity index 86% rename from docs/source/ubelt.util_hash.rst rename to docs/source/auto/ubelt.util_hash.rst index 564729ff9..5245aa324 100644 --- a/docs/source/ubelt.util_hash.rst +++ b/docs/source/auto/ubelt.util_hash.rst @@ -5,3 +5,4 @@ ubelt.util\_hash module :members: :undoc-members: :show-inheritance: + :private-members: diff --git a/docs/source/ubelt.util_import.rst b/docs/source/auto/ubelt.util_import.rst similarity index 87% rename from docs/source/ubelt.util_import.rst rename to docs/source/auto/ubelt.util_import.rst index 8b2c4c3d9..26a7d2961 100644 --- a/docs/source/ubelt.util_import.rst +++ b/docs/source/auto/ubelt.util_import.rst @@ -5,3 +5,4 @@ ubelt.util\_import module :members: :undoc-members: :show-inheritance: + :private-members: diff --git a/docs/source/ubelt.util_indexable.rst b/docs/source/auto/ubelt.util_indexable.rst similarity index 87% rename from docs/source/ubelt.util_indexable.rst rename to docs/source/auto/ubelt.util_indexable.rst index 11eca9a17..f52370070 100644 --- a/docs/source/ubelt.util_indexable.rst +++ b/docs/source/auto/ubelt.util_indexable.rst @@ -5,3 +5,4 @@ ubelt.util\_indexable module :members: :undoc-members: :show-inheritance: + :private-members: diff --git a/docs/source/ubelt.util_io.rst b/docs/source/auto/ubelt.util_io.rst similarity index 86% rename from docs/source/ubelt.util_io.rst rename to docs/source/auto/ubelt.util_io.rst index 0bd247616..99fb4ac8c 100644 --- a/docs/source/ubelt.util_io.rst +++ b/docs/source/auto/ubelt.util_io.rst @@ -5,3 +5,4 @@ ubelt.util\_io module :members: :undoc-members: :show-inheritance: + :private-members: diff --git a/docs/source/ubelt.util_links.rst b/docs/source/auto/ubelt.util_links.rst similarity index 86% rename from docs/source/ubelt.util_links.rst rename to docs/source/auto/ubelt.util_links.rst index e0a21fbe7..d028a75a8 100644 --- a/docs/source/ubelt.util_links.rst +++ b/docs/source/auto/ubelt.util_links.rst @@ -5,3 +5,4 @@ ubelt.util\_links module :members: :undoc-members: :show-inheritance: + :private-members: diff --git a/docs/source/ubelt.util_list.rst b/docs/source/auto/ubelt.util_list.rst similarity index 86% rename from docs/source/ubelt.util_list.rst rename to docs/source/auto/ubelt.util_list.rst index dd17ba052..573624656 100644 --- a/docs/source/ubelt.util_list.rst +++ b/docs/source/auto/ubelt.util_list.rst @@ -5,3 +5,4 @@ ubelt.util\_list module :members: :undoc-members: :show-inheritance: + :private-members: diff --git a/docs/source/ubelt.util_memoize.rst b/docs/source/auto/ubelt.util_memoize.rst similarity index 87% rename from docs/source/ubelt.util_memoize.rst rename to docs/source/auto/ubelt.util_memoize.rst index e6e12fadf..0ea9ffeb2 100644 --- a/docs/source/ubelt.util_memoize.rst +++ b/docs/source/auto/ubelt.util_memoize.rst @@ -5,3 +5,4 @@ ubelt.util\_memoize module :members: :undoc-members: :show-inheritance: + :private-members: diff --git a/docs/source/ubelt.util_mixins.rst b/docs/source/auto/ubelt.util_mixins.rst similarity index 87% rename from docs/source/ubelt.util_mixins.rst rename to docs/source/auto/ubelt.util_mixins.rst index 797125801..6659f8fbe 100644 --- a/docs/source/ubelt.util_mixins.rst +++ b/docs/source/auto/ubelt.util_mixins.rst @@ -5,3 +5,4 @@ ubelt.util\_mixins module :members: :undoc-members: :show-inheritance: + :private-members: diff --git a/docs/source/ubelt.util_path.rst b/docs/source/auto/ubelt.util_path.rst similarity index 86% rename from docs/source/ubelt.util_path.rst rename to docs/source/auto/ubelt.util_path.rst index 88d5daadc..12c8a39c2 100644 --- a/docs/source/ubelt.util_path.rst +++ b/docs/source/auto/ubelt.util_path.rst @@ -5,3 +5,4 @@ ubelt.util\_path module :members: :undoc-members: :show-inheritance: + :private-members: diff --git a/docs/source/ubelt.util_platform.rst b/docs/source/auto/ubelt.util_platform.rst similarity index 87% rename from docs/source/ubelt.util_platform.rst rename to docs/source/auto/ubelt.util_platform.rst index 1b2d7bef4..70a6df2b6 100644 --- a/docs/source/ubelt.util_platform.rst +++ b/docs/source/auto/ubelt.util_platform.rst @@ -5,3 +5,4 @@ ubelt.util\_platform module :members: :undoc-members: :show-inheritance: + :private-members: diff --git a/docs/source/ubelt.util_repr.rst b/docs/source/auto/ubelt.util_repr.rst similarity index 86% rename from docs/source/ubelt.util_repr.rst rename to docs/source/auto/ubelt.util_repr.rst index b680d8b0e..be661c9ad 100644 --- a/docs/source/ubelt.util_repr.rst +++ b/docs/source/auto/ubelt.util_repr.rst @@ -5,3 +5,4 @@ ubelt.util\_repr module :members: :undoc-members: :show-inheritance: + :private-members: diff --git a/docs/source/ubelt.util_str.rst b/docs/source/auto/ubelt.util_str.rst similarity index 86% rename from docs/source/ubelt.util_str.rst rename to docs/source/auto/ubelt.util_str.rst index cdc24ebff..31dd17064 100644 --- a/docs/source/ubelt.util_str.rst +++ b/docs/source/auto/ubelt.util_str.rst @@ -5,3 +5,4 @@ ubelt.util\_str module :members: :undoc-members: :show-inheritance: + :private-members: diff --git a/docs/source/ubelt.util_stream.rst b/docs/source/auto/ubelt.util_stream.rst similarity index 87% rename from docs/source/ubelt.util_stream.rst rename to docs/source/auto/ubelt.util_stream.rst index 5a8573bf4..6ca237ce2 100644 --- a/docs/source/ubelt.util_stream.rst +++ b/docs/source/auto/ubelt.util_stream.rst @@ -5,3 +5,4 @@ ubelt.util\_stream module :members: :undoc-members: :show-inheritance: + :private-members: diff --git a/docs/source/ubelt.util_time.rst b/docs/source/auto/ubelt.util_time.rst similarity index 86% rename from docs/source/ubelt.util_time.rst rename to docs/source/auto/ubelt.util_time.rst index e24f606a5..8805d0576 100644 --- a/docs/source/ubelt.util_time.rst +++ b/docs/source/auto/ubelt.util_time.rst @@ -5,3 +5,4 @@ ubelt.util\_time module :members: :undoc-members: :show-inheritance: + :private-members: diff --git a/docs/source/ubelt.util_zip.rst b/docs/source/auto/ubelt.util_zip.rst similarity index 86% rename from docs/source/ubelt.util_zip.rst rename to docs/source/auto/ubelt.util_zip.rst index 6ba943b9b..67ef17ae8 100644 --- a/docs/source/ubelt.util_zip.rst +++ b/docs/source/auto/ubelt.util_zip.rst @@ -5,3 +5,4 @@ ubelt.util\_zip module :members: :undoc-members: :show-inheritance: + :private-members: diff --git a/docs/source/conf.py b/docs/source/conf.py index 9caea3d2c..7f247303f 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -1,7 +1,7 @@ """ Notes: Based on template code in: - ~/code/xcookie/xcookie/builders/docs_conf.py + ~/code/xcookie/xcookie/builders/docs.py ~/code/xcookie/xcookie/rc/conf_ext.py http://docs.readthedocs.io/en/latest/getting_started.html @@ -17,10 +17,13 @@ # need to edit the conf.py cd ~/code/ubelt/docs - sphinx-apidoc --private -f -o ~/code/ubelt/docs/source ~/code/ubelt/ubelt --separate + sphinx-apidoc --private --separate -f -o ~/code/ubelt/docs/source/auto ~/code/ubelt/ubelt + + # Note: the module should importable before running this + # (e.g. install it in developer mode or munge the PYTHONPATH) make html - git add source/*.rst + git add source/auto/*.rst Also: To turn on PR checks @@ -50,15 +53,38 @@ https://readthedocs.org/dashboard/ubelt/integrations/create/ Then add gitlab incoming webhook and copy the URL (make sure - you copy the real url and not the text so https is included). + you copy the real url and not the text so https is included), + specifically: + + In the "Integration type:" dropdown menu, select + "Gitlab incoming webhook" + + Click "Add integration" + + Copy the text in the "Webhook URL" box to be used later. + + Copy the text in the "Secret" box to be used later. Then go to https://github.com/Erotemic/ubelt/hooks - and add the URL + Click "Add new webhook". + + Copy the text previously saved from the "Webhook URL" box + in the readthedocs form into the "URL" box in the gitlab + form. - select push, tag, and merge request + Copy the text previously saved from the "Secret" box + in the readthedocs form into the "Secret token" box in the + gitlab form. + + For trigger permissions select the following checkboxes: + push events, + tag push events, + merge request events + + Click the "Add webhook" button. See Docs for more details https://docs.readthedocs.io/en/stable/integrations.html @@ -110,14 +136,19 @@ def visit_Assign(self, node): return visitor.version project = 'ubelt' -copyright = '2023, Jon Crall' +copyright = '2024, Jon Crall' author = 'Jon Crall' modname = 'ubelt' -modpath = join(dirname(dirname(dirname(__file__))), 'ubelt', '__init__.py') +repo_dpath = dirname(dirname(dirname(__file__))) +mod_dpath = join(repo_dpath, 'ubelt') +src_dpath = dirname(mod_dpath) +modpath = join(mod_dpath, '__init__.py') release = parse_version(modpath) version = '.'.join(release.split('.')[0:2]) +# Hack to ensure the module is importable +# sys.path.insert(0, os.path.abspath(src_dpath)) # -- General configuration --------------------------------------------------- @@ -136,8 +167,8 @@ def visit_Assign(self, node): 'sphinx.ext.napoleon', 'sphinx.ext.todo', 'sphinx.ext.viewcode', - # 'myst_parser', # TODO - + 'myst_parser', # For markdown docs + 'sphinx.ext.imgconverter', # For building latexpdf 'sphinx.ext.githubpages', # 'sphinxcontrib.redirects', 'sphinx_reredirects', @@ -148,8 +179,21 @@ def visit_Assign(self, node): napoleon_use_param = False napoleon_use_ivar = True +#autoapi_type = 'python' +#autoapi_dirs = [mod_dpath] + autodoc_inherit_docstrings = False +# Hack for geowatch, todo configure +autosummary_mock_imports = [ + 'geowatch.utils.lightning_ext._jsonargparse_ext_ge_4_24_and_lt_4_xx', + 'geowatch.utils.lightning_ext._jsonargparse_ext_ge_4_22_and_lt_4_24', + 'geowatch.utils.lightning_ext._jsonargparse_ext_ge_4_21_and_lt_4_22', + 'geowatch.tasks.fusion.datamodules.temporal_sampling.affinity_sampling', + 'geowatch.tasks.depth_pcd.model', + 'geowatch.tasks.cold.export_change_map', +] + autodoc_member_order = 'bysource' autoclass_content = 'both' # autodoc_mock_imports = ['torch', 'torchvision', 'visdom'] @@ -163,6 +207,9 @@ def visit_Assign(self, node): # autoapi_dirs = [f'../../src/{modname}'] # autoapi_keep_files = True +# References: +# https://stackoverflow.com/questions/21538983/specifying-targets-for-intersphinx-links-to-numpy-scipy-and-matplotlib + intersphinx_mapping = { # 'pytorch': ('http://pytorch.org/docs/master/', None), 'python': ('https://docs.python.org/3', None), @@ -181,10 +228,20 @@ def visit_Assign(self, node): 'scriptconfig': ('https://scriptconfig.readthedocs.io/en/latest/', None), 'rich': ('https://rich.readthedocs.io/en/latest/', None), + 'numpy': ('https://numpy.org/doc/stable/', None), + 'sympy': ('https://docs.sympy.org/latest/', None), + 'scikit-learn': ('https://scikit-learn.org/stable/', None), + 'pandas': ('https://pandas.pydata.org/docs/', None), + 'matplotlib': ('https://matplotlib.org/stable/', None), + 'pytest': ('https://docs.pytest.org/en/latest/', None), + 'platformdirs': ('https://platformdirs.readthedocs.io/en/latest/', None), + + 'timerit': ('https://timerit.readthedocs.io/en/latest/', None), + 'progiter': ('https://progiter.readthedocs.io/en/latest/', None), + 'dateutil': ('https://dateutil.readthedocs.io/en/latest/', None), # 'pytest._pytest.doctest': ('https://docs.pytest.org/en/latest/_modules/_pytest/doctest.html', None), # 'colorama': ('https://pypi.org/project/colorama/', None), - # 'numpy': ('http://docs.scipy.org/doc/numpy/', None), # 'cv2' : ('http://docs.opencv.org/2.4/', None), # 'h5py' : ('http://docs.h5py.org/en/latest/', None) } @@ -246,6 +303,7 @@ def visit_Assign(self, node): html_theme_options = { 'collapse_navigation': False, 'display_version': True, + 'navigation_depth': -1, # 'logo_only': True, } # html_logo = '.static/ubelt.svg' @@ -275,6 +333,21 @@ def visit_Assign(self, node): # -- Options for LaTeX output ------------------------------------------------ +# References: +# https://tex.stackexchange.com/questions/546246/centos-8-the-font-freeserif-cannot-be-found + +""" +# https://www.sphinx-doc.org/en/master/usage/builders/index.html#sphinx.builders.latex.LaTeXBuilder +# https://tex.stackexchange.com/a/570691/83399 +sudo apt install fonts-freefont-otf texlive-luatex texlive-latex-extra texlive-fonts-recommended texlive-latex-recommended tex-gyre latexmk +make latexpdf LATEXMKOPTS="-shell-escape --synctex=-1 -src-specials -interaction=nonstopmode" +make latexpdf LATEXMKOPTS="-lualatex -interaction=nonstopmode" +make LATEXMKOPTS="-lualatex -interaction=nonstopmode" + +""" +# latex_engine = 'lualatex' +# latex_engine = 'xelatex' + latex_elements = { # The paper size ('letterpaper' or 'a4paper'). # @@ -330,13 +403,24 @@ def visit_Assign(self, node): from typing import Any, List # NOQA +# HACK TO PREVENT EXCESSIVE TIME. +# TODO: FIXME FOR REAL +MAX_TIME_MINUTES = None +if MAX_TIME_MINUTES: + import ubelt # NOQA + TIMER = ubelt.Timer() + TIMER.tic() + + class PatchedPythonDomain(PythonDomain): """ References: https://github.com/sphinx-doc/sphinx/issues/3866 """ def resolve_xref(self, env, fromdocname, builder, typ, target, node, contnode): - # TODO: can use this to resolve references nicely + """ + Helps to resolves cross-references + """ if target.startswith('ub.'): target = 'ubelt.' + target[3] if target.startswith('xdoc.'): @@ -353,6 +437,7 @@ class GoogleStyleDocstringProcessor: """ def __init__(self, autobuild=1): + self.debug = 0 self.registry = {} if autobuild: self._register_builtins() @@ -407,7 +492,7 @@ def benchmark(lines): redone = new_text.split('\n') new_lines.extend(redone) # import ubelt as ub - # print('new_lines = {}'.format(ub.repr2(new_lines, nl=1))) + # print('new_lines = {}'.format(ub.urepr(new_lines, nl=1))) # new_lines.append('') return new_lines @@ -421,6 +506,17 @@ def text_art(lines): new_lines.extend(lines[1:]) return new_lines + # @self.register_section(tag='TODO', alias=['.. todo::']) + # def todo_section(lines): + # """ + # Fixup todo sections + # """ + # import xdev + # xdev.embed() + # import ubelt as ub + # print('lines = {}'.format(ub.urepr(lines, nl=1))) + # return new_lines + @self.register_section(tag='Ignore') def ignore(lines): return [] @@ -531,10 +627,12 @@ def process_docstring_callback(self, app, what_: str, name: str, obj: Any, https://www.sphinx-doc.org/en/1.5.1/_modules/sphinx/ext/autodoc.html https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html """ - # print(f'name={name}') + if self.debug: + print(f'ProcessDocstring: name={name}, what_={what_}, num_lines={len(lines)}') + # print('BEFORE:') # import ubelt as ub - # print('lines = {}'.format(ub.repr2(lines, nl=1))) + # print('lines = {}'.format(ub.urepr(lines, nl=1))) self.process(lines) @@ -547,8 +645,12 @@ def process_docstring_callback(self, app, what_: str, name: str, obj: Any, # import xdev # xdev.embed() - RENDER_IMAGES = 0 - if RENDER_IMAGES: + render_doc_images = 0 + + if MAX_TIME_MINUTES and TIMER.toc() > (60 * MAX_TIME_MINUTES): + render_doc_images = False # FIXME too slow on RTD + + if render_doc_images: # DEVELOPING if any('REQUIRES(--show)' in line for line in lines): # import xdev @@ -610,7 +712,7 @@ def process_docstring_callback(self, app, what_: str, name: str, obj: Any, lines[edit_slice] = new_lines # print('AFTER:') - # print('lines = {}'.format(ub.repr2(lines, nl=1))) + # print('lines = {}'.format(ub.urepr(lines, nl=1))) # if name == 'kwimage.Affine.translate': # import sys @@ -858,27 +960,74 @@ class Skipped(Exception): insert_index = end_index else: raise KeyError(INSERT_AT) - lines.insert(insert_index, '.. image:: {}'.format(rel_to_root_fpath)) + lines.insert(insert_index, '.. image:: {}'.format('..' / rel_to_root_fpath)) + # lines.insert(insert_index, '.. image:: {}'.format(rel_to_root_fpath)) # lines.insert(insert_index, '.. image:: {}'.format(rel_to_static_fpath)) lines.insert(insert_index, '') +def postprocess_hyperlinks(app, doctree, docname): + """ + Extension to fixup hyperlinks. + This should be connected to the Sphinx application's + "autodoc-process-docstring" event. + """ + # Your hyperlink postprocessing logic here + from docutils import nodes + import pathlib + for node in doctree.traverse(nodes.reference): + if 'refuri' in node.attributes: + refuri = node.attributes['refuri'] + if '.rst' in refuri: + if 'source' in node.document: + fpath = pathlib.Path(node.document['source']) + parent_dpath = fpath.parent + if (parent_dpath / refuri).exists(): + node.attributes['refuri'] = refuri.replace('.rst', '.html') + else: + raise AssertionError + + +def fix_rst_todo_section(lines): + new_lines = [] + for line in lines: + ... + ... + + def setup(app): import sphinx app : sphinx.application.Sphinx = app app.add_domain(PatchedPythonDomain, override=True) + + app.connect("doctree-resolved", postprocess_hyperlinks) + docstring_processor = GoogleStyleDocstringProcessor() # https://stackoverflow.com/questions/26534184/can-sphinx-ignore-certain-tags-in-python-docstrings app.connect('autodoc-process-docstring', docstring_processor.process_docstring_callback) + def copy(src, dst): + import shutil + print(f'Copy {src} -> {dst}') + assert src.exists() + if not dst.parent.exists(): + dst.parent.mkdir() + shutil.copy(src, dst) + ### Hack for kwcoco: TODO: figure out a way for the user to configure this. HACK_FOR_KWCOCO = 0 if HACK_FOR_KWCOCO: import pathlib - import shutil - doc_outdir = pathlib.Path(app.outdir) - doc_srcdir = pathlib.Path(app.srcdir) - schema_src = (doc_srcdir / '../../kwcoco/coco_schema.json') - shutil.copy(schema_src, doc_outdir / 'coco_schema.json') - shutil.copy(schema_src, doc_srcdir / 'coco_schema.json') + doc_outdir = pathlib.Path(app.outdir) / 'auto' + doc_srcdir = pathlib.Path(app.srcdir) / 'auto' + + mod_dpath = doc_srcdir / '../../../kwcoco' + + src_fpath = (mod_dpath / 'coco_schema.json') + copy(src_fpath, doc_outdir / src_fpath.name) + copy(src_fpath, doc_srcdir / src_fpath.name) + + src_fpath = (mod_dpath / 'coco_schema_informal.rst') + copy(src_fpath, doc_outdir / src_fpath.name) + copy(src_fpath, doc_srcdir / src_fpath.name) return app diff --git a/docs/source/index.rst b/docs/source/index.rst index cbe86ccbe..08408a80d 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,10 +1,13 @@ -.. UBelt documentation master file, created by - sphinx-quickstart on Sun Apr 8 19:53:51 2018. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - :github_url: https://github.com/Erotemic/ubelt +.. The large version wont work because github strips rst image rescaling. https://i.imgur.com/AcWVroL.png + # TODO: Add a logo + .. image:: https://i.imgur.com/PoYIsWE.png + :height: 100px + :align: left + +.. Autogenerated by templates in /home/joncrall/code/xcookie/xcookie/builders/docs.py + .. The large version wont work because github strips rst image rescaling. https://i.imgur.com/AcWVroL.png .. image:: https://i.imgur.com/PoYIsWE.png @@ -20,13 +23,14 @@ UBelt documentation :show-inheritance: .. # Computed function usefulness -.. include:: function_usefulness.rst +.. include:: manual/function_usefulness.rst .. toctree:: :maxdepth: 8 :caption: Package Layout - ubelt + auto/ubelt + auto/modules Indices and tables @@ -34,3 +38,4 @@ Indices and tables * :ref:`genindex` * :ref:`modindex` +* :ref:`search` diff --git a/docs/source/function_usefulness.rst b/docs/source/manual/function_usefulness.rst similarity index 85% rename from docs/source/function_usefulness.rst rename to docs/source/manual/function_usefulness.rst index 58cfb8803..79762a143 100644 --- a/docs/source/function_usefulness.rst +++ b/docs/source/manual/function_usefulness.rst @@ -1,7 +1,7 @@ The API by usefulness ===================== -.. to help generate python ~/code/ubelt/dev/gen_api_for_docs.py --extra_modname=bioharn,watch --remove_zeros=False +.. to help generate python ~/code/ubelt/dev/maintain/gen_api_for_docs.py --extra_modname=bioharn,watch --remove_zeros=False Perhaps the most useful way to learn this API is to sort by "usefulness". I measure usefulness as the number of times I've used a particular function in @@ -11,102 +11,105 @@ my own code (excluding ubelt itself). ================================================================================= ================ Function name Usefulness ================================================================================= ================ -:func:`ubelt.urepr` 2893 -:func:`ubelt.Path` 992 -:func:`ubelt.ProgIter` 544 -:func:`ubelt.paragraph` 482 -:func:`ubelt.take` 387 -:func:`ubelt.codeblock` 358 -:func:`ubelt.expandpath` 331 -:func:`ubelt.cmd` 302 -:func:`ubelt.udict` 271 -:func:`ubelt.ensuredir` 256 -:func:`ubelt.odict` 253 -:func:`ubelt.iterable` 252 -:func:`ubelt.ddict` 238 -:func:`ubelt.NiceRepr` 221 -:func:`ubelt.NoParam` 216 -:func:`ubelt.map_vals` 215 -:func:`ubelt.flatten` 214 -:func:`ubelt.dzip` 200 -:func:`ubelt.oset` 198 -:func:`ubelt.peek` 196 -:func:`ubelt.argflag` 177 -:func:`ubelt.group_items` 171 -:func:`ubelt.hash_data` 165 -:func:`ubelt.grabdata` 131 -:func:`ubelt.argval` 125 -:func:`ubelt.Timer` 120 -:func:`ubelt.dict_isect` 113 -:func:`ubelt.dict_hist` 111 -:func:`ubelt.augpath` 106 -:func:`ubelt.identity` 106 -:func:`ubelt.ensure_app_cache_dir` 105 -:func:`ubelt.allsame` 102 -:func:`ubelt.memoize` 99 -:func:`ubelt.color_text` 98 -:func:`ubelt.dict_diff` 95 -:func:`ubelt.delete` 89 -:func:`ubelt.hzcat` 88 -:func:`ubelt.schedule_deprecation` 87 -:func:`ubelt.named_product` 85 -:func:`ubelt.compress` 85 -:func:`ubelt.IndexableWalker` 74 -:func:`ubelt.indent` 68 -:func:`ubelt.JobPool` 67 -:func:`ubelt.unique` 63 -:func:`ubelt.dict_union` 57 -:func:`ubelt.map_keys` 49 -:func:`ubelt.invert_dict` 48 -:func:`ubelt.iter_window` 46 -:func:`ubelt.timestamp` 46 -:func:`ubelt.argsort` 44 -:func:`ubelt.Cacher` 43 -:func:`ubelt.find_exe` 41 -:func:`ubelt.symlink` 41 -:func:`ubelt.dict_subset` 41 -:func:`ubelt.writeto` 40 -:func:`ubelt.find_duplicates` 39 -:func:`ubelt.chunks` 38 -:func:`ubelt.hash_file` 37 -:func:`ubelt.modname_to_modpath` 37 -:func:`ubelt.ensure_unicode` 33 -:func:`ubelt.memoize_property` 33 -:func:`ubelt.highlight_code` 33 -:func:`ubelt.sorted_vals` 32 -:func:`ubelt.CacheStamp` 30 -:func:`ubelt.WIN32` 28 -:func:`ubelt.import_module_from_name` 27 -:func:`ubelt.argmax` 27 +:func:`ubelt.urepr` 4327 +:func:`ubelt.Path` 2125 +:func:`ubelt.paragraph` 1349 +:func:`ubelt.ProgIter` 747 +:func:`ubelt.cmd` 657 +:func:`ubelt.codeblock` 611 +:func:`ubelt.udict` 603 +:func:`ubelt.expandpath` 508 +:func:`ubelt.take` 462 +:func:`ubelt.oset` 342 +:func:`ubelt.ddict` 341 +:func:`ubelt.iterable` 313 +:func:`ubelt.flatten` 303 +:func:`ubelt.group_items` 287 +:func:`ubelt.NiceRepr` 270 +:func:`ubelt.ensuredir` 267 +:func:`ubelt.map_vals` 265 +:func:`ubelt.peek` 262 +:func:`ubelt.NoParam` 248 +:func:`ubelt.dzip` 239 +:func:`ubelt.odict` 236 +:func:`ubelt.hash_data` 200 +:func:`ubelt.argflag` 184 +:func:`ubelt.grabdata` 161 +:func:`ubelt.dict_hist` 156 +:func:`ubelt.identity` 156 +:func:`ubelt.dict_isect` 152 +:func:`ubelt.Timer` 145 +:func:`ubelt.memoize` 142 +:func:`ubelt.argval` 134 +:func:`ubelt.allsame` 133 +:func:`ubelt.color_text` 129 +:func:`ubelt.schedule_deprecation` 123 +:func:`ubelt.augpath` 120 +:func:`ubelt.dict_diff` 117 +:func:`ubelt.IndexableWalker` 116 +:func:`ubelt.compress` 116 +:func:`ubelt.JobPool` 107 +:func:`ubelt.named_product` 104 +:func:`ubelt.hzcat` 90 +:func:`ubelt.delete` 88 +:func:`ubelt.unique` 84 +:func:`ubelt.WIN32` 78 +:func:`ubelt.dict_union` 76 +:func:`ubelt.symlink` 76 +:func:`ubelt.indent` 69 +:func:`ubelt.ensure_app_cache_dir` 67 +:func:`ubelt.iter_window` 62 +:func:`ubelt.invert_dict` 58 +:func:`ubelt.memoize_property` 57 +:func:`ubelt.import_module_from_name` 56 +:func:`ubelt.argsort` 55 +:func:`ubelt.timestamp` 54 +:func:`ubelt.modname_to_modpath` 53 +:func:`ubelt.find_duplicates` 53 +:func:`ubelt.hash_file` 51 +:func:`ubelt.find_exe` 50 +:func:`ubelt.map_keys` 50 +:func:`ubelt.dict_subset` 50 +:func:`ubelt.Cacher` 49 +:func:`ubelt.chunks` 47 +:func:`ubelt.sorted_vals` 40 +:func:`ubelt.CacheStamp` 38 +:func:`ubelt.highlight_code` 37 +:func:`ubelt.argmax` 36 +:func:`ubelt.writeto` 36 +:func:`ubelt.ensure_unicode` 32 +:func:`ubelt.sorted_keys` 30 +:func:`ubelt.memoize_method` 29 +:func:`ubelt.compatible` 24 +:func:`ubelt.import_module_from_path` 24 +:func:`ubelt.Executor` 23 :func:`ubelt.readfrom` 23 -:func:`ubelt.import_module_from_path` 22 -:func:`ubelt.compatible` 17 +:func:`ubelt.modpath_to_modname` 17 +:func:`ubelt.AutoDict` 17 :func:`ubelt.touch` 17 -:func:`ubelt.Executor` 16 -:func:`ubelt.memoize_method` 16 -:func:`ubelt.sorted_keys` 14 -:func:`ubelt.AutoDict` 11 +:func:`ubelt.inject_method` 14 +:func:`ubelt.timeparse` 13 +:func:`ubelt.ChDir` 11 :func:`ubelt.shrinkuser` 11 -:func:`ubelt.inject_method` 10 +:func:`ubelt.argmin` 10 :func:`ubelt.varied_values` 9 :func:`ubelt.split_modpath` 8 -:func:`ubelt.modpath_to_modname` 8 -:func:`ubelt.get_app_cache_dir` 8 -:func:`ubelt.zopen` 7 -:func:`ubelt.LINUX` 7 +:func:`ubelt.LINUX` 8 +:func:`ubelt.download` 7 +:func:`ubelt.NO_COLOR` 7 +:func:`ubelt.OrderedSet` 6 +:func:`ubelt.zopen` 6 :func:`ubelt.CaptureStdout` 6 -:func:`ubelt.download` 5 -:func:`ubelt.timeparse` 5 :func:`ubelt.DARWIN` 5 -:func:`ubelt.argmin` 5 +:func:`ubelt.boolmask` 4 :func:`ubelt.find_path` 4 -:func:`ubelt.indexable_allclose` 4 -:func:`ubelt.boolmask` 3 -:func:`ubelt.map_values` 2 +:func:`ubelt.get_app_cache_dir` 4 +:func:`ubelt.indexable_allclose` 3 +:func:`ubelt.UDict` 3 +:func:`ubelt.SetDict` 2 :func:`ubelt.AutoOrderedDict` 2 :func:`ubelt.argunique` 2 -:func:`ubelt.NO_COLOR` 2 -:func:`ubelt.UDict` 1 +:func:`ubelt.map_values` 1 :func:`ubelt.unique_flags` 1 :func:`ubelt.userhome` 0 :func:`ubelt.split_archive` 0 @@ -121,10 +124,8 @@ my own code (excluding ubelt itself). :func:`ubelt.ensure_app_config_dir` 0 :func:`ubelt.TempDir` 0 :func:`ubelt.TeeStringIO` 0 -:func:`ubelt.SetDict` 0 :func:`ubelt.ReprExtensions` 0 :func:`ubelt.POSIX` 0 -:func:`ubelt.OrderedSet` 0 :func:`ubelt.DownloadManager` 0 :func:`ubelt.CaptureStream` 0 ================================================================================= ================ @@ -132,16 +133,16 @@ my own code (excluding ubelt itself). .. code:: python usage stats = { - 'mean': 109.39655, - 'std': 292.5527, + 'mean': 164.10257, + 'std': 467.12064, 'min': 0.0, - 'max': 2893.0, - 'q_0.25': 5.0, - 'q_0.50': 37.0, - 'q_0.75': 106.0, - 'med': 37.0, - 'sum': 12690, - 'shape': (116,), + 'max': 4327.0, + 'q_0.25': 6.0, + 'q_0.50': 50.0, + 'q_0.75': 134.0, + 'med': 50.0, + 'sum': 19200, + 'shape': (117,), } :mod:`ubelt.orderedset` @@ -301,6 +302,7 @@ my own code (excluding ubelt itself). :func:`` :func:`` :func:`` +:func:`` :mod:`ubelt.util_platform` -------------------------- diff --git a/publish.sh b/publish.sh index 6a1d9c67d..237ee0d87 100755 --- a/publish.sh +++ b/publish.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash __doc__=' Script to publish a new version of this library on PyPI. @@ -24,6 +24,10 @@ Args: If True, sign the packages with a GPG key specified by `GPG_KEYID`. defaults to auto. + DO_OTS (bool) : + If True, make an opentimestamp for the package and signature (if + available) + DO_UPLOAD (bool) : If True, upload the packages to the pypi server specified by `TWINE_REPOSITORY_URL`. @@ -138,11 +142,21 @@ DO_UPLOAD=${DO_UPLOAD:=$ARG_1} DO_TAG=${DO_TAG:=$ARG_1} DO_GPG=${DO_GPG:="auto"} -# Verify that we want to build if [ "$DO_GPG" == "auto" ]; then DO_GPG="True" fi +DO_OTS=${DO_OTS:="auto"} +if [ "$DO_OTS" == "auto" ]; then + # Do opentimestamp if it is available + # python -m pip install opentimestamps-client + if type ots ; then + DO_OTS="True" + else + DO_OTS="False" + fi +fi + DO_BUILD=${DO_BUILD:="auto"} # Verify that we want to build if [ "$DO_BUILD" == "auto" ]; then @@ -150,6 +164,7 @@ if [ "$DO_BUILD" == "auto" ]; then fi DO_GPG=$(normalize_boolean "$DO_GPG") +DO_OTS=$(normalize_boolean "$DO_OTS") DO_BUILD=$(normalize_boolean "$DO_BUILD") DO_UPLOAD=$(normalize_boolean "$DO_UPLOAD") DO_TAG=$(normalize_boolean "$DO_TAG") @@ -237,6 +252,7 @@ GPG_KEYID = '$GPG_KEYID' DO_UPLOAD=${DO_UPLOAD} DO_TAG=${DO_TAG} DO_GPG=${DO_GPG} +DO_OTS=${DO_OTS} DO_BUILD=${DO_BUILD} MODE_LIST_STR=${MODE_LIST_STR} " @@ -375,7 +391,7 @@ ls_array(){ } -WHEEL_PATHS=() +WHEEL_FPATHS=() for _MODE in "${MODE_LIST[@]}" do if [[ "$_MODE" == "sdist" ]]; then @@ -393,32 +409,32 @@ do for new_item in "${_NEW_WHEEL_PATHS[@]}" do if [[ "$new_item" != "" ]]; then - WHEEL_PATHS+=("$new_item") + WHEEL_FPATHS+=("$new_item") fi done done # Dedup the paths -readarray -t WHEEL_PATHS < <(printf '%s\n' "${WHEEL_PATHS[@]}" | sort -u) +readarray -t WHEEL_FPATHS < <(printf '%s\n' "${WHEEL_FPATHS[@]}" | sort -u) -WHEEL_PATHS_STR=$(printf '"%s" ' "${WHEEL_PATHS[@]}") +WHEEL_PATHS_STR=$(printf '"%s" ' "${WHEEL_FPATHS[@]}") echo "WHEEL_PATHS_STR = $WHEEL_PATHS_STR" echo " MODE=$MODE VERSION='$VERSION' -WHEEL_PATHS='$WHEEL_PATHS_STR' +WHEEL_FPATHS='$WHEEL_PATHS_STR' " - +WHEEL_SIGNATURE_FPATHS=() if [ "$DO_GPG" == "True" ]; then echo " === === " - for WHEEL_FPATH in "${WHEEL_PATHS[@]}" + for WHEEL_FPATH in "${WHEEL_FPATHS[@]}" do echo "WHEEL_FPATH = $WHEEL_FPATH" check_variable WHEEL_FPATH @@ -439,6 +455,8 @@ if [ "$DO_GPG" == "True" ]; then echo "Verifying wheels" $GPG_EXECUTABLE --verify "$WHEEL_FPATH".asc "$WHEEL_FPATH" || { echo 'could not verify wheels' ; exit 1; } + + WHEEL_SIGNATURE_FPATHS+=("$WHEEL_FPATH".asc) done echo " === === @@ -448,6 +466,27 @@ else fi + +if [ "$DO_OTS" == "True" ]; then + + echo " + === === + " + if [ "$DO_GPG" == "True" ]; then + # Stamp the wheels and the signatures + ots stamp "${WHEEL_FPATHS[@]}" "${WHEEL_SIGNATURE_FPATHS[@]}" + else + # Stamp only the wheels + ots stamp "${WHEEL_FPATHS[@]}" + fi + echo " + === === + " +else + echo "DO_OTS=False, Skipping OTS sign" +fi + + if [[ "$DO_TAG" == "True" ]]; then TAG_NAME="v${VERSION}" # if we messed up we can delete the tag @@ -467,7 +506,7 @@ if [[ "$DO_UPLOAD" == "True" ]]; then check_variable TWINE_USERNAME check_variable TWINE_PASSWORD "hide" - for WHEEL_FPATH in "${WHEEL_PATHS[@]}" + for WHEEL_FPATH in "${WHEEL_FPATHS[@]}" do twine upload --username "$TWINE_USERNAME" "--password=$TWINE_PASSWORD" \ --repository-url "$TWINE_REPOSITORY_URL" \ @@ -496,3 +535,39 @@ else !!! FINISH: DRY RUN !!! """ fi + +__devel__=' +# Checking to see how easy it is to upload packages to gitlab. +# This logic should go in the CI script, not sure if it belongs here. + + +export HOST=https://gitlab.kitware.com +export GROUP_NAME=computer-vision +export PROJECT_NAME=geowatch +PROJECT_VERSION=$(geowatch --version) +echo "$PROJECT_VERSION" + +load_secrets +export PRIVATE_GITLAB_TOKEN=$(git_token_for "$HOST") +TMP_DIR=$(mktemp -d -t ci-XXXXXXXXXX) + +curl --header "PRIVATE-TOKEN: $PRIVATE_GITLAB_TOKEN" "$HOST/api/v4/groups" > "$TMP_DIR/all_group_info" +GROUP_ID=$(cat "$TMP_DIR/all_group_info" | jq ". | map(select(.name==\"$GROUP_NAME\")) | .[0].id") +echo "GROUP_ID = $GROUP_ID" + +curl --header "PRIVATE-TOKEN: $PRIVATE_GITLAB_TOKEN" "$HOST/api/v4/groups/$GROUP_ID" > "$TMP_DIR/group_info" +PROJ_ID=$(cat "$TMP_DIR/group_info" | jq ".projects | map(select(.name==\"$PROJECT_NAME\")) | .[0].id") +echo "PROJ_ID = $PROJ_ID" + +ls_array DIST_FPATHS "dist/*" + +for FPATH in "${DIST_FPATHS[@]}" +do + FNAME=$(basename $FPATH) + echo $FNAME + curl --header "PRIVATE-TOKEN: $PRIVATE_GITLAB_TOKEN" \ + --upload-file $FPATH \ + "https://gitlab.kitware.com/api/v4/projects/$PROJ_ID/packages/generic/$PROJECT_NAME/$PROJECT_VERSION/$FNAME" +done + +' diff --git a/requirements/optional.txt b/requirements/optional.txt index 8fa8f33eb..80d85cbc7 100644 --- a/requirements/optional.txt +++ b/requirements/optional.txt @@ -24,3 +24,5 @@ Pygments>=2.2.0 colorama>=0.4.3;platform_system=="Windows" python_dateutil>=2.8.1 + +packaging>=21.0 diff --git a/run_tests.py b/run_tests.py index 069e6f11b..5cfaa9933 100755 --- a/run_tests.py +++ b/run_tests.py @@ -9,6 +9,7 @@ '--cov-config', 'pyproject.toml', '--cov-report', 'html', '--cov-report', 'term', + '--durations', '100', '--xdoctest', '--cov=' + package_name, mod_dpath, test_dpath diff --git a/tests/test_cmd.py b/tests/test_cmd.py index a40951393..b37b4bb73 100644 --- a/tests/test_cmd.py +++ b/tests/test_cmd.py @@ -55,7 +55,10 @@ def test_cmd_stderr(): assert result['err'].strip() == 'hello stderr' -def test_cmd_with_pathlib(): +def test_cmd_with_list_of_pathlib(): + """ + ub.cmd can accept a pathlib.Path in a list of its arguments. + """ if not ub.POSIX: pytest.skip('posix only') fpath = ub.Path(ub.__file__) @@ -63,6 +66,17 @@ def test_cmd_with_pathlib(): assert str(fpath) in result['out'] +def test_cmd_with_single_pathlib(): + """ + ub.cmd can accept a pathlib.Path as its single argument + """ + if not ub.POSIX: + pytest.skip('posix only') + ls_exe = ub.Path(ub.find_exe('ls')) + result = ub.cmd(ls_exe) + result.check_returncode() + + def test_cmd_tee_auto(): """ pytest ubelt/tests/test_cmd.py -k tee_backend diff --git a/tests/test_download.py b/tests/test_download.py index 097ae813d..1159e4367 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -419,7 +419,7 @@ class SingletonTestServer(ub.NiceRepr): independent IPython session. CommandLine: - xdoctest -m /home/joncrall/code/ubelt/tests/test_download.py SingletonTestServer + xdoctest -m tests/test_download.py SingletonTestServer Note: We rely on python process close mechanisms to clean this server up. @@ -463,7 +463,7 @@ def __init__(self): def find_free_port(): """ References: - https://stackoverflow.com/questions/1365265/on-localhost-how-do-i-pick-a-free-port-number + .. [SO1365265] https://stackoverflow.com/questions/1365265/on-localhost-how-do-i-pick-a-free-port-number """ with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: s.bind(('', 0)) diff --git a/tests/test_editable_modules.py b/tests/test_editable_modules.py index cc193c161..8bf9fe30c 100644 --- a/tests/test_editable_modules.py +++ b/tests/test_editable_modules.py @@ -400,7 +400,7 @@ def serialize_install(self): site_dpath = ub.Path(distutils.sysconfig.get_python_lib()) egg_link_fpaths = list(site_dpath.glob(self.mod_name.replace('_', '*') + '*.egg-link')) editable_fpaths = list(site_dpath.glob('__editable__*' + self.mod_name.replace('_', '*') + '*')) - easy_install_fpath = site_dpath / 'easy-install.pth' + easy_install_fpath = site_dpath / 'easy-install.pth' # NOQA print(f'egg_link_fpaths={egg_link_fpaths}') print(f'editable_fpaths={editable_fpaths}') diff --git a/tests/test_hash.py b/tests/test_hash.py index 4490d7092..62ef7fee5 100644 --- a/tests/test_hash.py +++ b/tests/test_hash.py @@ -18,7 +18,7 @@ def _benchmark(): On 64-bit processors sha512 may be faster than sha256 References: - https://crypto.stackexchange.com/questions/26336/sha512-faster-than-sha256 + .. [SE26336] https://crypto.stackexchange.com/questions/26336/sha512-faster-than-sha256 """ result = ub.AutoOrderedDict() algos = ['sha1', 'sha256', 'sha512'] @@ -463,10 +463,10 @@ def test_compatible_hash_bases(): WITH PADDING AND VIA BYTE FORM, NOT INTEGER FORM. References: - https://stackoverflow.com/questions/43920799/convert-byte-to-base64-and-ascii-in-python - https://github.com/multiformats/multibase - https://stackoverflow.com/questions/6916805/why-does-a-base64-encoded-string-have-an-sign-at-the-end - https://github.com/semente/python-baseconv + .. [SO43920799] https://stackoverflow.com/questions/43920799/convert-byte-to-base64-and-ascii-in-python + .. [MultiBase] https://github.com/multiformats/multibase + .. [SO6916805] https://stackoverflow.com/questions/6916805/why-does-a-base64-encoded-string-have-an-sign-at-the-end + .. [SementeBaseConv] https://github.com/semente/python-baseconv """ import pytest pytest.skip('FIXME THIS ISSUE IS NOT RESOLVE YET.') diff --git a/tests/test_progiter.py b/tests/test_progiter.py index 601d4c476..cbf3d600d 100644 --- a/tests/test_progiter.py +++ b/tests/test_progiter.py @@ -369,6 +369,11 @@ def test_tqdm_compatibility(): prog.set_postfix_str('bar baz', refresh=False) assert 'bar baz' not in cap.text.strip() + with CaptureStdout() as cap: + prog = ProgIter(show_times=False) + prog.set_postfix('bar baz', refresh=False) + assert 'bar baz' not in cap.text.strip() + class IntObject: def __init__(self): diff --git a/tests/test_repr.py b/tests/test_repr.py index db59dd0ea..26f106736 100644 --- a/tests/test_repr.py +++ b/tests/test_repr.py @@ -379,7 +379,8 @@ def test_autosort(): } if sys.version_info[0:2] >= (3, 7): import pytest - with pytest.warns(DeprecationWarning): + # with pytest.warns(DeprecationWarning): + with pytest.deprecated_call(): assert ub.repr2(dict_, sort='auto', nl=1) == ub.codeblock( ''' { diff --git a/ubelt/__init__.py b/ubelt/__init__.py index 53b21f33d..895820804 100644 --- a/ubelt/__init__.py +++ b/ubelt/__init__.py @@ -24,7 +24,7 @@ xdoctest ubelt """ -__version__ = '1.3.4' +__version__ = '1.3.5' # Deprecated functions from ubelt.util_platform import ( @@ -127,6 +127,30 @@ from ubelt import util_time from ubelt import util_zip + +# Deprecated parts of the top-level API +# These functions are mostly moved into internal classes +__deprecated__ = [ + 'AutoOrderedDict', + 'dict_diff', + 'dict_isect', + 'dict_subset', + 'invert_dict', + 'map_keys', + 'map_vals', + 'map_values', + 'sorted_keys', + 'sorted_vals', + 'delete', + 'touch', + 'augpath', + 'ensuredir', + 'expandpath', + 'shrinkuser', + 'userhome', +] + + from ubelt.util_arg import (argflag, argval,) from ubelt.util_cache import (CacheStamp, Cacher,) from ubelt.util_colors import (NO_COLOR, color_text, highlight_code,) diff --git a/ubelt/_win32_links.py b/ubelt/_win32_links.py index 845199475..143108eba 100644 --- a/ubelt/_win32_links.py +++ b/ubelt/_win32_links.py @@ -2,13 +2,12 @@ For dealing with symlinks, junctions, and hard-links on windows. References: - https://stackoverflow.com/questions/18883892/batch-file-windows-cmd-exe-test-if-a-directory-is-a-link-symlink - - https://stackoverflow.com/questions/21561850/python-test-for-junction-point-target - http://timgolden.me.uk/python/win32_how_do_i/see_if_two_files_are_the_same_file.html - https://stackoverflow.com/questions/6260149/os-symlink-support-in-windows - https://msdn.microsoft.com/en-us/library/windows/desktop/aa365006(v=vs.85).aspx - https://superuser.com/a/902082/215232 + .. [SO18883892] https://stackoverflow.com/questions/18883892/batch-file-windows-cmd-exe-test-if-a-directory-is-a-link-symlink + .. [SO21561850] https://stackoverflow.com/questions/21561850/python-test-for-junction-point-target + .. [WinTwoFilesSame] http://timgolden.me.uk/python/win32_how_do_i/see_if_two_files_are_the_same_file.html + .. [SO6260149] https://stackoverflow.com/questions/6260149/os-symlink-support-in-windows + .. [WinDesktopAA365006] https://msdn.microsoft.com/en-us/library/windows/desktop/aa365006(v=vs.85).aspx + .. [SU902082] https://superuser.com/a/902082/215232 Weird Behavior: - [ ] In many cases using the win32 API seems to result in privilege errors @@ -26,7 +25,7 @@ import jaraco.windows.filesystem as jwfs -__win32_can_symlink__ = None +__win32_can_symlink__ = None # type: bool | None def _win32_can_symlink(verbose=0, force=0, testing=0): diff --git a/ubelt/_win32_links.pyi b/ubelt/_win32_links.pyi index 4dd5ba5e2..beda9680a 100644 --- a/ubelt/_win32_links.pyi +++ b/ubelt/_win32_links.pyi @@ -1,3 +1 @@ -from _typeshed import Incomplete - -__win32_can_symlink__: Incomplete +__win32_can_symlink__: bool | None diff --git a/ubelt/orderedset.py b/ubelt/orderedset.py index 0e38e9aa5..b701a5216 100644 --- a/ubelt/orderedset.py +++ b/ubelt/orderedset.py @@ -52,6 +52,9 @@ def is_iterable(obj): We don't need to check for the Python 2 `unicode` type, because it doesn't have an `__iter__` attribute anyway. + + Returns: + bool """ return ( hasattr(obj, "__iter__") @@ -93,6 +96,9 @@ def __len__(self): 0 >>> len(OrderedSet([1, 2])) 2 + + Returns: + int """ return len(self.items) @@ -100,15 +106,22 @@ def __getitem__(self, index): """ Get the item at a given index. - If `index` is a slice, you will get back that slice of items, as a + If ``index`` is a slice, you will get back that slice of items, as a new OrderedSet. - If `index` is a list or a similar iterable, you'll get a list of + If ``index`` is a list or a similar iterable, you'll get a list of items corresponding to those indices. This is similar to NumPy's "fancy indexing". The result is not an OrderedSet because you may ask for duplicate indices, and the number of elements returned should be the number of elements asked for. + Args: + index (int | slice | Any): + a simple or fancy index + + Returns: + List | OrderedSet | Any : item or items + Example: >>> oset = OrderedSet([1, 2, 3]) >>> oset[1] @@ -166,6 +179,9 @@ def __contains__(self, key): """ Test if the item is in this ordered set + Args: + key (Any): check if this item exists in the set + Returns: bool @@ -177,13 +193,20 @@ def __contains__(self, key): """ return key in self.map - def add(self, key): + def add(self, key): # type: ignore """ - Add `key` as an item to this OrderedSet, then return its index. + Add ``key`` as an item to this OrderedSet, then return its index. - If `key` is already in the OrderedSet, return the index it already + If ``key`` is already in the OrderedSet, return the index it already had. + Args: + key (Any): the item to add + + Returns: + int: the index of the items. Note, violates the Liskov Substitution + Principle and might be changed. + Example: >>> oset = OrderedSet() >>> oset.append(3) @@ -203,6 +226,9 @@ def update(self, sequence): Update the set with the given iterable sequence, then return the index of the last element inserted. + Args: + sequence (Iterable): items to add to this set + Example: >>> oset = OrderedSet([1, 2, 3]) >>> oset.update([3, 1, 5, 1, 4]) @@ -220,7 +246,7 @@ def update(self, sequence): ) return item_index - def index(self, key): # type: ignore + def index(self, key, start=0, stop=None): # type: ignore """ Get the index of a given entry, raising an IndexError if it's not present. @@ -228,6 +254,16 @@ def index(self, key): # type: ignore `key` can be a non-string iterable of entries, in which case this returns a list of indices. + Args: + key (Any): item to find the position of + + start (int): not supported yet + + stop (int | None): not supported yet + + Returns: + int + Example: >>> oset = OrderedSet([1, 2, 3]) >>> oset.index(2) @@ -275,6 +311,9 @@ def discard(self, key): The MutableSet mixin uses this to implement the .remove() method, which *does* raise an error when asked to remove a non-existent item. + Args: + key (Any): item to remove. + Example: >>> oset = OrderedSet([1, 2, 3]) >>> oset.discard(2) @@ -321,16 +360,26 @@ def __reversed__(self): """ return reversed(self.items) - def __repr__(self) -> str: + def __repr__(self): + """ + Returns: + str + """ if not self: return "%s()" % (self.__class__.__name__,) return "%s(%r)" % (self.__class__.__name__, list(self)) - def __eq__(self, other) -> bool: + def __eq__(self, other): """ Returns true if the containers have the same items. If `other` is a Sequence, then order is checked, otherwise it is ignored. + Args: + other (Any): item to compare against + + Returns: + bool + Example: >>> oset = OrderedSet([1, 3, 2]) >>> oset == [1, 3, 2] @@ -361,6 +410,9 @@ def union(self, *sets): Combines all unique items. Each items order is defined by its first appearance. + Args: + *sets : zero or more other iterables to operate on + Returns: OrderedSet @@ -387,6 +439,9 @@ def intersection(self, *sets): Returns elements in common between all sets. Order is defined only by the first set. + Args: + *sets : zero or more other iterables to operate on + Returns: OrderedSet @@ -412,6 +467,9 @@ def difference(self, *sets): """ Returns all elements that are in this set but not the others. + Args: + *sets : zero or more other iterables to operate on + Returns: OrderedSet @@ -433,10 +491,16 @@ def difference(self, *sets): items = self return cls(items) - def issubset(self, other) -> bool: + def issubset(self, other): """ Report whether another set contains this set. + Args: + other (Iterable): check if items in other are all contained in self. + + Returns: + bool + Example: >>> OrderedSet([1, 2, 3]).issubset({1, 2}) False @@ -451,10 +515,16 @@ def issubset(self, other) -> bool: # todo: contiguous subset / subsequence_index? - def issuperset(self, other) -> bool: + def issuperset(self, other): """ Report whether this set contains another set. + Args: + other (Iterable): check all items in self are contained in other. + + Returns: + bool + Example: >>> OrderedSet([1, 2]).issuperset([1, 2, 3]) False @@ -476,6 +546,9 @@ def symmetric_difference(self, other): Their order will be preserved, with elements from `self` preceding elements from `other`. + Args: + other (Iterable): items to operate on + Returns: OrderedSet @@ -523,6 +596,9 @@ def intersection_update(self, other): Update this OrderedSet to keep only items in another set, preserving their order in this set. + Args: + other (Iterable): items to operate on + Example: >>> this = OrderedSet([1, 4, 3, 5, 7]) >>> other = OrderedSet([9, 7, 1, 3, 2]) @@ -538,6 +614,9 @@ def symmetric_difference_update(self, other): Update this OrderedSet to remove items from another set, then add items from the other set that were not present in this set. + Args: + other (Iterable): items to operate on + Example: >>> this = OrderedSet([1, 4, 3, 5, 7]) >>> other = OrderedSet([9, 7, 1, 3, 2]) diff --git a/ubelt/orderedset.pyi b/ubelt/orderedset.pyi index 2b97673bc..df6db6097 100644 --- a/ubelt/orderedset.pyi +++ b/ubelt/orderedset.pyi @@ -8,7 +8,7 @@ from collections.abc import MutableSet, Sequence SLICE_ALL: slice -def is_iterable(obj): +def is_iterable(obj) -> bool: ... @@ -19,27 +19,27 @@ class OrderedSet(MutableSet, Sequence): def __init__(self, iterable: None | Iterable = None) -> None: ... - def __len__(self): + def __len__(self) -> int: ... - def __getitem__(self, index): + def __getitem__(self, index: int | slice | Any) -> List | OrderedSet | Any: ... def copy(self) -> OrderedSet: ... - def __contains__(self, key) -> bool: + def __contains__(self, key: Any) -> bool: ... - def add(self, key): + def add(self, key: Any): ... append = add - def update(self, sequence): + def update(self, sequence: Iterable): ... - def index(self, key): + def index(self, key: Any, start: int = 0, stop: int | None = None) -> int: ... get_loc = index @@ -48,7 +48,7 @@ class OrderedSet(MutableSet, Sequence): def pop(self) -> Any: ... - def discard(self, key) -> None: + def discard(self, key: Any) -> None: ... def clear(self) -> None: @@ -60,7 +60,7 @@ class OrderedSet(MutableSet, Sequence): def __reversed__(self) -> Iterator: ... - def __eq__(self, other) -> bool: + def __eq__(self, other: Any) -> bool: ... def union(self, *sets) -> OrderedSet: @@ -75,22 +75,22 @@ class OrderedSet(MutableSet, Sequence): def difference(self, *sets) -> OrderedSet: ... - def issubset(self, other) -> bool: + def issubset(self, other: Iterable) -> bool: ... - def issuperset(self, other) -> bool: + def issuperset(self, other: Iterable) -> bool: ... - def symmetric_difference(self, other) -> OrderedSet: + def symmetric_difference(self, other: Iterable) -> OrderedSet: ... def difference_update(self, *sets) -> None: ... - def intersection_update(self, other) -> None: + def intersection_update(self, other: Iterable) -> None: ... - def symmetric_difference_update(self, other) -> None: + def symmetric_difference_update(self, other: Iterable) -> None: ... diff --git a/ubelt/progiter.py b/ubelt/progiter.py index 87d274eaa..e739d1179 100644 --- a/ubelt/progiter.py +++ b/ubelt/progiter.py @@ -5,21 +5,6 @@ can be done either via an iterable interface or using the manual API. Using the iterable interface is most common. -ProgIter was originally developed independently of tqdm, but the newer versions -of this library have been designed to be compatible with tqdm-API. -:class:`ProgIter` is now a (mostly) drop-in alternative to :func:`tqdm.tqdm`. The -:mod:`tqdm` library may be more appropriate in some cases. *The main advantage of -:class:`ProgIter` is that it does not use any python threading*, and therefore can -be safer with code that makes heavy use of multiprocessing. -`The reason `_ -for this is that threading before forking may cause locks to be duplicated -across processes, which may lead to deadlocks. - -ProgIter is simpler than tqdm, which may be desirable for some applications. -However, this also means ProgIter is not as extensible as tqdm. -If you want a pretty bar or need something fancy, use tqdm; -if you want useful information about your iteration by default, use progiter. - The basic usage of ProgIter is simple and intuitive. Just wrap a python iterable. The following example wraps a ``range`` iterable and prints reported progress to stdout as the iterable is consumed. @@ -98,7 +83,7 @@ 'ProgIter', ] -default_timer = time.perf_counter +default_timer = time.perf_counter # type: Callable # A measurment takes place at a given iteration and posixtime. Measurement = collections.namedtuple('Measurement', ['idx', 'time']) @@ -113,6 +98,12 @@ def _infer_length(iterable): Try and infer the length using the PEP 424 length hint if available. adapted from click implementation + + Args: + iterable (Iterable): + + Returns: + int | None """ try: return len(iterable) @@ -139,19 +130,37 @@ class _TQDMCompat(object): @classmethod def write(cls, s, file=None, end='\n', nolock=False): - """ simply writes to stdout """ + """ + simply writes to stdout + + Args: + s (str): string + file (None | SupportsWrite): + end (str): end of line + nolock (bool): + """ fp = file if file is not None else sys.stdout fp.write(s) fp.write(end) def set_description(self, desc=None, refresh=True): - """ tqdm api compatibility. Changes the description of progress """ + """ + tqdm api compatibility. Changes the description of progress + + Args: + desc (str | None): description + """ self.desc = desc if refresh: self.refresh() def set_description_str(self, desc=None, refresh=True): - """ tqdm api compatibility. Changes the description of progress """ + """ + tqdm api compatibility. Changes the description of progress + + Args: + desc (str | None): description string + """ self.set_description(desc, refresh) def update(self, n=1): @@ -185,6 +194,10 @@ def refresh(self, nolock=False): @property def pos(self): + """ + Returns: + int + """ return 0 @classmethod @@ -197,8 +210,15 @@ def get_lock(cls): """ tqdm api compatibility. does nothing """ pass - def set_postfix(self, ordered_dict=None, refresh=True, **kwargs): - """ tqdm api compatibility. calls set_extra """ + def set_postfix_dict(self, ordered_dict=None, refresh=True, **kwargs): + """ + tqdm api compatibility. calls set_extra + + Args: + ordered_dict (None | dict): + refresh (bool): + **kwargs: + """ # Sort in alphabetical order to be more deterministic postfix = collections.OrderedDict( [] if ordered_dict is None else ordered_dict) @@ -219,6 +239,12 @@ def set_postfix(self, ordered_dict=None, refresh=True, **kwargs): for key in postfix.keys()) self.set_postfix_str(postfix, refresh=refresh) + def set_postfix(self, postfix, **kwargs): + if isinstance(postfix, str): + self.set_postfix_str(postfix, **kwargs) + else: + self.set_postfix_dict(ordered_dict=postfix, **kwargs) + def set_postfix_str(self, s='', refresh=True): """ tqdm api compatibility. calls set_extra """ self.set_extra(str(s)) @@ -265,87 +291,6 @@ class ProgIter(_TQDMCompat, _BackwardsCompat): ProgIter does not use threading whereas `tqdm` does. Attributes: - iterable (List | Iterable): - A list or iterable to loop over - - desc (str): - description label to show with progress - - total (int): - Maximum length of the process. If not specified, we estimate it - from the iterable, if possible. - - freq (int): - How many iterations to wait between messages. - Defaults to 1. - - eta_window (int): - number of previous measurements to use in eta calculation, default=64 - - clearline (bool): - if True messages are printed on the same line otherwise each new - progress message is printed on new line. - default=True - - adjust (bool): - if True `freq` is adjusted based on time_thresh. This may be - overwritten depending on the setting of verbose. - default=True - - time_thresh (float): - desired amount of time to wait between messages if adjust is True - otherwise does nothing, default=2.0 - - show_percent (bool): - if True show percent progress. Default=True - - show_times (bool): - if False do not show rate, eta, or wall time. default=True - Deprecated. Use show_rate / show_eta / show_wall instead. - - show_rate (bool): - show / hide rate, default=True - - show_eta (bool): - show / hide estimated time of arival (i.e. time to completion), - default=True - - show_wall (bool): - show / hide wall time, default=False - - initial (int): - starting index offset, default=0 - - stream (typing.IO): - stream where progress information is written to, default=sys.stdout - - timer (callable): - the timer object to use. Defaults to :func:`time.perf_counter`. - - enabled (bool): if False nothing happens. default=True - - chunksize (int | None): - indicates that each iteration processes a batch of this size. - Iteration rate is displayed in terms of single-items. - - rel_adjust_limit (float): - Maximum factor update frequency can be adjusted by in a single - step. default=4.0 - - verbose (int): - verbosity mode, which controls clearline, adjust, and enabled. The - following maps the value of `verbose` to its effect. - 0: enabled=False, - 1: enabled=True with clearline=True and adjust=True, - 2: enabled=True with clearline=False and adjust=True, - 3: enabled=True with clearline=False and adjust=False - - homogeneous (bool | str): - Indicate if the iterable is likely to take a uniform or homogeneous - amount of time per iteration. When True we can enable a speed - optimization. When False, the time estimates are more accurate. - Default to "auto", which attempts to determine if it is safe to use - True. Has no effect if ``adjust`` is False. Note: Either use ProgIter in a with statement or call prog.end() at the end @@ -362,7 +307,7 @@ class ProgIter(_TQDMCompat, _BackwardsCompat): tqdm - https://pypi.python.org/pypi/tqdm References: - http://datagenetics.com/blog/february12017/index.html + .. [DatagenProgBars] http://datagenetics.com/blog/february12017/index.html Example: >>> # doctest: +SKIP @@ -381,9 +326,95 @@ def __init__(self, iterable=None, desc=None, total=None, freq=1, chunksize=None, rel_adjust_limit=4.0, homogeneous='auto', timer=None, **kwargs): """ - Note: - See attributes for arg information - **kwargs accepts most of the tqdm api + See attributes more arg information + + Args: + iterable (List | Iterable): + A list or iterable to loop over + + desc (str | None): + description label to show with progress + + total (int | None): + Maximum length of the process. If not specified, we estimate it + from the iterable, if possible. + + freq (int): + How many iterations to wait between messages. + Defaults to 1. + + initial (int): + starting index offset, default=0 + + eta_window (int): + number of previous measurements to use in eta calculation, default=64 + + clearline (bool): + if True messages are printed on the same line otherwise each new + progress message is printed on new line. + default=True + + adjust (bool): + if True `freq` is adjusted based on time_thresh. This may be + overwritten depending on the setting of verbose. + default=True + + time_thresh (float): + desired amount of time to wait between messages if adjust is True + otherwise does nothing, default=2.0 + + show_percent (bool): + if True show percent progress. Default=True + + show_times (bool): + if False do not show rate, eta, or wall time. default=True + Deprecated. Use show_rate / show_eta / show_wall instead. + + show_rate (bool): + show / hide rate, default=True + + show_eta (bool): + show / hide estimated time of arival (i.e. time to completion), + default=True + + show_wall (bool): + show / hide wall time, default=False + + stream (typing.IO): + stream where progress information is written to, default=sys.stdout + + timer (callable): + the timer object to use. Defaults to :func:`time.perf_counter`. + + enabled (bool): if False nothing happens. default=True + + chunksize (int | None): + indicates that each iteration processes a batch of this size. + Iteration rate is displayed in terms of single-items. + + rel_adjust_limit (float): + Maximum factor update frequency can be adjusted by in a single + step. default=4.0 + + verbose (int): + verbosity mode, which controls clearline, adjust, and enabled. The + following maps the value of `verbose` to its effect. + 0: enabled=False, + 1: enabled=True with clearline=True and adjust=True, + 2: enabled=True with clearline=False and adjust=True, + 3: enabled=True with clearline=False and adjust=False + + homogeneous (bool | str): + Indicate if the iterable is likely to take a uniform or homogeneous + amount of time per iteration. When True we can enable a speed + optimization. When False, the time estimates are more accurate. + Default to "auto", which attempts to determine if it is safe to use + True. Has no effect if ``adjust`` is False. + + show_total (bool): + if True show total time. + + **kwargs: accepts most of the tqdm api """ if desc is None: desc = '' @@ -465,11 +496,27 @@ def __init__(self, iterable=None, desc=None, total=None, freq=1, self._reset_internals() def __call__(self, iterable): + """ + Overwrites the current iterator with iterable and starts iterating on + it. + + Warning: + Using this function is not recommended. + + Args: + iterable (Iterable): + + Returns: + Iterable + """ self.iterable = iterable return iter(self) def __enter__(self): """ + Returns: + ProgIter + Example: >>> # can be used as a context manager in iter mode >>> n = 3 @@ -479,13 +526,26 @@ def __enter__(self): self.begin() return self - def __exit__(self, type_, value, trace): - if trace is not None: + def __exit__(self, ex_type, ex_value, ex_traceback): + """ + Args: + ex_type (Type[BaseException] | None): + ex_value (BaseException | None): + ex_traceback (TracebackType | None): + + Returns: + bool | None + """ + if ex_traceback is not None: # nocover return False else: self.end() def __iter__(self): + """ + Returns: + Iterable + """ if not self.enabled: return iter(self.iterable) else: @@ -692,8 +752,8 @@ def step(self, inc=1, force=False): Manually step progress update, either directly or by an increment. Args: - inc (int, default=1): number of steps to increment - force (bool, default=False): if True forces progress display + inc (int): number of steps to increment. Defaults to 1. + force (bool): if True forces progress display. Defaults to False. Example: >>> n = 3 @@ -787,6 +847,9 @@ def _build_message_template(self): """ Defines the template for the progress line + Returns: + Tuple[str, str, str] + Example: >>> self = ProgIter() >>> print(self._build_message_template()[1].strip()) @@ -859,14 +922,20 @@ def format_message(self): Exists only for backwards compatibility. See `format_message_parts` for more recent API. + + Returns: + str """ return ''.join(self.format_message_parts()) def format_message_parts(self): r""" - builds a formatted progres message with the current values. + builds a formatted progress message with the current values. This contains the special characters needed to clear lines. + Returns: + Tuple[str, str, str] + Example: >>> self = ProgIter(clearline=False, show_times=False) >>> print(repr(self.format_message_parts()[1])) @@ -1003,5 +1072,10 @@ def _tryflush(self): pass def _write(self, msg): - """ write to the internal stream """ + """ + write to the internal stream + + Args: + msg (str): message to write + """ self.stream.write(msg) diff --git a/ubelt/progiter.pyi b/ubelt/progiter.pyi index f6e2d9f6e..2fe783e32 100644 --- a/ubelt/progiter.pyi +++ b/ubelt/progiter.pyi @@ -1,11 +1,15 @@ from typing import Iterable +from _typeshed import SupportsWrite from typing import List import typing from typing import Callable +from typing import Type +from types import TracebackType +from typing import Tuple from _typeshed import Incomplete from typing import NamedTuple -default_timer: Incomplete +default_timer: Callable class Measurement(NamedTuple): @@ -21,21 +25,21 @@ class _TQDMCompat: @classmethod def write(cls, - s, - file: Incomplete | None = ..., - end: str = ..., - nolock: bool = ...) -> None: + s: str, + file: None | SupportsWrite = None, + end: str = '\n', + nolock: bool = False) -> None: ... - desc: Incomplete + desc: str | None def set_description(self, - desc: Incomplete | None = ..., + desc: str | None = None, refresh: bool = ...) -> None: ... def set_description_str(self, - desc: Incomplete | None = ..., + desc: str | None = None, refresh: bool = ...) -> None: ... @@ -58,7 +62,7 @@ class _TQDMCompat: ... @property - def pos(self): + def pos(self) -> int: ... @classmethod @@ -69,10 +73,13 @@ class _TQDMCompat: def get_lock(cls) -> None: ... - def set_postfix(self, - ordered_dict: Incomplete | None = ..., - refresh: bool = ..., - **kwargs) -> None: + def set_postfix_dict(self, + ordered_dict: None | dict = None, + refresh: bool = True, + **kwargs) -> None: + ... + + def set_postfix(self, postfix, **kwargs) -> None: ... def set_postfix_str(self, s: str = ..., refresh: bool = ...) -> None: @@ -97,68 +104,68 @@ class _BackwardsCompat: class ProgIter(_TQDMCompat, _BackwardsCompat): + stream: typing.IO iterable: List | Iterable - desc: str - total: int + desc: str | None + total: int | None freq: int - eta_window: int - clearline: bool + initial: int + enabled: bool adjust: bool - time_thresh: float show_percent: bool show_times: bool show_rate: bool show_eta: bool + show_total: bool show_wall: bool - initial: int - stream: typing.IO - timer: Callable - enabled: bool + eta_window: int + time_thresh: float + clearline: bool chunksize: int | None rel_adjust_limit: float - verbose: int - homogeneous: bool | str - show_total: Incomplete extra: str started: bool finished: bool + homogeneous: bool | str def __init__(self, - iterable: Incomplete | None = ..., - desc: Incomplete | None = ..., - total: Incomplete | None = ..., - freq: int = ..., - initial: int = ..., - eta_window: int = ..., - clearline: bool = ..., - adjust: bool = ..., - time_thresh: float = ..., - show_percent: bool = ..., - show_times: bool = ..., - show_rate: bool = ..., - show_eta: bool = ..., - show_total: bool = ..., - show_wall: bool = ..., - enabled: bool = ..., - verbose: Incomplete | None = ..., - stream: Incomplete | None = ..., - chunksize: Incomplete | None = ..., - rel_adjust_limit: float = ..., - homogeneous: str = ..., - timer: Incomplete | None = ..., + iterable: List | Iterable | None = None, + desc: str | None = None, + total: int | None = None, + freq: int = 1, + initial: int = 0, + eta_window: int = 64, + clearline: bool = True, + adjust: bool = True, + time_thresh: float = 2.0, + show_percent: bool = True, + show_times: bool = True, + show_rate: bool = True, + show_eta: bool = True, + show_total: bool = True, + show_wall: bool = False, + enabled: bool = True, + verbose: int | None = None, + stream: typing.IO | None = None, + chunksize: int | None = None, + rel_adjust_limit: float = 4.0, + homogeneous: bool | str = 'auto', + timer: Callable | None = None, **kwargs) -> None: ... - def __call__(self, iterable): + def __call__(self, iterable: Iterable) -> Iterable: ... - def __enter__(self): + def __enter__(self) -> ProgIter: ... - def __exit__(self, type_, value, trace): + def __exit__(self, ex_type: Type[BaseException] | None, + ex_value: BaseException | None, + ex_traceback: TracebackType | None) -> bool | None: ... - def __iter__(self): + def __iter__(self) -> Iterable: ... def set_extra(self, extra: str | Callable) -> None: @@ -173,10 +180,10 @@ class ProgIter(_TQDMCompat, _BackwardsCompat): def step(self, inc: int = 1, force: bool = False) -> None: ... - def format_message(self): + def format_message(self) -> str: ... - def format_message_parts(self): + def format_message_parts(self) -> Tuple[str, str, str]: ... def ensure_newline(self) -> None: diff --git a/ubelt/util_cache.py b/ubelt/util_cache.py index b973d475a..b53b911bb 100644 --- a/ubelt/util_cache.py +++ b/ubelt/util_cache.py @@ -119,7 +119,7 @@ from os.path import join, normpath, basename, exists -class Cacher(object): +class Cacher: """ Saves data to disk and reloads it based on specified dependencies. @@ -241,8 +241,13 @@ def __init__(self, fname, depends=None, dpath=None, appname='ubelt', if verbose is None: verbose = self.VERBOSE if dpath is None: # pragma: no branch - from ubelt import util_path - dpath = os.fspath(util_path.Path.appdir(appname, type='cache')) + from ubelt.util_platform import platform_cache_dir + import pathlib + cache_dpath = pathlib.Path(platform_cache_dir()) + dpath = cache_dpath / (appname or 'ubelt') + dpath.mkdir(parents=True, exist_ok=True) + # from ubelt.util_path import Path + # dpath = os.fspath(Path.appdir(appname, type='cache')) if backend == 'auto': if ext == '.pkl': @@ -288,12 +293,12 @@ def _rectify_cfgstr(self, cfgstr=None): cfgstr = self.cfgstr if cfgstr is None else cfgstr if cfgstr is None and self.depends is not None: - from ubelt import util_hash # lazy hashing of depends data into cfgstr if isinstance(self.depends, str): self.cfgstr = self.depends else: - self.cfgstr = util_hash.hash_data(self.depends) + from ubelt.util_hash import hash_data + self.cfgstr = hash_data(self.depends) cfgstr = self.cfgstr if cfgstr is None and self.enabled: @@ -310,9 +315,8 @@ def _condense_cfgstr(self, cfgstr=None): # underscore, and a 40 char sha1 hash. max_len = 49 if len(cfgstr) > max_len: - from ubelt import util_hash - condensed = util_hash.hash_data(cfgstr, hasher=self.hasher, - base='hex') + from ubelt.util_hash import hash_data + condensed = hash_data(cfgstr, hasher=self.hasher, base='hex') condensed = condensed[0:max_len] else: condensed = cfgstr @@ -320,8 +324,8 @@ def _condense_cfgstr(self, cfgstr=None): @property def fpath(self) -> os.PathLike: - import ubelt as ub - return ub.Path(self.get_fpath()) + from ubelt.util_path import Path + return Path(self.get_fpath()) def get_fpath(self, cfgstr=None): """ @@ -565,8 +569,8 @@ def save(self, data, cfgstr=None): >>> cacher2.save('data') >>> assert not exists(cacher2.get_fpath()), 'should be disabled' """ - from ubelt import util_path - from ubelt import util_time + from ubelt.util_path import ensuredir + from ubelt.util_time import timestamp if not self.enabled: return if self.verbose > 0: @@ -576,7 +580,7 @@ def save(self, data, cfgstr=None): condensed = self._condense_cfgstr(cfgstr) # Make sure the cache directory exists - util_path.ensuredir(self.dpath) + ensuredir(self.dpath) data_fpath = self.get_fpath(cfgstr=cfgstr) meta_fpath = data_fpath + '.meta' @@ -585,7 +589,7 @@ def save(self, data, cfgstr=None): # This may be deprecated in the future. with open(meta_fpath, 'a') as file_: # TODO: maybe append this in json or YML format? - file_.write('\n\nsaving {}\n'.format(util_time.timestamp())) + file_.write('\n\nsaving {}\n'.format(timestamp())) file_.write(self.fname + '\n') file_.write(condensed + '\n') file_.write(cfgstr_ + '\n') @@ -733,6 +737,9 @@ class CacheStamp(object): The size, mtime, and hash mechanism is similar to how Makefile and redo caches work. + Attributes: + cacher (Cacher): underlying cacher object + Example: >>> import ubelt as ub >>> # Stamp the computation of expensive-to-compute.txt @@ -844,13 +851,13 @@ def _rectify_products(self, product=None): Returns: List[Path] """ - from ubelt import util_path + from ubelt.util_path import Path products = self.product if product is None else product if products is None: return None if not isinstance(products, (list, tuple)): products = [products] - products = list(map(util_path.Path, products)) + products = list(map(Path, products)) return products def _rectify_hash_prefixes(self): @@ -899,10 +906,10 @@ def _product_file_hash(self, product=None): if self.hasher is None: product_file_hash = None else: - from ubelt import util_hash + from ubelt.util_hash import hash_file products = self._rectify_products(product) product_file_hash = [ - util_hash.hash_file(p, hasher=self.hasher, base='hex') + hash_file(p, hasher=self.hasher, base='hex') for p in products ] return product_file_hash @@ -1015,10 +1022,10 @@ def expired(self, cfgstr=None, product=None): expires = certificate.get('expires', None) if expires is not None: - from ubelt import util_time + from ubelt.util_time import timeparse # Need to add in the local timezone to compare against the cert. now = _localnow() - expires_abs = util_time.timeparse(expires) + expires_abs = timeparse(expires) if now >= expires_abs: # We are expired err = 'expired_cert' @@ -1124,7 +1131,7 @@ def _expires(self, now=None): >>> assert self._expires(dt) == dt + self.expires """ # Rectify into a datetime - from ubelt import util_time + from ubelt.util_time import timeparse import datetime as datetime_mod import numbers if now is None: @@ -1137,7 +1144,7 @@ def _expires(self, now=None): elif isinstance(expires, datetime_mod.timedelta): expires_abs = now + expires elif isinstance(expires, str): - expires_abs = util_time.timeparse(expires) + expires_abs = timeparse(expires) elif isinstance(expires, datetime_mod.datetime): expires_abs = expires else: @@ -1165,13 +1172,13 @@ def _new_certificate(self, cfgstr=None, product=None): >>> cert = self._new_certificate() >>> assert cert['expires'] is not None """ - from ubelt import util_time + from ubelt.util_time import timestamp products = self._rectify_products(product) now = _localnow() expires = self._expires(now) certificate = { - 'timestamp': util_time.timestamp(now, precision=4), - 'expires': None if expires is None else util_time.timestamp(expires, precision=4), + 'timestamp': timestamp(now, precision=4), + 'expires': None if expires is None else timestamp(expires, precision=4), 'product': None if products is None else [os.fspath(p) for p in products], } if products is not None: diff --git a/ubelt/util_cache.pyi b/ubelt/util_cache.pyi index a3816c810..c224433a7 100644 --- a/ubelt/util_cache.pyi +++ b/ubelt/util_cache.pyi @@ -5,7 +5,6 @@ from typing import Any from typing import Sequence import datetime import os -from _typeshed import Incomplete from collections.abc import Generator @@ -76,7 +75,7 @@ class Cacher: class CacheStamp: - cacher: Incomplete + cacher: Cacher product: str | PathLike | Sequence[str | PathLike] | None hasher: str expires: str | int | datetime.datetime | datetime.timedelta | None diff --git a/ubelt/util_cmd.py b/ubelt/util_cmd.py index 0674feb10..d6fc93c20 100644 --- a/ubelt/util_cmd.py +++ b/ubelt/util_cmd.py @@ -120,14 +120,26 @@ class CmdOutput(dict): @property def stdout(self): + """ + Returns: + str | bytes + """ return self['out'] @property def stderr(self): + """ + Returns: + str | bytes + """ return self['err'] @property def returncode(self): + """ + Returns: + int + """ return self['ret'] def check_returncode(self): @@ -321,6 +333,9 @@ def cmd(command, shell=False, detach=False, verbose=0, tee=None, cwd=None, if isinstance(command, str): command_text = command command_tup = None + elif isinstance(command, os.PathLike): + command_text = os.fspath(command) + command_tup = None else: import shlex command_parts = [] diff --git a/ubelt/util_cmd.pyi b/ubelt/util_cmd.pyi index e4fc08df3..ab6eca37c 100644 --- a/ubelt/util_cmd.pyi +++ b/ubelt/util_cmd.pyi @@ -10,15 +10,15 @@ WIN32: bool class CmdOutput(dict): @property - def stdout(self): + def stdout(self) -> str | bytes: ... @property - def stderr(self): + def stderr(self) -> str | bytes: ... @property - def returncode(self): + def returncode(self) -> int: ... def check_returncode(self) -> None: diff --git a/ubelt/util_colors.py b/ubelt/util_colors.py index 6aa2b21c1..4742a662e 100644 --- a/ubelt/util_colors.py +++ b/ubelt/util_colors.py @@ -137,7 +137,7 @@ def _rich_highlight(text, lexer_name): # nocover Alternative rich-based highlighter References: - https://github.com/Textualize/rich/discussions/3076 + .. [RichDiscuss3076] https://github.com/Textualize/rich/discussions/3076 """ from rich.syntax import Syntax from rich.console import Console diff --git a/ubelt/util_const.py b/ubelt/util_const.py index 697a2bf95..8c14f28b6 100644 --- a/ubelt/util_const.py +++ b/ubelt/util_const.py @@ -12,7 +12,7 @@ [SO_41048643]_ for more details. References: - .. [SO_41048643]: http://stackoverflow.com/questions/41048643/a-second-none + .. [SO_41048643] http://stackoverflow.com/questions/41048643/a-second-none Example: >>> import ubelt as ub @@ -127,18 +127,46 @@ class sparingly. >>> assert all(not bool(v) for v in versions.values()) """ def __new__(cls): + """ + Returns: + NoParamType + """ return NoParam def __reduce__(self): + """ + Returns: + Tuple[type, Tuple] + """ return (NoParamType, ()) def __copy__(self): + """ + Returns: + NoParamType + """ return NoParam def __deepcopy__(self, memo): + """ + Returns: + NoParamType + """ return NoParam def __str__(cls): + """ + Returns: + str + """ return 'NoParam' def __repr__(cls): + """ + Returns: + str + """ return 'NoParam' def __bool__(self): + """ + Returns: + bool + """ # Ensure NoParam is Falsey return False diff --git a/ubelt/util_const.pyi b/ubelt/util_const.pyi index 4f72e5cc0..202fa3033 100644 --- a/ubelt/util_const.pyi +++ b/ubelt/util_const.pyi @@ -1,18 +1,21 @@ +from typing import Tuple + + class NoParamType: - def __new__(cls): + def __new__(cls) -> NoParamType: ... - def __reduce__(self): + def __reduce__(self) -> Tuple[type, Tuple]: ... - def __copy__(self): + def __copy__(self) -> NoParamType: ... - def __deepcopy__(self, memo): + def __deepcopy__(self, memo) -> NoParamType: ... - def __bool__(self): + def __bool__(self) -> bool: ... diff --git a/ubelt/util_deprecate.py b/ubelt/util_deprecate.py index 748df322c..4c74bc8f3 100644 --- a/ubelt/util_deprecate.py +++ b/ubelt/util_deprecate.py @@ -5,7 +5,7 @@ """ -def schedule_deprecation(modname, name='?', type='?', migration='', +def schedule_deprecation(modname=None, name='?', type='?', migration='', deprecate=None, error=None, remove=None, # TODO: let the user have more control over the # message. @@ -23,10 +23,11 @@ def schedule_deprecation(modname, name='?', type='?', migration='', users and developers. Args: - modname (str): + modname (str | None): The name of the underlying module associated with the feature to be deprecated. The module must already be imported and have a passable - ``__version__`` attribute. + ``__version__`` attribute. If unspecified, version info cannot be + used. name (str): The name of the feature to deprecate. This is usually a function or @@ -78,6 +79,7 @@ def schedule_deprecation(modname, name='?', type='?', migration='', https://docs.python.org/3/library/warnings.html Example: + >>> # xdoctest: +REQUIRES(module:packaging) >>> import ubelt as ub >>> import sys >>> import types @@ -101,6 +103,7 @@ def schedule_deprecation(modname, name='?', type='?', migration='', something else Example: + >>> # xdoctest: +REQUIRES(module:packaging) >>> # Demo the various cases >>> import ubelt as ub >>> import sys diff --git a/ubelt/util_dict.py b/ubelt/util_dict.py index 79ec326c5..2e4794972 100644 --- a/ubelt/util_dict.py +++ b/ubelt/util_dict.py @@ -394,7 +394,7 @@ def dict_union(*args): dictionary union (or any other dictionary set operator). References: - https://stackoverflow.com/questions/38987/merge-two-dict + .. [SO38987] https://stackoverflow.com/questions/38987/merge-two-dict SeeAlso: :func:`collections.ChainMap` - a standard python builtin data structure @@ -1255,25 +1255,63 @@ def copy(self): # We could just use the builtin variant for this specific operation def __or__(self, other): - """ The | union operator """ + """ + The ``|`` union operator + + Args: + other (SupportsKeysAndGetItem[Any, Any] | Iterable[Tuple[Any, Any]]): + + Returns: + SetDict + """ return self.union(other) def __and__(self, other): - """ The & intersection operator """ + """ + The ``&`` intersection operator + + Args: + other (Mapping | Iterable): + + Returns: + SetDict + """ return self.intersection(other) def __sub__(self, other): - """ The - difference operator """ + """ + The ``-`` difference operator + + Args: + other (Mapping | Iterable): + + Returns: + SetDict + """ return self.difference(other) def __xor__(self, other): - """ The ^ symmetric_difference operator """ + """ + The ``^`` symmetric_difference operator + + Args: + other (Mapping): + + Returns: + SetDict + """ return self.symmetric_difference(other) # - reverse versions def __ror__(self, other): """ + Args: + other (Mapping): + + Returns: + dict + Example: >>> import ubelt as ub >>> self = ub.sdict({1: 1, 2: 2, 3: 3}) @@ -1291,6 +1329,12 @@ def __ror__(self, other): def __rand__(self, other): """ + Args: + other (Mapping): + + Returns: + dict + Example: >>> import ubelt as ub >>> self = ub.sdict({1: 1, 2: 2, 3: 3}) @@ -1308,6 +1352,12 @@ def __rand__(self, other): def __rsub__(self, other): """ + Args: + other (Mapping): + + Returns: + dict + Example: >>> import ubelt as ub >>> self = ub.sdict({1: 1, 2: 2, 3: 3}) @@ -1325,6 +1375,12 @@ def __rsub__(self, other): def __rxor__(self, other): """ + Args: + other (Mapping): + + Returns: + dict + Example: >>> import ubelt as ub >>> self = ub.sdict({1: 1, 2: 2, 3: 3}) @@ -1346,6 +1402,12 @@ def __ior__(self, other): """ The inplace union operator ``|=``. + Args: + other (SupportsKeysAndGetItem[Any, Any] | Iterable[Tuple[Any, Any]]): + + Returns: + SetDict + Example: >>> import ubelt as ub >>> self = orig_ref = ub.sdict({1: 1, 2: 2, 3: 3}) @@ -1364,6 +1426,9 @@ def __iand__(self, other): """ The inplace intersection operator ``&=``. + Args: + other (Mapping | Iterable): + Example: >>> import ubelt as ub >>> self = orig_ref = ub.sdict({1: 1, 2: 2, 3: 3}) @@ -1384,6 +1449,9 @@ def __isub__(self, other): """ The inplace difference operator ``-=``. + Args: + other (Mapping | Iterable): + Example: >>> import ubelt as ub >>> self = orig_ref = ub.sdict({1: 1, 2: 2, 3: 3}) @@ -1423,6 +1491,9 @@ def __ixor__(self, other): """ The inplace symmetric difference operator ``^=``. + Args: + other (Mapping): + Example: >>> import ubelt as ub >>> self = orig_ref = ub.sdict({1: 1, 2: 2, 3: 3}) @@ -1859,8 +1930,7 @@ def invert(self, unique_vals=True): >>> inverted = ub.udict({'a': 1, 'b': 2}).invert() >>> assert inverted == {1: 'a', 2: 'b'} """ - import ubelt as ub - return ub.invert_dict(self, unique_vals=unique_vals, cls=self.__class__) + return invert_dict(self, unique_vals=unique_vals, cls=self.__class__) def map_keys(self, func): """ @@ -1880,8 +1950,7 @@ def map_keys(self, func): >>> new = ub.udict({'a': [1, 2, 3], 'b': []}).map_keys(ord) >>> assert new == {97: [1, 2, 3], 98: []} """ - import ubelt as ub - return ub.map_keys(func, self, cls=self.__class__) + return map_keys(func, self, cls=self.__class__) def map_values(self, func): """ @@ -1901,8 +1970,7 @@ def map_values(self, func): >>> newdict = ub.udict({'a': [1, 2, 3], 'b': []}).map_values(len) >>> assert newdict == {'a': 3, 'b': 0} """ - import ubelt as ub - return ub.map_values(func, self, cls=self.__class__) + return map_values(func, self, cls=self.__class__) def sorted_keys(self, key=None, reverse=False): """ @@ -1928,8 +1996,7 @@ def sorted_keys(self, key=None, reverse=False): >>> new = ub.udict({'spam': 2.62, 'eggs': 1.20, 'jam': 2.92}).sorted_keys() >>> assert new == ub.odict([('eggs', 1.2), ('jam', 2.92), ('spam', 2.62)]) """ - import ubelt as ub - return ub.sorted_keys(self, key=key, reverse=reverse, cls=self.__class__) + return sorted_keys(self, key=key, reverse=reverse, cls=self.__class__) def sorted_values(self, key=None, reverse=False): """ @@ -1955,8 +2022,7 @@ def sorted_values(self, key=None, reverse=False): >>> new = ub.udict({'spam': 2.62, 'eggs': 1.20, 'jam': 2.92}).sorted_values() >>> assert new == ub.odict([('eggs', 1.2), ('spam', 2.62), ('jam', 2.92)]) """ - import ubelt as ub - return ub.sorted_values(self, key=key, reverse=reverse, cls=self.__class__) + return sorted_values(self, key=key, reverse=reverse, cls=self.__class__) def peek_key(self, default=NoParam): """ @@ -1975,8 +2041,8 @@ def peek_key(self, default=NoParam): >>> import ubelt as ub >>> assert ub.udict({1: 2}).peek_key() == 1 """ - import ubelt as ub - return ub.peek(self.keys(), default=default) + from ubelt.util_list import peek + return peek(self.keys(), default=default) def peek_value(self, default=NoParam): """ @@ -1994,8 +2060,8 @@ def peek_value(self, default=NoParam): >>> import ubelt as ub >>> assert ub.udict({1: 2}).peek_value() == 2 """ - import ubelt as ub - return ub.peek(self.values(), default=default) + from ubelt.util_list import peek + return peek(self.values(), default=default) class AutoDict(UDict): @@ -2017,6 +2083,13 @@ class AutoDict(UDict): _base = UDict def __getitem__(self, key): + """ + Args: + key (KT): key to lookup + + Returns: + VT | AutoDict: an existing value or a new AutoDict + """ try: # value = super(AutoDict, self).__getitem__(key) value = self._base.__getitem__(self, key) diff --git a/ubelt/util_dict.pyi b/ubelt/util_dict.pyi index 906863b0b..31118dcf4 100644 --- a/ubelt/util_dict.pyi +++ b/ubelt/util_dict.pyi @@ -11,6 +11,8 @@ from ubelt.util_const import NoParamType from collections import OrderedDict from typing import Mapping from typing import Set +from typing import Tuple +from _typeshed import SupportsKeysAndGetItem from collections import OrderedDict, defaultdict from collections.abc import Generator from typing import Any, TypeVar @@ -126,40 +128,46 @@ class SetDict(dict): def copy(self): ... - def __or__(self, other): + def __or__( + self, + other: SupportsKeysAndGetItem[Any, Any] | Iterable[Tuple[Any, Any]] + ) -> SetDict: ... - def __and__(self, other): + def __and__(self, other: Mapping | Iterable) -> SetDict: ... - def __sub__(self, other): + def __sub__(self, other: Mapping | Iterable) -> SetDict: ... - def __xor__(self, other): + def __xor__(self, other: Mapping) -> SetDict: ... - def __ror__(self, other): + def __ror__(self, other: Mapping) -> dict: ... - def __rand__(self, other): + def __rand__(self, other: Mapping) -> dict: ... - def __rsub__(self, other): + def __rsub__(self, other: Mapping) -> dict: ... - def __rxor__(self, other): + def __rxor__(self, other: Mapping) -> dict: ... - def __ior__(self, other): + def __ior__( + self, + other: SupportsKeysAndGetItem[Any, Any] | Iterable[Tuple[Any, Any]] + ) -> SetDict: ... - def __iand__(self, other): + def __iand__(self, other: Mapping | Iterable): ... - def __isub__(self, other): + def __isub__(self, other: Mapping | Iterable): ... - def __ixor__(self, other): + def __ixor__(self, other: Mapping): ... def union(self, @@ -235,7 +243,7 @@ class UDict(SetDict): class AutoDict(UDict): - def __getitem__(self, key): + def __getitem__(self, key: KT) -> VT | AutoDict: ... def to_dict(self) -> dict: diff --git a/ubelt/util_download.py b/ubelt/util_download.py index f333a50d1..084261ce2 100644 --- a/ubelt/util_download.py +++ b/ubelt/util_download.py @@ -140,7 +140,8 @@ def download(url, fpath=None, dpath=None, fname=None, appname=None, >>> ub.download(url, hasher='sha512', hash_prefix='BAD_HASH') """ from ubelt import ProgIter as Progress - from ubelt.util_path import Path + from ubelt.util_platform import platform_cache_dir + import pathlib import shutil import tempfile import hashlib @@ -155,7 +156,9 @@ def download(url, fpath=None, dpath=None, fname=None, appname=None, raise ValueError('Cannot specify fpath with dpath or fname') if fpath is None: if dpath is None: - dpath = Path.appdir(appname or 'ubelt').ensuredir() + cache_dpath = pathlib.Path(platform_cache_dir()) + dpath = cache_dpath / (appname or 'ubelt') + dpath.mkdir(parents=True, exist_ok=True) if fname is None: fname = basename(url) fpath = join(dpath, fname) @@ -403,7 +406,8 @@ def grabdata(url, fpath=None, dpath=None, fname=None, redo=False, >>> fpath = ub.grabdata(url2, fname=fname, hash_prefix=prefix2) >>> assert json.loads(stamp_fpath.read_text())['hash'][0].startswith(prefix2) """ - from ubelt.util_path import Path + import pathlib + from ubelt.util_platform import platform_cache_dir from ubelt.util_cache import CacheStamp if appname and dpath: raise ValueError('Cannot specify appname with dpath') @@ -412,7 +416,9 @@ def grabdata(url, fpath=None, dpath=None, fname=None, redo=False, if fpath is None: if dpath is None: - dpath = Path.appdir(appname or 'ubelt').ensuredir() + cache_dpath = pathlib.Path(platform_cache_dir()) + dpath = cache_dpath / (appname or 'ubelt') + dpath.mkdir(parents=True, exist_ok=True) if fname is None: fname = basename(url) fpath = join(dpath, fname) diff --git a/ubelt/util_download_manager.py b/ubelt/util_download_manager.py index d6aec2315..8f9c12afe 100644 --- a/ubelt/util_download_manager.py +++ b/ubelt/util_download_manager.py @@ -111,6 +111,18 @@ def as_completed(self, prog=None, desc=None, verbose=1): """ Generate completed jobs as they become available + Args: + + prog (None | bool | type): + if True, uses a ub.ProgIter progress bar. Can also be a class + with a compatible progiter API. + + desc (str | None): + if specified, reports progress with a + :class:`ubelt.progiter.ProgIter` object. + + verbose (int): verbosity + Example: >>> import pytest >>> import ubelt as ub @@ -140,7 +152,15 @@ def shutdown(self): self._pool.executor.shutdown() def __iter__(self): + """ + Returns: + Iterable + """ return self.as_completed() def __len__(self): + """ + Returns: + int + """ return len(self._pool) diff --git a/ubelt/util_download_manager.pyi b/ubelt/util_download_manager.pyi index f57be2f85..8eb1cb438 100644 --- a/ubelt/util_download_manager.pyi +++ b/ubelt/util_download_manager.pyi @@ -1,7 +1,7 @@ from os import PathLike import concurrent import concurrent.futures -from _typeshed import Incomplete +from typing import Iterable class DownloadManager: @@ -23,16 +23,16 @@ class DownloadManager: ... def as_completed(self, - prog: Incomplete | None = ..., - desc: Incomplete | None = ..., - verbose: int = ...): + prog: None | bool | type = None, + desc: str | None = None, + verbose: int = 1): ... def shutdown(self) -> None: ... - def __iter__(self): + def __iter__(self) -> Iterable: ... - def __len__(self): + def __len__(self) -> int: ... diff --git a/ubelt/util_format.py b/ubelt/util_format.py index 3ce4aaa2b..dbbcd944a 100644 --- a/ubelt/util_format.py +++ b/ubelt/util_format.py @@ -1,12 +1,17 @@ """ -This module is deprecated. Use util_repr instead. +Warning: + + This module is deprecated. Use :mod:`ubelt.util_repr` instead. """ from .util_repr import urepr, ReprExtensions, _REPR_EXTENSIONS def repr2(data, **kwargs): """ - Deprecated for urepr + Alias of :func:`ubelt.util_repr.urepr`. + + Warning: + Deprecated for urepr Example: >>> # Test that repr2 remains backwards compatible @@ -39,8 +44,8 @@ def repr2(data, **kwargs): 'simple_list': [1, 2, 'red', 'blue'], } """ - import ubelt as ub - ub.schedule_deprecation( + from ubelt.util_deprecate import schedule_deprecation + schedule_deprecation( modname='ubelt', name='repr2', type='function', migration='use urepr instead', deprecate='1.2.5', error='2.0.0', remove='2.1.0', diff --git a/ubelt/util_futures.py b/ubelt/util_futures.py index 31afb566b..b43caa3a7 100644 --- a/ubelt/util_futures.py +++ b/ubelt/util_futures.py @@ -549,12 +549,12 @@ def as_completed(self, timeout=None, desc=None, progkw=None): ... pass >>> pool.shutdown() """ - import ubelt as ub + from ubelt.progiter import ProgIter job_iter = as_completed(self.jobs, timeout=timeout) if desc is not None: if progkw is None: progkw = {} - job_iter = ub.ProgIter( + job_iter = ProgIter( job_iter, desc=desc, total=len(self.jobs), **progkw) self._prog = job_iter for job in job_iter: diff --git a/ubelt/util_hash.py b/ubelt/util_hash.py index 03d4ccaba..2c99f5ea3 100644 --- a/ubelt/util_hash.py +++ b/ubelt/util_hash.py @@ -1,13 +1,13 @@ r""" Wrappers around hashlib functions to generate hash signatures for common data. -The hashes are deterministic across python versions and operating systems. -This is verified by CI testing on Windows, Linux, Python with 2.7, 3.4, and -greater, and on 32 and 64 bit versions. +The hashes are deterministic across Python versions and operating systems. +This is verified by CI testing on 32 and 64 bit versions of Windows, Linux, and +OSX with all supported Python. Use Case #1: You have data that you want to hash. If we assume the data is in -standard python scalars or ordered sequences: e.g. tuple, list, odict, oset, -int, str, etc..., then the solution is :func:`hash_data`. +standard python scalars or ordered sequences: e.g. tuple, list, OrderedDict, +OrderedSet, int, str, etc..., then the solution is :func:`hash_data`. Use Case #2: You have a file you want to hash, but your system doesn't have a sha1sum executable (or you dont want to use Popen). The solution is @@ -89,10 +89,17 @@ 'Y', 'Z', '2', '3', '4', '5', '6', '7'] -DEFAULT_ALPHABET = _ALPHABET_16 +DEFAULT_ALPHABET = _ALPHABET_16 # type: List[str] def b(s): + """ + Args: + s (str): + + Returns: + bytes + """ return s.encode("latin-1") # Sensible choices for default hashers are sha1, sha512, and xxh64. @@ -111,7 +118,9 @@ def b(s): # DEFAULT_HASHER = xxhash.xxh32 # DEFAULT_HASHER = xxhash.xxh64 # xxh64 is the fastest, but non-standard # DEFAULT_HASHER = hashlib.sha1 # fast algo, but has a known collision -DEFAULT_HASHER = hashlib.sha512 # most robust algo, but slower than others +# DEFAULT_HASHER = hashlib.sha512 # most robust algo, but slower than others + +DEFAULT_HASHER = hashlib.sha512 # type: Callable _COMPATIBLE_HASHABLE_SEQUENCE_TYPES_DEFAULT = True @@ -130,6 +139,12 @@ def _int_to_bytes(int_): Converts an integer into its byte representation assumes int32 by default, but dynamically handles larger ints + Args: + int_ (int): + + Returns: + bytes + Example: >>> from ubelt.util_hash import _int_to_bytes, _bytes_to_int >>> int_ = 1 @@ -150,6 +165,12 @@ def _bytes_to_int(bytes_): r""" Converts a string of bytes into its integer representation (big-endian) + Args: + bytes_ (bytes): + + Returns: + int + Example: >>> bytes_ = b'\x01' >>> assert _int_to_bytes((_bytes_to_int(bytes_))) == bytes_ @@ -163,10 +184,14 @@ class _Hashers(object): """ We offer hashers beyond what is available in hashlib. This class is used to lazy load them. + + Attributes: + algos (Dict[str, object]): + aliases (Dict[str, str]): """ def __init__(self): - self.algos = {} # type: Dict[str, object] # NOQA - self.aliases = {} # type: Dict[str, str] # NOQA + self.algos = {} # type: Dict[str, object] + self.aliases = {} # type: Dict[str, str] self._lazy_queue = [ self._register_xxhash, self._register_blake3, @@ -174,6 +199,12 @@ def __init__(self): ] def available(self): + """ + The names of available hash algorithms + + Returns: + List[str] + """ if self._lazy_queue: # nocover self._evaluate_registration_queue() return list(self.algos.keys()) @@ -187,6 +218,13 @@ def _evaluate_registration_queue(self): self._lazy_queue.clear() def __contains__(self, key): + """ + Args: + key (str): name of hash algo to check + + Returns: + bool: if the algo is available + """ if self._lazy_queue: # nocover self._evaluate_registration_queue() return key in self.algos or key in self.aliases @@ -219,6 +257,14 @@ def _register_hashlib(self): self.algos[key] = hashlib.new(key) def lookup(self, hasher): + """ + Args: + hasher (NoParamType | str | Any): + something coercable to a hasher + + Returns: + Callable: a function to construct the requested hahser + """ if hasher is NoParam or hasher == 'default': hasher = DEFAULT_HASHER elif hasattr(hasher, 'hexdigest'): @@ -325,9 +371,11 @@ class HashableExtensions(object): Note: We are introducing experimental functionality where custom instances of this class can be created and passed as arguments to hash_data. + + Attributes: + iterable_checks (List[Callable]): """ def __init__(self): - self.keyed_extensions = {} self.iterable_checks = [] self._lazy_queue = [] # type: List[Callable] # NOQA @@ -444,6 +492,12 @@ def lookup(self, data): Returns an appropriate function to hash ``data`` if one has been registered. + Args: + data (object): the object the user would like to hash + + Returns: + Callable: a function that can hash the object + Raises: TypeError : if data has no registered hash methods @@ -512,6 +566,12 @@ def lookup(self, data): def add_iterable_check(self, func): """ Registers a function that detects when a type is iterable + + Args: + func (Callable): + + Returns: + Callable """ self.iterable_checks.append(func) return func @@ -690,9 +750,9 @@ def _convert_set(data): # what raises a TypeError differs between Python 2 and 3 ordered_ = sorted(data) except TypeError: - import ubelt as ub + from ubelt.util_list import argsort data_ = list(data) - sortx = ub.argsort(data_, key=str) + sortx = argsort(data_, key=str) ordered_ = [data_[k] for k in sortx] # See: [util_hash.Note.1] hashable = b''.join(_hashable_sequence( @@ -707,8 +767,8 @@ def _convert_dict(data): ordered_ = sorted(data.items()) # what raises a TypeError differs between Python 2 and 3 except TypeError: - import ubelt as ub - sortx = ub.argsort(data, key=str) + from ubelt.util_list import argsort + sortx = argsort(data, key=str) ordered_ = [(k, data[k]) for k in sortx] # See: [util_hash.Note.1] hashable = b''.join(_hashable_sequence( @@ -786,15 +846,28 @@ def _lazy_init(): class _HashTracer(object): - """ helper class to extract hashed sequences """ + """ + Helper class to extract hashed sequences + + Attributes: + sequence (List[bytes]): + """ def __init__(self): - self.sequence = [] + self.sequence = [] # type: List[bytes] - def update(self, bytes): - self.sequence.append(bytes) + def update(self, item): + """ + Args: + item (bytes): + """ + self.sequence.append(item) def hexdigest(self): + """ + Returns: + bytes + """ return b''.join(self.sequence) @@ -1158,6 +1231,9 @@ def hash_file(fpath, blocksize=1048576, stride=1, maxbytes=None, Valid keys are 'abc', 'hex', and 'dec', 10, 16, 26, 32. Defaults to 'hex'. + Returns: + str: the hash text + References: .. [SO_3431825] http://stackoverflow.com/questions/3431825/md5-checksum-of-a-file .. [SO_5001893] http://stackoverflow.com/questions/5001893/when-to-use-sha-1-vs-sha-2 diff --git a/ubelt/util_hash.pyi b/ubelt/util_hash.pyi index 796b7dfd5..777422348 100644 --- a/ubelt/util_hash.pyi +++ b/ubelt/util_hash.pyi @@ -1,44 +1,44 @@ -from typing import Tuple -from typing import Callable +from typing import Dict +from typing import List from ubelt.util_const import NoParam from ubelt.util_const import NoParamType -from typing import List +from typing import Any +from typing import Callable +from typing import Tuple from os import PathLike -from _typeshed import Incomplete -from typing import TypeVar +from typing import Any, TypeVar Hasher = TypeVar("Hasher") HASH_VERSION: int -DEFAULT_ALPHABET: Incomplete +DEFAULT_ALPHABET: List[str] -def b(s): +def b(s: str) -> bytes: ... -DEFAULT_HASHER: Incomplete +DEFAULT_HASHER: Callable class _Hashers: - algos: Incomplete - aliases: Incomplete + algos: Dict[str, object] + aliases: Dict[str, str] def __init__(self) -> None: ... - def available(self): + def available(self) -> List[str]: ... - def __contains__(self, key): + def __contains__(self, key: str) -> bool: ... - def lookup(self, hasher): + def lookup(self, hasher: NoParamType | str | Any) -> Callable: ... class HashableExtensions: - keyed_extensions: Incomplete - iterable_checks: Incomplete + iterable_checks: List[Callable] def __init__(self) -> None: ... @@ -46,23 +46,23 @@ class HashableExtensions: def register(self, hash_types: type | Tuple[type]) -> Callable: ... - def lookup(self, data): + def lookup(self, data: object) -> Callable: ... - def add_iterable_check(self, func): + def add_iterable_check(self, func: Callable) -> Callable: ... class _HashTracer: - sequence: Incomplete + sequence: List[bytes] def __init__(self) -> None: ... - def update(self, bytes) -> None: + def update(self, item: bytes) -> None: ... - def hexdigest(self): + def hexdigest(self) -> bytes: ... @@ -80,5 +80,5 @@ def hash_file(fpath: PathLike, stride: int = 1, maxbytes: int | None = None, hasher: str | Hasher | NoParamType = NoParam, - base: List[str] | int | str | NoParamType = NoParam): + base: List[str] | int | str | NoParamType = NoParam) -> str: ... diff --git a/ubelt/util_import.py b/ubelt/util_import.py index ad0a8f95c..661470cfb 100644 --- a/ubelt/util_import.py +++ b/ubelt/util_import.py @@ -26,7 +26,7 @@ 'import_module_from_path', ] -IS_PY_GE_308 = sys.version_info[0] >= 3 and sys.version_info[1] >= 8 +IS_PY_GE_308 = sys.version_info[0] >= 3 and sys.version_info[1] >= 8 # type: bool class PythonPathContext(object): @@ -519,7 +519,13 @@ def check_dpath(dpath): # but it would be nice to ensure we are not matching suffixes. # however, we should probably match and handle different versions. _editable_fname_pth_pat = '__editable__.' + _pkg_name + '-*.pth' - _editable_fname_finder_py_pat = '__editable___' + _pkg_name + '_*finder.py' + + # NOTE: the __editable__ finders are named after the package, but the + # module could have a different name, so we cannot use the package name + # (which in this case is really the module name) in the pattern, and we + # have to check all of the finders. + # _editable_fname_finder_py_pat = '__editable___' + _pkg_name + '_*finder.py' + _editable_fname_finder_py_pat = '__editable___*_*finder.py' found_modpath = None for dpath in candidate_dpaths: @@ -788,13 +794,22 @@ def modpath_to_modname(modpath, hide_init=True, hide_main=False, check=True, encountered. Args: - modpath (str): module filepath - hide_init (bool, default=True): removes the __init__ suffix - hide_main (bool, default=False): removes the __main__ suffix - check (bool, default=True): if False, does not raise an error if - modpath is a dir and does not contain an __init__ file. - relativeto (str | None, default=None): if specified, all checks are ignored - and this is considered the path to the root module. + modpath (str): + Module filepath + + hide_init (bool): + Removes the __init__ suffix. Defaults to True. + + hide_main (bool): + Removes the __main__ suffix. Defaults to False. + + check (bool): + If False, does not raise an error if modpath is a dir and does not + contain an __init__ file. Defaults to True. + + relativeto (str | None): + If specified, all checks are ignored and this is considered the + path to the root module. Defaults to None. TODO: - [ ] Does this need modification to support PEP 420? @@ -989,9 +1004,9 @@ def _parse_static_node_value(node): """ import ast from collections import OrderedDict - # TODO: ast.Constant for 3.8 - if isinstance(node, ast.Num): - value = node.n + import numbers + if (isinstance(node, ast.Constant) and isinstance(node.value, numbers.Number) if IS_PY_GE_308 else isinstance(node, ast.Num)): + value = node.value if IS_PY_GE_308 else node.n elif (isinstance(node, ast.Constant) and isinstance(node.value, str) if IS_PY_GE_308 else isinstance(node, ast.Str)): value = node.value if IS_PY_GE_308 else node.s elif isinstance(node, ast.List): diff --git a/ubelt/util_import.pyi b/ubelt/util_import.pyi index 4e4980658..859bebd87 100644 --- a/ubelt/util_import.pyi +++ b/ubelt/util_import.pyi @@ -5,6 +5,8 @@ from types import ModuleType from typing import List from typing import Tuple +IS_PY_GE_308: bool + class PythonPathContext: dpath: str | PathLike diff --git a/ubelt/util_indexable.py b/ubelt/util_indexable.py index 9971ffde0..bee5453ce 100644 --- a/ubelt/util_indexable.py +++ b/ubelt/util_indexable.py @@ -42,6 +42,9 @@ class IndexableWalker(Generator): the types that should be considered list-like for the purposes of nested iteration. Defaults to ``(list, tuple)``. + indexable_cls (Tuple[type]): + combined dict_cls and list_cls + Example: >>> import ubelt as ub >>> # Given Nested Data @@ -193,10 +196,28 @@ def send(self, arg): # Note: this will error if called before __next__ self._walk_gen.send(arg) - def throw(self, type=None, value=None, traceback=None): + def throw(self, typ, val=None, tb=None): # type: ignore """ throw(typ[,val[,tb]]) -> raise exception in generator, return next yielded value or raise StopIteration. + + Args: + typ (Any): + Type of the exception. + Should be a ``type[BaseException]``, type checking is not working right here. + + val (Optional[object]): + + tb (Optional[TracebackType]): + + Returns: + Any + + Raises: + StopIteration + + References: + .. [GeneratorThrow] https://docs.python.org/3/reference/expressions.html#generator.throw """ raise StopIteration @@ -516,8 +537,8 @@ def indexable_allclose(items1, items2, rel_tol=1e-9, abs_tol=0.0, return_info=Fa >>> print('return_info = {}'.format(ub.repr2(return_info, nl=1))) >>> print('flag = {!r}'.format(flag)) """ - import ubelt as ub - ub.schedule_deprecation( + from ubelt.util_deprecate import schedule_deprecation + schedule_deprecation( 'ubelt', 'indexable_allclose', 'function', migration=( 'Use `ub.IndexableWalker(items1).allclose(items2)` instead' diff --git a/ubelt/util_indexable.pyi b/ubelt/util_indexable.pyi index acd666409..85cff3987 100644 --- a/ubelt/util_indexable.pyi +++ b/ubelt/util_indexable.pyi @@ -1,8 +1,9 @@ from typing import Tuple from typing import List from typing import Any +from typing import Optional +from types import TracebackType from typing import Dict -from _typeshed import Incomplete from collections.abc import Generator @@ -10,7 +11,7 @@ class IndexableWalker(Generator): data: dict | list | tuple dict_cls: Tuple[type] list_cls: Tuple[type] - indexable_cls: Incomplete + indexable_cls: Tuple[type] def __init__(self, data, dict_cls=..., list_cls=...) -> None: ... @@ -25,9 +26,9 @@ class IndexableWalker(Generator): ... def throw(self, - type: Incomplete | None = ..., - value: Incomplete | None = ..., - traceback: Incomplete | None = ...) -> None: + typ: Any, + val: Optional[object] = None, + tb: Optional[TracebackType] = None) -> Any: ... def __setitem__(self, path: List, value: Any) -> None: diff --git a/ubelt/util_io.py b/ubelt/util_io.py index 37a409acf..ea7c5196a 100644 --- a/ubelt/util_io.py +++ b/ubelt/util_io.py @@ -85,8 +85,8 @@ def writeto(fpath, to_write, aslines=False, verbose=None): if verbose: print('Writing to text file: %r ' % (fpath,)) - import ubelt as ub - ub.schedule_deprecation( + from ubelt import schedule_deprecation + schedule_deprecation( modname='ubelt', name='writeto', type='function', migration='use ubelt.Path(...).write_text() instead', deprecate='1.2.0', error='2.0.0', remove='2.1.0') @@ -128,8 +128,8 @@ def readfrom(fpath, aslines=False, errors='replace', verbose=None): print('Reading text file: %r ' % (fpath,)) if not exists(fpath): raise IOError('File %r does not exist' % (fpath,)) - import ubelt as ub - ub.schedule_deprecation( + from ubelt import schedule_deprecation + schedule_deprecation( modname='ubelt', name='readfrom', type='function', migration='use ubelt.Path(...).read_text() instead', deprecate='1.2.0', error='2.0.0', remove='2.1.0') diff --git a/ubelt/util_list.py b/ubelt/util_list.py index 7996d43c2..c682f4649 100644 --- a/ubelt/util_list.py +++ b/ubelt/util_list.py @@ -63,6 +63,9 @@ class chunks(object): List[T]: subsequent non-overlapping chunks of the input items + Attributes: + remainder (int): number of leftover items that don't divide cleanly + References: .. [SO_434287] http://stackoverflow.com/questions/434287/iterate-over-a-list-in-chunks diff --git a/ubelt/util_list.pyi b/ubelt/util_list.pyi index 9c60cc83f..8994b352b 100644 --- a/ubelt/util_list.pyi +++ b/ubelt/util_list.pyi @@ -15,8 +15,8 @@ KT = TypeVar("KT") class chunks: + remainder: int legacy: bool - remainder: Incomplete items: Iterable total: int | None nchunks: int | None diff --git a/ubelt/util_mixins.py b/ubelt/util_mixins.py index 73c9d691b..c1ca442bd 100644 --- a/ubelt/util_mixins.py +++ b/ubelt/util_mixins.py @@ -6,8 +6,8 @@ method, then the ``__nice__`` method defaults to something sensible, otherwise it is treated as abstract and raises ``NotImplementedError``. -To use simply have your object inherit from :class:`NiceRepr` -(multi-inheritance should be ok). +To use, have your object inherit from :class:`NiceRepr`. To customize, define +the ``__nice__`` method. Example: >>> # Objects that define __nice__ have a default __str__ and __repr__ diff --git a/ubelt/util_path.py b/ubelt/util_path.py index 356251e99..b9a3ae959 100644 --- a/ubelt/util_path.py +++ b/ubelt/util_path.py @@ -11,7 +11,7 @@ functionality is made redundant by :class:`Path`. For completeness these functions are listed -The :func:`expandpath` function expands the tilde to $HOME and environment +The :func:`expandpath` function expands the tilde to ``$HOME`` and environment variables to their values. The :func:`augpath` function creates variants of an existing path without @@ -34,9 +34,9 @@ ) import os import sys -from ubelt import util_io import pathlib import warnings +from ubelt import util_io __all__ = [ @@ -321,8 +321,8 @@ def ensuredir(dpath, mode=0o1777, verbose=0, recreate=False): dpath = join(*dpath) if recreate: - import ubelt as ub - ub.schedule_deprecation( + from ubelt import schedule_deprecation + schedule_deprecation( modname='ubelt', migration='Use ``ub.Path(dpath).delete().ensuredir()`` instead', name='recreate', type='argument of ensuredir', deprecate='1.3.0', error='2.0.0', @@ -349,11 +349,6 @@ class ChDir: exception that it will do nothing if the input path is None (i.e. the user did not want to change directories). - Args: - dpath (str | PathLike | None): - The new directory to work in. - If None, then the context manager is disabled. - SeeAlso: :func:`contextlib.chdir` @@ -388,6 +383,12 @@ class ChDir: >>> assert ub.Path.cwd() == dpath """ def __init__(self, dpath): + """ + Args: + dpath (str | PathLike | None): + The new directory to work in. + If None, then the context manager is disabled. + """ self._context_dpath = dpath self._orig_dpath = None @@ -419,7 +420,9 @@ class TempDir: """ Context for creating and cleaning up temporary directories. - DEPRECATED. Use :mod:`tempfile` instead. + Warning: + + DEPRECATED. Use :mod:`tempfile` instead. Note: This exists because :class:`tempfile.TemporaryDirectory` was @@ -427,7 +430,7 @@ class TempDir: python 2.7, this class will be deprecated. Attributes: - dpath (str | None) + dpath (str | None): the temporary path Note: # WE MAY WANT TO KEEP THIS FOR WINDOWS. @@ -448,8 +451,8 @@ class TempDir: >>> assert not exists(dpath) """ def __init__(self): - import ubelt as ub - ub.schedule_deprecation( + from ubelt import schedule_deprecation + schedule_deprecation( modname='ubelt', migration='Use tempfile instead', name='TempDir', type='class', deprecate='1.2.0', error='1.4.0', @@ -461,6 +464,10 @@ def __del__(self): self.cleanup() def ensure(self): + """ + Returns: + str: the path + """ import tempfile if not self.dpath: self.dpath = tempfile.mkdtemp() @@ -473,10 +480,18 @@ def cleanup(self): self.dpath = None def start(self): + """ + Returns: + TempDir: self + """ self.ensure() return self def __enter__(self): + """ + Returns: + TempDir: self + """ return self.start() def __exit__(self, ex_type, ex_value, ex_traceback): @@ -537,6 +552,9 @@ class Path(_PathBase): * :py:meth:`ubelt.Path.touch` - returns self to support chaining + * :py:meth:`ubelt.Path.chmod` - returns self to support chaining and + now accepts string-based permission codes. + Example: >>> # Ubelt extends pathlib functionality >>> import ubelt as ub @@ -604,7 +622,6 @@ class Path(_PathBase): * :py:meth:`pathlib.Path.mkdir` - we recommend :py:meth:`ubelt.Path.ensuredir` instead. - * :py:meth:`pathlib.Path.chmod` * :py:meth:`pathlib.Path.lchmod` * :py:meth:`pathlib.Path.unlink` @@ -777,7 +794,7 @@ def augment(self, prefix='', stemsuffix='', ext=None, stem=None, dpath=None, Returns: Path: augmented path - Note: + Warning: NOTICE OF BACKWARDS INCOMPATABILITY. THE INITIAL RELEASE OF Path.augment suffered from an unfortunate @@ -1070,6 +1087,89 @@ def shrinkuser(self, home='~'): new = self.__class__(shrunk) return new + def chmod(self, mode, follow_symlinks=True): + """ + Change the permissions of the path, like os.chmod(). + + Args: + mode (int | str): either a stat code to pass directly to + :func:`os.chmod` or a string-based code to construct modified + permissions. See note for details on the string-based chmod + codes. + + follow_symlinks (bool): + if True, and this path is a symlink, modify permission of the + file it points to, otherwise if False, modify the link + permission. + + Note: + From the chmod man page: + + The format of a symbolic mode is [ugoa...][[-+=][perms...]...], where + perms is either zero or more letters from the set rwxXst, or a single + letter from the set ugo. Multiple symbolic modes can be given, + separated by commas. + + Note: + Like :func:`os.chmod`, this may not work on Windows or on certain + filesystems. + + Returns: + Path: returns self for chaining + + Example: + >>> # xdoctest: +REQUIRES(POSIX) + >>> import ubelt as ub + >>> from ubelt.util_path import _encode_chmod_int + >>> dpath = ub.Path.appdir('ubelt/tests/chmod').ensuredir() + >>> fpath = (dpath / 'file.txt').touch() + >>> fpath.chmod('ugo+rw,ugo-x') + >>> print(_encode_chmod_int(fpath.stat().st_mode)) + u=rw,g=rw,o=rw + >>> fpath.chmod('o-rwx') + >>> print(_encode_chmod_int(fpath.stat().st_mode)) + u=rw,g=rw + >>> fpath.chmod(0o646) + >>> print(_encode_chmod_int(fpath.stat().st_mode)) + u=rw,g=r,o=rw + """ + if isinstance(mode, str): + # Resolve mode + # Follow symlinks was added to pathlib.Path.stat in 3.10 + # but os.stat has had it since 3.3, so use that instead. + old_mode = os.stat(self, follow_symlinks=follow_symlinks).st_mode + # old_mode = self.stat(follow_symlinks=follow_symlinks).st_mode + mode = _resolve_chmod_code(old_mode, mode) + os.chmod(self, mode, follow_symlinks=follow_symlinks) + return self + + # Should not need to modify unless we want chanability here. + # def lchmod(self, mode): + # """ + # Like chmod(), except if the path points to a symlink, the symlink's + # permissions are changed, rather than its target's. + # + # Args: + # mode (int | str): either a stat code to pass directly to + # :func:`os.chmod` or a string-based code to construct modified + # permissions. + # + # Returns: + # Path: returns self for chaining + # + # Example: + # >>> import ubelt as ub + # >>> from ubelt.util_path import _encode_chmod_int + # >>> dpath = ub.Path.appdir('ubelt/tests/chmod').ensuredir() + # >>> fpath = (dpath / 'file1.txt').delete().touch() + # >>> lpath = (dpath / 'link1.txt').delete() + # >>> lpath.symlink_to(fpath) + # >>> print(_encode_chmod_int(fpath.stat().st_mode)) + # >>> lpath.lchmod('a+rwx') + # >>> print(_encode_chmod_int(fpath.stat().st_mode)) + # """ + # return self.chmod(mode, follow_symlinks=False) + def touch(self, mode=0o666, exist_ok=True): """ Create this file with the given access mode, if it doesn't exist. @@ -1312,28 +1412,6 @@ def copy(self, dst, follow_file_symlinks=False, follow_dir_symlinks=False, the one case where a file is copied into an existing directory. In this case the name is used to construct a fully qualified destination. - Ignore: - # Enumerate cases - rows = [ - {'src': 'no-exist', 'dst': 'no-exist', 'result': 'error'}, - {'src': 'no-exist', 'dst': 'file', 'result': 'error'}, - {'src': 'no-exist', 'dst': 'dir', 'result': 'error'}, - - {'src': 'file', 'dst': 'no-exist', 'result': 'dst'}, - {'src': 'file', 'dst': 'dir', 'result': 'dst / src.name'}, - {'src': 'file', 'dst': 'file', 'result': 'error-or-overwrite-dst'}, - - {'src': 'dir', 'dst': 'no-exist', 'result': 'dst'}, - {'src': 'dir', 'dst': 'dir', 'result': 'error-or-overwrite-dst'}, - {'src': 'dir', 'dst': 'file', 'result': 'error'}, - ] - import pandas as pd - df = pd.DataFrame(rows) - piv = df.pivot(['src'], ['dst'], 'result') - print(piv.to_markdown(tablefmt="grid", index=True)) - - See: ~/code/ubelt/tests/test_path.py for test cases - Args: dst (str | PathLike): if ``src`` is a file and ``dst`` does not exist, copies this to ``dst`` @@ -1405,6 +1483,28 @@ def copy(self, dst, follow_file_symlinks=False, follow_dir_symlinks=False, >>> paths['empty_dpath'].copy((clone1 / 'empty_dpath_alt').ensuredir(), overwrite=True) >>> paths['nested_dpath'].copy(clone0 / 'nested_dpath') >>> paths['nested_dpath'].copy((clone1 / 'nested_dpath_alt').ensuredir(), overwrite=True) + + Ignore: + # Enumerate cases + rows = [ + {'src': 'no-exist', 'dst': 'no-exist', 'result': 'error'}, + {'src': 'no-exist', 'dst': 'file', 'result': 'error'}, + {'src': 'no-exist', 'dst': 'dir', 'result': 'error'}, + + {'src': 'file', 'dst': 'no-exist', 'result': 'dst'}, + {'src': 'file', 'dst': 'dir', 'result': 'dst / src.name'}, + {'src': 'file', 'dst': 'file', 'result': 'error-or-overwrite-dst'}, + + {'src': 'dir', 'dst': 'no-exist', 'result': 'dst'}, + {'src': 'dir', 'dst': 'dir', 'result': 'error-or-overwrite-dst'}, + {'src': 'dir', 'dst': 'file', 'result': 'error'}, + ] + import pandas as pd + df = pd.DataFrame(rows) + piv = df.pivot(['src'], ['dst'], 'result') + print(piv.to_markdown(tablefmt="grid", index=True)) + + See: ~/code/ubelt/tests/test_path.py for test cases """ import shutil copy_function = self._request_copy_function( @@ -1502,6 +1602,169 @@ def move(self, dst, follow_file_symlinks=False, follow_dir_symlinks=False, real_dst = shutil.move(self, dst, copy_function=copy_function) return Path(real_dst) + +def _parse_chmod_code(code): + """ + Expand a chmod code into a list of actions. + + Args: + code (str): of the form: [ugoa…][-+=]perms…[,…] + perms is either zero or more letters from the set rwxXst, or a + single letter from the set ugo. + + Yields: + Tuple[str, str, str]: target, op, and perms. + + The target is modified by the operation using the value. + target -- specified 'u' for user, 'g' for group, 'o' for other. + op -- specified as '+' to add, '-' to remove, or '=' to assign. + val -- specified as 'r' for read, 'w' for write, or 'x' for execute. + + Example: + >>> from ubelt.util_path import _parse_chmod_code + >>> print(list(_parse_chmod_code('ugo+rw,+r,g=rwx'))) + >>> print(list(_parse_chmod_code('o+x'))) + >>> print(list(_parse_chmod_code('u-x'))) + >>> print(list(_parse_chmod_code('x'))) + >>> print(list(_parse_chmod_code('ugo+rwx'))) + [('ugo', '+', 'rw'), ('ugo', '+', 'r'), ('g', '=', 'rwx')] + [('o', '+', 'x')] + [('u', '-', 'x')] + [('u', '+', 'x')] + [('ugo', '+', 'rwx')] + >>> import pytest + >>> with pytest.raises(ValueError): + >>> list(_parse_chmod_code('a+b+c')) + """ + import re + pat = re.compile(r'([\+\-\=])') + parts = code.split(',') + for part in parts: + ab = pat.split(part) + len_ab = len(ab) + if len_ab == 3: + targets, op, perms = ab + elif len_ab == 1: + perms = ab[0] + op = '+' + targets = 'u' + else: + raise ValueError('unknown chmod code pattern: part={part}') + if targets == '' or targets == 'a': + targets = 'ugo' + yield (targets, op, perms) + + +def _resolve_chmod_code(old_mode, code): + """ + Modifies integer stat permissions based on a string code. + + Args: + old_mode (int): old mode from st_stat + code (str): chmod style codeold mode from st_stat + + Returns: + int : new code + + Example: + >>> from ubelt.util_path import _resolve_chmod_code + >>> print(oct(_resolve_chmod_code(0, '+rwx'))) + >>> print(oct(_resolve_chmod_code(0, 'ugo+rwx'))) + >>> print(oct(_resolve_chmod_code(0, 'a-rwx'))) + >>> print(oct(_resolve_chmod_code(0, 'u+rw,go+r,go-wx'))) + >>> print(oct(_resolve_chmod_code(0o777, 'u+rw,go+r,go-wx'))) + 0o777 + 0o777 + 0o0 + 0o644 + 0o744 + >>> import pytest + >>> with pytest.raises(NotImplementedError): + >>> print(oct(_resolve_chmod_code(0, 'u=rw'))) + >>> with pytest.raises(ValueError): + >>> _resolve_chmod_code(0, 'u?w') + """ + import stat + import itertools as it + action_lut = { + # TODO: handle suid, sgid, and sticky? + # suid = stat.S_ISUID + # sgid = stat.S_ISGID + # sticky = stat.S_ISVTX + 'ur' : stat.S_IRUSR, + 'uw' : stat.S_IWUSR, + 'ux' : stat.S_IXUSR, + + 'gr' : stat.S_IRGRP, + 'gw' : stat.S_IWGRP, + 'gx' : stat.S_IXGRP, + + 'or' : stat.S_IROTH, + 'ow' : stat.S_IWOTH, + 'ox' : stat.S_IXOTH, + } + actions = _parse_chmod_code(code) + new_mode = int(old_mode) # (could optimize to modify inplace if needed) + for action in actions: + targets, op, perms = action + try: + action_keys = (target + perm for target, perm in it.product(targets, perms)) + action_values = (action_lut[key] for key in action_keys) + action_values = list(action_values) + if op == '+': + for val in action_values: + new_mode |= val + elif op == '-': + for val in action_values: + new_mode &= (~val) + elif op == '=': + raise NotImplementedError(f'new chmod code for op={op}') + else: + raise AssertionError( + f'should not be able to get here. unknown op code: op={op}') + except KeyError: + # Give a better error message if something goes wrong + raise ValueError(f'Unknown action: {action}') + return new_mode + + +def _encode_chmod_int(int_code): + """ + Convert a chmod integer code to a string + + Currently unused, but may be useful in the future. + + Example: + >>> from ubelt.util_path import _encode_chmod_int + >>> int_code = 0o744 + >>> print(_encode_chmod_int(int_code)) + u=rwx,g=r,o=r + """ + import stat + action_lut = { + 'ur' : stat.S_IRUSR, + 'uw' : stat.S_IWUSR, + 'ux' : stat.S_IXUSR, + + 'gr' : stat.S_IRGRP, + 'gw' : stat.S_IWGRP, + 'gx' : stat.S_IXGRP, + + 'or' : stat.S_IROTH, + 'ow' : stat.S_IWOTH, + 'ox' : stat.S_IXOTH, + } + from collections import defaultdict + target_to_perms = defaultdict(list) + for key, val in action_lut.items(): + target, perm = key + if int_code & val: + target_to_perms[target].append(perm) + parts = [k + '=' + ''.join(vs) for k, vs in target_to_perms.items()] + code = ','.join(parts) + return code + + if sys.version_info[0:2] < (3, 8): # nocover # Vendor in a nearly modern copytree for Python 3.6 and 3.7 diff --git a/ubelt/util_path.pyi b/ubelt/util_path.pyi index 36f08b10f..4b493073f 100644 --- a/ubelt/util_path.pyi +++ b/ubelt/util_path.pyi @@ -4,7 +4,6 @@ from typing import Type from types import TracebackType from typing import List from typing import Callable -from _typeshed import Incomplete from collections.abc import Generator @@ -41,7 +40,7 @@ def ensuredir(dpath: str | PathLike | Tuple[str | PathLike], class ChDir: - def __init__(self, dpath) -> None: + def __init__(self, dpath: str | PathLike | None) -> None: ... def __enter__(self) -> ChDir: @@ -54,7 +53,7 @@ class ChDir: class TempDir: - dpath: Incomplete + dpath: str | None def __init__(self) -> None: ... @@ -62,16 +61,16 @@ class TempDir: def __del__(self) -> None: ... - def ensure(self): + def ensure(self) -> str: ... def cleanup(self) -> None: ... - def start(self): + def start(self) -> TempDir: ... - def __enter__(self): + def __enter__(self) -> TempDir: ... def __exit__(self, ex_type: Type[BaseException] | None, diff --git a/ubelt/util_platform.py b/ubelt/util_platform.py index fa27a3634..9f3ce053b 100644 --- a/ubelt/util_platform.py +++ b/ubelt/util_platform.py @@ -4,7 +4,8 @@ Standard application directory structure: cache, config, and other XDG standards [XDG_Spec]_. This is similar to the more focused :mod:`appdirs` -module [AS_appdirs]_. In the future ubelt may directly use :mod:`appdirs`. +module [AS_appdirs]_ (deprecated as of 2023-02-10) and its successor +:mod:`platformdirs` [PlatDirs]_. Note: Table mapping the type of directory to the system default environment @@ -34,6 +35,7 @@ .. [SO_11113974] https://stackoverflow.com/questions/11113974/cross-plat-path .. [harawata_appdirs] https://github.com/harawata/appdirs#supported-directories .. [AS_appdirs] https://github.com/ActiveState/appdirs + .. [PlatDirs] https://pypi.org/project/platformdirs/ """ import os import sys @@ -138,8 +140,8 @@ def get_app_data_dir(appname, *args): SeeAlso: :func:`ensure_app_data_dir` """ - import ubelt as ub - ub.schedule_deprecation( + from ubelt.util_deprecate import schedule_deprecation + schedule_deprecation( modname='ubelt', name='get_app_data_dir and ensure_app_data_dir', type='function', migration='use ubelt.Path.appdir(type="data") instead', deprecate='1.2.0', error='2.0.0', remove='2.1.0') @@ -195,8 +197,8 @@ def get_app_config_dir(appname, *args): SeeAlso: :func:`ensure_app_config_dir` """ - import ubelt as ub - ub.schedule_deprecation( + from ubelt.util_deprecate import schedule_deprecation + schedule_deprecation( modname='ubelt', name='get_app_config_dir and ensure_app_config_dir', type='function', migration='use ubelt.Path.appdir(type="config") instead', deprecate='1.2.0', error='2.0.0', remove='2.1.0') @@ -255,8 +257,8 @@ def get_app_cache_dir(appname, *args): SeeAlso: :func:`ensure_app_cache_dir` """ - import ubelt as ub - ub.schedule_deprecation( + from ubelt.util_deprecate import schedule_deprecation + schedule_deprecation( modname='ubelt', name='get_app_cache_dir and ensure_app_cache_dir', type='function', migration='use ubelt.Path.appdir(type="cache") instead', deprecate='1.2.0', error='2.0.0', remove='2.1.0') diff --git a/ubelt/util_repr.py b/ubelt/util_repr.py index c5845a3f9..6a3786366 100644 --- a/ubelt/util_repr.py +++ b/ubelt/util_repr.py @@ -1,6 +1,6 @@ """ Defines the function :func:`urepr`, which allows for a bit more customization -than :func:`repr` or :func:`pprint`. See the docstring for more details. +than :func:`repr` or :func:`pprint.pformat`. See the docstring for more details. Two main goals of urepr are to provide nice string representations of nested @@ -28,9 +28,10 @@ You can also define or overwrite how representations for different types are created. You can either create your own extension object, or you can -monkey-patch `ub.util_repr._REPR_EXTENSIONS` without specifying the +monkey-patch ``ub.util_repr._REPR_EXTENSIONS`` without specifying the extensions keyword argument (although this will be a global change). +>>> import ubelt as ub >>> extensions = ub.util_repr.ReprExtensions() >>> @extensions.register(float) >>> def my_float_formater(data, **kw): @@ -39,10 +40,11 @@ {1: monkey(nan), 2: monkey(inf), 3: monkey(3.0)} As of ubelt 1.1.0 you can now access and update the default extensions via the -urepr function itself. +``EXTENSIONS`` attribute of the :func:`urepr` function itself. >>> # xdoctest: +SKIP >>> # We skip this at test time to not modify global state +>>> import ubelt as ub >>> @ub.urepr.EXTENSIONS.register(float) >>> def my_float_formater(data, **kw): >>> return "monkey2({})".format(data) diff --git a/ubelt/util_str.py b/ubelt/util_str.py index 95562b6e1..18b901bf1 100644 --- a/ubelt/util_str.py +++ b/ubelt/util_str.py @@ -1,9 +1,6 @@ """ Functions for working with text and strings. -The :func:`ensure_unicode` function does its best to coerce Python 2/3 bytes -and text into a consistent unicode text representation. - The :func:`codeblock` and :func:`paragraph` wrap multiline strings to help write text blocks without hindering the surrounding code indentation. @@ -181,7 +178,11 @@ def hzcat(args, sep=''): def ensure_unicode(text): r""" - Casts bytes into utf8 (mostly for python2 compatibility) + Casts bytes into utf8 (mostly for python2 compatibility). + + Warning: + This function is deprecated and will no longer be available in version + 2.0.0. Args: text (str | bytes): @@ -191,7 +192,7 @@ def ensure_unicode(text): str References: - [SO_12561063] http://stackoverflow.com/questions/12561063/extract-data-from-file + .. [SO_12561063] http://stackoverflow.com/questions/12561063/extract-data-from-file Example: >>> from ubelt.util_str import * @@ -202,8 +203,8 @@ def ensure_unicode(text): >>> assert ensure_unicode('text1'.encode('utf8')) == 'text1' >>> assert (codecs.BOM_UTF8 + 'text»¿'.encode('utf8')).decode('utf8') """ - import ubelt as ub - ub.schedule_deprecation( + from ubelt.util_deprecate import schedule_deprecation + schedule_deprecation( modname='ubelt', name='ensure_unicode', type='function', migration='This should not be needed in Python 3', deprecate='1.2.0', error='2.0.0', remove='2.1.0') diff --git a/ubelt/util_time.py b/ubelt/util_time.py index 3f249d545..750dfad2f 100644 --- a/ubelt/util_time.py +++ b/ubelt/util_time.py @@ -37,8 +37,8 @@ def _needs_workaround39103(): singer-python also had a similar issue: https://github.com/singer-io/singer-python/issues/86 - See Also - https://github.com/jaraco/tempora/blob/main/tempora/__init__.py#L59 + See Also: + https://github.com/jaraco/tempora/blob/main/tempora/__init__.py#L59 """ from datetime import datetime as datetime_cls return len(datetime_cls(1, 1, 1).strftime('%Y')) != 4 @@ -223,9 +223,9 @@ def timeparse(stamp, default_timezone='local', allow_dateutil=True): Create a :class:`datetime.datetime` object from a string timestamp. Without any extra dependencies this will parse the output of - :func:`ubelt.util_time.timestamp()` into a datetime object. In the case - where the format differs, :func:`dateutil.parser.parse` will be used if the - :py:mod:`python-dateutil` package is installed. + :func:`ubelt.util_time.timestamp` into a datetime object. In the case + where the format differs, :func:`dateutil.parser.parse` will be used + if the :py:mod:`python-dateutil` package is installed. Args: stamp (str): @@ -479,23 +479,11 @@ class Timer(object): Measures time elapsed between a start and end point. Can be used as a with-statement context manager, or using the tic/toc api. - Args: - label (str, default=''): - identifier for printing - - verbose (int, default=None): - verbosity flag, defaults to True if label is given, otherwise 0. - - newline (bool, default=True): - if False and verbose, print tic and toc on the same line. - - ns (bool, default=False): - if True, a nano-second resolution timer to avoid precision loss - caused by the float type. - Attributes: elapsed (float): number of seconds measured by the context manager tstart (float): time of last `tic` reported by `self._time()` + write (Callable): function used to write + flush (Callable): function used to flush Example: >>> # Create and start the timer using the context manager @@ -535,6 +523,22 @@ class Timer(object): _default_time = time.perf_counter def __init__(self, label='', verbose=None, newline=True, ns=False): + """ + Args: + label (str): + identifier for printing. Default to ''. + + verbose (int | None): + verbosity flag, defaults to True if label is given, otherwise 0. + + newline (bool): + if False and verbose, print tic and toc on the same line. + Defaults to True. + + ns (bool): + if True, a nano-second resolution timer to avoid precision loss + caused by the float type. Defaults to False. + """ if verbose is None: verbose = bool(label) self.label = label @@ -551,7 +555,12 @@ def __init__(self, label='', verbose=None, newline=True, ns=False): self._time = self._default_time def tic(self): - """ starts the timer """ + """ + starts the timer + + Returns: + Timer: self + """ if self.verbose: self.flush() self.write('\ntic(%r)' % self.label) @@ -562,7 +571,12 @@ def tic(self): return self def toc(self): - """ stops the timer """ + """ + stops the timer + + Returns: + float | int: number of second or nanoseconds + """ elapsed = self._time() - self.tstart if self.verbose: if self.ns: @@ -573,6 +587,10 @@ def toc(self): return elapsed def __enter__(self): + """ + Returns: + Timer: self + """ self.tic() return self diff --git a/ubelt/util_time.pyi b/ubelt/util_time.pyi index bf3738819..23c808b7a 100644 --- a/ubelt/util_time.pyi +++ b/ubelt/util_time.pyi @@ -1,7 +1,7 @@ import datetime +from typing import Callable from typing import Type from types import TracebackType -from _typeshed import Incomplete def timestamp(datetime: datetime.datetime | datetime.date | None = None, @@ -20,27 +20,27 @@ def timeparse(stamp: str, class Timer: elapsed: float tstart: float - label: Incomplete - verbose: Incomplete - newline: Incomplete - write: Incomplete - flush: Incomplete - ns: Incomplete + write: Callable + flush: Callable + label: str + verbose: int | None + newline: bool + ns: bool def __init__(self, - label: str = ..., - verbose: Incomplete | None = ..., - newline: bool = ..., - ns: bool = ...) -> None: + label: str = '', + verbose: int | None = None, + newline: bool = True, + ns: bool = False) -> None: ... - def tic(self): + def tic(self) -> Timer: ... - def toc(self): + def toc(self) -> float | int: ... - def __enter__(self): + def __enter__(self) -> Timer: ... def __exit__(self, ex_type: Type[BaseException] | None, diff --git a/ubelt/util_zip.py b/ubelt/util_zip.py index 6bd218fa6..ec9ea96e8 100644 --- a/ubelt/util_zip.py +++ b/ubelt/util_zip.py @@ -95,6 +95,10 @@ class zopen(NiceRepr): open zipfile reference maybe?). - [ ] Write mode in some restricted setting? + Attributes: + name (str | PathLike): + path to a file or reference to an item in a zipfile. + Example: >>> from ubelt.util_zip import * # NOQA >>> import pickle @@ -317,8 +321,8 @@ def _cleanup(self): self._handle = None if self._temp_dpath and exists(self._temp_dpath): # os.unlink(self._temp_dpath) - import ubelt as ub - ub.delete(self._temp_dpath) + from ubelt.util_io import delete + delete(self._temp_dpath) def __del__(self): self._cleanup() diff --git a/ubelt/util_zip.pyi b/ubelt/util_zip.pyi index 6b1344b5d..c549ded10 100644 --- a/ubelt/util_zip.pyi +++ b/ubelt/util_zip.pyi @@ -2,7 +2,6 @@ from os import PathLike from typing import Tuple from typing import Type from types import TracebackType -from _typeshed import Incomplete from ubelt.util_mixins import NiceRepr @@ -12,9 +11,9 @@ def split_archive(fpath: str | PathLike, class zopen(NiceRepr): + name: str | PathLike fpath: str | PathLike ext: str - name: Incomplete mode: str def __init__(self,