diff --git a/requirements/runtime.txt b/requirements/runtime.txt index 1b512a1..1d34236 100644 --- a/requirements/runtime.txt +++ b/requirements/runtime.txt @@ -7,7 +7,7 @@ xdev>=1.3.3 ; python_version >= '3.6' # Python 3.6+ -ubelt>=1.3.2 ; python_version >= '3.6' # Python 3.6+ +ubelt>=1.3.6 ; python_version >= '3.6' # Python 3.6+ toml>=0.9.6 ; python_version >= '2.7' # Python 2.7+ diff --git a/xcookie/builders/gitlab_ci.py b/xcookie/builders/gitlab_ci.py index 3795204..7539466 100644 --- a/xcookie/builders/gitlab_ci.py +++ b/xcookie/builders/gitlab_ci.py @@ -548,19 +548,25 @@ def build_deploy_job(common_template, deploy_image, wheelhouse_dpath, self): export PROJECT_VERSION=$(python -c "import setup; print(setup.VERSION)") echo "PROJECT_VERSION=$PROJECT_VERSION" - # --header "PRIVATE-TOKEN: $PRIVATE_GITLAB_TOKEN" \ + # If running on CI use CI authentication, otherwise + # assume we developer authentication available. + if [[ -z "$CI_JOB_TOKEN" ]]; then + AUTH_HEADER="PRIVATE-TOKEN: $PRIVATE_GITLAB_TOKEN" + else + AUTH_HEADER="JOB-TOKEN: $CI_JOB_TOKEN" + fi - # We will loop over all of the assets in the wheelhouse (i.e. - # dist) and upload them to a package registery. We also store - # the links to the artifacts so we can attach them to a release + # Loop over all of the assets in the wheelhouse (i.e. dist) + # and upload them to a package registery. We also store the + # links to the artifacts so we can attach them to a release # page. PACKAGE_ARTIFACT_ARRAY=() for FPATH in "{wheelhouse_dpath}"/*; do FNAME=$(basename $FPATH) - echo $FNAME + echo "Upload artifact: $FNAME" PACKAGE_URL="$CI_API_V4_URL/projects/$CI_PROJECT_ID/packages/generic/$CI_PROJECT_NAME/$PROJECT_VERSION/$FNAME" curl \ - --header "JOB-TOKEN: $CI_JOB_TOKEN" \ + --header "$AUTH_HEADER" \ --upload-file $FPATH \ "$PACKAGE_URL" PACKAGE_ARTIFACT_ARRAY+=("$PACKAGE_URL") @@ -600,14 +606,19 @@ def build_deploy_job(common_template, deploy_image, wheelhouse_dpath, self): export PROJECT_VERSION=$(python -c "import setup; print(setup.VERSION)") - # Building this dummy variable requires some wheels built in the local dir + # Building this dummy variable requires some wheels built + # in the local dir, the next step wont use them directly + # it will just use their names, and only point to the ones + # in the package registry. + DO_UPLOAD=0 DO_TAG=0 ./publish.sh + export PACKAGE_ARTIFACT_ARRAY=() for FPATH in "dist"/*; do FNAME=$(basename $FPATH) PACKAGE_URL="$CI_API_V4_URL/projects/$CI_PROJECT_ID/packages/generic/$CI_PROJECT_NAME/$PROJECT_VERSION/$FNAME" - echo "$PACKAGE_URL" PACKAGE_ARTIFACT_ARRAY+=("$PACKAGE_URL") done + echo "PACKAGE_ARTIFACT_ARRAY=${PACKAGE_ARTIFACT_ARRAY[@]}" """ __note__ deploy_script += [ @@ -637,9 +648,17 @@ def build_deploy_job(common_template, deploy_image, wheelhouse_dpath, self): }" echo "$RELEASE_DATA_JSON" + # If running on CI use CI authentication, otherwise + # assume we developer authentication available. + if [[ -z "${CI_JOB_TOKEN}" ]]; then + AUTH_HEADER="PRIVATE-TOKEN: $PRIVATE_GITLAB_TOKEN" + else + AUTH_HEADER="JOB-TOKEN: $CI_JOB_TOKEN" + fi + curl \ --header 'Content-Type: application/json' \ - --header "PRIVATE-TOKEN: $PRIVATE_GITLAB_TOKEN" \ + --header "$AUTH_HEADER" \ --data "$RELEASE_DATA_JSON" \ --request POST \ "$CI_API_V4_URL/projects/$CI_PROJECT_ID/releases" diff --git a/xcookie/builders/setup.py b/xcookie/builders/setup.py index bc47f4f..01a03a4 100644 --- a/xcookie/builders/setup.py +++ b/xcookie/builders/setup.py @@ -108,14 +108,13 @@ def build_setup(self): extras = ['runtime', 'tests', 'optional'] - parts.append(ub.identity( + parts.append(ub.codeblock( ''' -if __name__ == '__main__': - setupkw = {} - - setupkw['install_requires'] = parse_requirements('requirements/runtime.txt', versions='loose') - setupkw['extras_require'] = { + if __name__ == '__main__': + setupkw = {} + setupkw['install_requires'] = parse_requirements('requirements/runtime.txt', versions='loose') + setupkw['extras_require'] = { ''')) # 'all': parse_requirements('requirements.txt', versions='loose'), # 'tests': parse_requirements('requirements/tests.txt', versions='loose'), @@ -201,15 +200,16 @@ def build_setup(self): setupkw_parts['classifiers'] = f'{classifier_text}' package_data = {} + package_data[''] = [ 'requirements/*.txt' ] - package_data[f'{self.mod_name}.rc.requirements'] = [ - 'docs.txt', - 'optional.txt', - 'runtime.txt', - 'tests.txt', - ] + # HACK: need to determine if we have the modname.rc structure or not + # for now just check if it exists, which wont work the first time. + has_rc_requirements = (self.mod_dpath / 'rc/requirements').exists() + if has_rc_requirements: + setupkw_parts['include_package_data'] = True + package_data[f'{self.mod_name}.rc.requirements'] = ['*.txt'] if self.config['typed']: package_data[self.mod_name] = ['py.typed', '*.pyi'] diff --git a/xcookie/main.py b/xcookie/main.py index f1aecce..d6ce42f 100755 --- a/xcookie/main.py +++ b/xcookie/main.py @@ -294,8 +294,6 @@ def __post_init__(self): from xcookie.constants import KNOWN_PYTHON_VERSIONS min_python = str(self['min_python']).lower() max_python = str(self['max_python']).lower() - print(f'min_python={min_python}') - print(f'max_python={max_python}') def satisfies_minmax(v): v = Version(v) @@ -311,8 +309,6 @@ def satisfies_minmax(v): python_versions = [v for v in KNOWN_PYTHON_VERSIONS if satisfies_minmax(v)] - print(f'KNOWN_PYTHON_VERSIONS={KNOWN_PYTHON_VERSIONS}') - print(f'python_versions={python_versions}') self['supported_python_versions'] = python_versions if self['ci_cpython_versions'] == 'auto': @@ -590,11 +586,11 @@ def _build_template_registry(self): # {'template': 0, 'overwrite': 1, 'fname': 'dev/make_strict_req.sh', 'perms': 'x'}, - {'template': 0, 'overwrite': 1, 'fname': 'requirements.txt', 'dynamic': 'build_requirements'}, - {'template': 0, 'overwrite': 1, 'fname': 'requirements/graphics.txt', 'tags': 'cv2'}, - {'template': 0, 'overwrite': 1, 'fname': 'requirements/headless.txt', 'tags': 'cv2'}, - {'template': 0, 'overwrite': 1, 'fname': 'requirements/gdal.txt', 'tags': 'gdal'}, - {'template': 0, 'overwrite': 1, 'fname': 'requirements/gdal-strict.txt', 'tags': 'gdal'}, + {'template': 0, 'overwrite': 1, 'fname': 'requirements.txt', 'dynamic': 'build_requirements_txt'}, + {'template': 1, 'overwrite': 1, 'fname': 'requirements/graphics.txt', 'tags': 'cv2', 'dynamic': 'build_cv2_graphics_requirements_txt'}, + {'template': 1, 'overwrite': 1, 'fname': 'requirements/headless.txt', 'tags': 'cv2', 'dynamic': 'build_cv2_headless_requirements_txt'}, + {'template': 1, 'overwrite': 1, 'fname': 'requirements/gdal.txt', 'tags': 'gdal', 'dynamic': 'build_gdal_requirements_txt'}, + {'template': 0, 'overwrite': 0, 'fname': 'requirements/optional.txt'}, {'template': 0, 'overwrite': 0, 'fname': 'requirements/runtime.txt'}, {'template': 0, 'overwrite': 0, 'fname': 'requirements/tests.txt'}, @@ -950,6 +946,8 @@ def _stage_file(self, info): raise SkipFile('file was disabled') try: stage_fpath.write_text(text) + if 'x' in info.get('perms', ''): + stage_fpath.chmod('+x') except Exception: print(f'text={text}') raise @@ -1152,7 +1150,7 @@ def gather_tasks(self): print('stats = {}'.format(ub.urepr(stats, nl=2))) return stats, tasks - def build_requirements(self): + def build_requirements_txt(self): # existing = (self.repodir / 'requirements').ls() candidate_all_requirements = [ 'requirements/runtime.txt', @@ -1318,6 +1316,78 @@ def build_docs_requirements(self): from xcookie.builders import docs return docs.build_docs_requirements(self) + def _build_special_requirements(self, variant, version_defaults, header_lines): + """ + Example: + >>> from xcookie.main import * # NOQA + >>> dpath = ub.Path.appdir('xcookie/tests/test-stage').delete().ensuredir() + >>> kwargs = { + >>> 'repodir': dpath / 'testrepo', + >>> 'tags': ['gitlab', 'kitware', 'purepy', 'cv2'], + >>> 'rotate_secrets': False, + >>> 'is_new': False, + >>> 'min_python': '3.9', + >>> 'max_python': '3.12', + >>> 'interactive': False, + >>> } + >>> config = XCookieConfig.cli(cmdline=0, data=kwargs) + >>> print('config = {}'.format(ub.urepr(dict(config), nl=1))) + >>> self = TemplateApplier(config) + >>> print(chr(10) + 'headless.txt') + >>> print(self.build_cv2_headless_requirements_txt()) + >>> print(chr(10) + 'gdal.txt') + >>> print(self.build_gdal_requirements_txt()) + """ + req_lines = ['# Generated dynamically via: ~/code/xcookie/xcookie/main.py::TemplateApplier._build_special_requirements'] + req_lines.extend(header_lines) + max_pyver = Version(self.config['max_python'] or '4.0') + min_pyver = Version(self.config['min_python']) + + for row in version_defaults: + lt = row['pyver_lt'] + # lt = min(row['pyver_lt'], max_pyver) # FIXME, exclusive vs inclusive + ge = max(row['pyver_ge'], min_pyver) + skip = row['pyver_ge'] > max_pyver + skip |= row['pyver_lt'] < min_pyver + # print(lt, ge, skip, min_pyver, max_pyver, row) + if not skip: + req_lines.append(f'{variant}{row["version"]} ; python_version < {lt} and python_version >= {ge}') + req_text = '\n'.join(req_lines) + return req_text + + def _build_cv2_requirements(self, variant): + header_lines = [ + f'# xdev availpkg {variant}', + '# --prefer-binary', + ] + version_defaults = [ + {'version': '>=4.5.4.58', 'pyver_ge': Version('3.10'), 'pyver_lt': Version('4.0')}, + {'version': '>=3.4.15.55', 'pyver_ge': Version('3.6'), 'pyver_lt': Version('3.9')}, + {'version': '>=3.4.2.16', 'pyver_ge': Version('2.7'), 'pyver_lt': Version('3.6')}, + ] + return self._build_special_requirements(variant, version_defaults, header_lines) + + def build_cv2_headless_requirements_txt(self): + variant = 'opencv-python-headless' + return self._build_cv2_requirements(variant) + + def build_cv2_graphics_requirements_txt(self): + variant = 'opencv-python' + return self._build_cv2_requirements(variant) + + def build_gdal_requirements_txt(self): + # TODO: make more dynamic + variant = 'GDAL' + header_lines = [ + '--find-links https://girder.github.io/large_image_wheels', + ] + version_defaults = [ + {'version': '>=3.7.2', 'pyver_ge': Version('3.12'), 'pyver_lt': Version('4.0')}, + {'version': '>=3.5.2', 'pyver_ge': Version('3.11'), 'pyver_lt': Version('3.12')}, + {'version': '>=3.4.1', 'pyver_ge': Version('3.6'), 'pyver_lt': Version('3.11')}, + ] + return self._build_special_requirements(variant, version_defaults, header_lines) + def build_run_doctests(self): return ub.codeblock( f''' diff --git a/xcookie/rc/setup.py.in b/xcookie/rc/setup.py.in index 0268e9e..34a9873 100644 --- a/xcookie/rc/setup.py.in +++ b/xcookie/rc/setup.py.in @@ -186,7 +186,8 @@ def parse_requirements(fname='requirements.txt', versions=False): if plat_deps is not None: parts.append(';' + plat_deps) item = ''.join(parts) - yield item + if item: + yield item packages = list(gen_packages_items()) return packages