Skip to content

Commit

Permalink
Better setup perms
Browse files Browse the repository at this point in the history
  • Loading branch information
@Erotemic
Erotemic committed Sep 10, 2024
1 parent 8650b70 commit f53d4c8
Show file tree
Hide file tree
Showing 5 changed files with 123 additions and 33 deletions.
2 changes: 1 addition & 1 deletion requirements/runtime.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

xdev>=1.3.3 ; python_version >= '3.6' # Python 3.6+

ubelt>=1.3.2 ; python_version >= '3.6' # Python 3.6+
ubelt>=1.3.6 ; python_version >= '3.6' # Python 3.6+

toml>=0.9.6 ; python_version >= '2.7' # Python 2.7+

Expand Down
37 changes: 28 additions & 9 deletions xcookie/builders/gitlab_ci.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -548,19 +548,25 @@ def build_deploy_job(common_template, deploy_image, wheelhouse_dpath, self):
export PROJECT_VERSION=$(python -c "import setup; print(setup.VERSION)")
echo "PROJECT_VERSION=$PROJECT_VERSION"
# --header "PRIVATE-TOKEN: $PRIVATE_GITLAB_TOKEN" \
# If running on CI use CI authentication, otherwise
# assume we developer authentication available.
if [[ -z "$CI_JOB_TOKEN" ]]; then
AUTH_HEADER="PRIVATE-TOKEN: $PRIVATE_GITLAB_TOKEN"
else
AUTH_HEADER="JOB-TOKEN: $CI_JOB_TOKEN"
fi
# We will loop over all of the assets in the wheelhouse (i.e.
# dist) and upload them to a package registery. We also store
# the links to the artifacts so we can attach them to a release
# Loop over all of the assets in the wheelhouse (i.e. dist)
# and upload them to a package registery. We also store the
# links to the artifacts so we can attach them to a release
# page.
PACKAGE_ARTIFACT_ARRAY=()
for FPATH in "{wheelhouse_dpath}"/*; do
FNAME=$(basename $FPATH)
echo $FNAME
echo "Upload artifact: $FNAME"
PACKAGE_URL="$CI_API_V4_URL/projects/$CI_PROJECT_ID/packages/generic/$CI_PROJECT_NAME/$PROJECT_VERSION/$FNAME"
curl \
--header "JOB-TOKEN: $CI_JOB_TOKEN" \
--header "$AUTH_HEADER" \
--upload-file $FPATH \
"$PACKAGE_URL"
PACKAGE_ARTIFACT_ARRAY+=("$PACKAGE_URL")
Expand Down Expand Up @@ -600,14 +606,19 @@ def build_deploy_job(common_template, deploy_image, wheelhouse_dpath, self):
export PROJECT_VERSION=$(python -c "import setup; print(setup.VERSION)")
# Building this dummy variable requires some wheels built in the local dir
# Building this dummy variable requires some wheels built
# in the local dir, the next step wont use them directly
# it will just use their names, and only point to the ones
# in the package registry.
DO_UPLOAD=0 DO_TAG=0 ./publish.sh
export PACKAGE_ARTIFACT_ARRAY=()
for FPATH in "dist"/*; do
FNAME=$(basename $FPATH)
PACKAGE_URL="$CI_API_V4_URL/projects/$CI_PROJECT_ID/packages/generic/$CI_PROJECT_NAME/$PROJECT_VERSION/$FNAME"
echo "$PACKAGE_URL"
PACKAGE_ARTIFACT_ARRAY+=("$PACKAGE_URL")
done
echo "PACKAGE_ARTIFACT_ARRAY=${PACKAGE_ARTIFACT_ARRAY[@]}"
"""
__note__
deploy_script += [
Expand Down Expand Up @@ -637,9 +648,17 @@ def build_deploy_job(common_template, deploy_image, wheelhouse_dpath, self):
}"
echo "$RELEASE_DATA_JSON"
# If running on CI use CI authentication, otherwise
# assume we developer authentication available.
if [[ -z "${CI_JOB_TOKEN}" ]]; then
AUTH_HEADER="PRIVATE-TOKEN: $PRIVATE_GITLAB_TOKEN"
else
AUTH_HEADER="JOB-TOKEN: $CI_JOB_TOKEN"
fi
curl \
--header 'Content-Type: application/json' \
--header "PRIVATE-TOKEN: $PRIVATE_GITLAB_TOKEN" \
--header "$AUTH_HEADER" \
--data "$RELEASE_DATA_JSON" \
--request POST \
"$CI_API_V4_URL/projects/$CI_PROJECT_ID/releases"
Expand Down
24 changes: 12 additions & 12 deletions xcookie/builders/setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,14 +108,13 @@ def build_setup(self):

extras = ['runtime', 'tests', 'optional']

parts.append(ub.identity(
parts.append(ub.codeblock(
'''
if __name__ == '__main__':
setupkw = {}
setupkw['install_requires'] = parse_requirements('requirements/runtime.txt', versions='loose')
setupkw['extras_require'] = {
if __name__ == '__main__':
setupkw = {}
setupkw['install_requires'] = parse_requirements('requirements/runtime.txt', versions='loose')
setupkw['extras_require'] = {
'''))
# 'all': parse_requirements('requirements.txt', versions='loose'),
# 'tests': parse_requirements('requirements/tests.txt', versions='loose'),
Expand Down Expand Up @@ -201,15 +200,16 @@ def build_setup(self):
setupkw_parts['classifiers'] = f'{classifier_text}'

package_data = {}

package_data[''] = [
'requirements/*.txt'
]
package_data[f'{self.mod_name}.rc.requirements'] = [
'docs.txt',
'optional.txt',
'runtime.txt',
'tests.txt',
]
# HACK: need to determine if we have the modname.rc structure or not
# for now just check if it exists, which wont work the first time.
has_rc_requirements = (self.mod_dpath / 'rc/requirements').exists()
if has_rc_requirements:
setupkw_parts['include_package_data'] = True
package_data[f'{self.mod_name}.rc.requirements'] = ['*.txt']

if self.config['typed']:
package_data[self.mod_name] = ['py.typed', '*.pyi']
Expand Down
90 changes: 80 additions & 10 deletions xcookie/main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -294,8 +294,6 @@ def __post_init__(self):
from xcookie.constants import KNOWN_PYTHON_VERSIONS
min_python = str(self['min_python']).lower()
max_python = str(self['max_python']).lower()
print(f'min_python={min_python}')
print(f'max_python={max_python}')

def satisfies_minmax(v):
v = Version(v)
Expand All @@ -311,8 +309,6 @@ def satisfies_minmax(v):

python_versions = [v for v in KNOWN_PYTHON_VERSIONS
if satisfies_minmax(v)]
print(f'KNOWN_PYTHON_VERSIONS={KNOWN_PYTHON_VERSIONS}')
print(f'python_versions={python_versions}')
self['supported_python_versions'] = python_versions

if self['ci_cpython_versions'] == 'auto':
Expand Down Expand Up @@ -590,11 +586,11 @@ def _build_template_registry(self):

# {'template': 0, 'overwrite': 1, 'fname': 'dev/make_strict_req.sh', 'perms': 'x'},

{'template': 0, 'overwrite': 1, 'fname': 'requirements.txt', 'dynamic': 'build_requirements'},
{'template': 0, 'overwrite': 1, 'fname': 'requirements/graphics.txt', 'tags': 'cv2'},
{'template': 0, 'overwrite': 1, 'fname': 'requirements/headless.txt', 'tags': 'cv2'},
{'template': 0, 'overwrite': 1, 'fname': 'requirements/gdal.txt', 'tags': 'gdal'},
{'template': 0, 'overwrite': 1, 'fname': 'requirements/gdal-strict.txt', 'tags': 'gdal'},
{'template': 0, 'overwrite': 1, 'fname': 'requirements.txt', 'dynamic': 'build_requirements_txt'},
{'template': 1, 'overwrite': 1, 'fname': 'requirements/graphics.txt', 'tags': 'cv2', 'dynamic': 'build_cv2_graphics_requirements_txt'},
{'template': 1, 'overwrite': 1, 'fname': 'requirements/headless.txt', 'tags': 'cv2', 'dynamic': 'build_cv2_headless_requirements_txt'},
{'template': 1, 'overwrite': 1, 'fname': 'requirements/gdal.txt', 'tags': 'gdal', 'dynamic': 'build_gdal_requirements_txt'},

{'template': 0, 'overwrite': 0, 'fname': 'requirements/optional.txt'},
{'template': 0, 'overwrite': 0, 'fname': 'requirements/runtime.txt'},
{'template': 0, 'overwrite': 0, 'fname': 'requirements/tests.txt'},
Expand Down Expand Up @@ -950,6 +946,8 @@ def _stage_file(self, info):
raise SkipFile('file was disabled')
try:
stage_fpath.write_text(text)
if 'x' in info.get('perms', ''):
stage_fpath.chmod('+x')
except Exception:
print(f'text={text}')
raise
Expand Down Expand Up @@ -1152,7 +1150,7 @@ def gather_tasks(self):
print('stats = {}'.format(ub.urepr(stats, nl=2)))
return stats, tasks

def build_requirements(self):
def build_requirements_txt(self):
# existing = (self.repodir / 'requirements').ls()
candidate_all_requirements = [
'requirements/runtime.txt',
Expand Down Expand Up @@ -1318,6 +1316,78 @@ def build_docs_requirements(self):
from xcookie.builders import docs
return docs.build_docs_requirements(self)

def _build_special_requirements(self, variant, version_defaults, header_lines):
"""
Example:
>>> from xcookie.main import * # NOQA
>>> dpath = ub.Path.appdir('xcookie/tests/test-stage').delete().ensuredir()
>>> kwargs = {
>>> 'repodir': dpath / 'testrepo',
>>> 'tags': ['gitlab', 'kitware', 'purepy', 'cv2'],
>>> 'rotate_secrets': False,
>>> 'is_new': False,
>>> 'min_python': '3.9',
>>> 'max_python': '3.12',
>>> 'interactive': False,
>>> }
>>> config = XCookieConfig.cli(cmdline=0, data=kwargs)
>>> print('config = {}'.format(ub.urepr(dict(config), nl=1)))
>>> self = TemplateApplier(config)
>>> print(chr(10) + 'headless.txt')
>>> print(self.build_cv2_headless_requirements_txt())
>>> print(chr(10) + 'gdal.txt')
>>> print(self.build_gdal_requirements_txt())
"""
req_lines = ['# Generated dynamically via: ~/code/xcookie/xcookie/main.py::TemplateApplier._build_special_requirements']
req_lines.extend(header_lines)
max_pyver = Version(self.config['max_python'] or '4.0')
min_pyver = Version(self.config['min_python'])

for row in version_defaults:
lt = row['pyver_lt']
# lt = min(row['pyver_lt'], max_pyver) # FIXME, exclusive vs inclusive
ge = max(row['pyver_ge'], min_pyver)
skip = row['pyver_ge'] > max_pyver
skip |= row['pyver_lt'] < min_pyver
# print(lt, ge, skip, min_pyver, max_pyver, row)
if not skip:
req_lines.append(f'{variant}{row["version"]} ; python_version < {lt} and python_version >= {ge}')
req_text = '\n'.join(req_lines)
return req_text

def _build_cv2_requirements(self, variant):
header_lines = [
f'# xdev availpkg {variant}',
'# --prefer-binary',
]
version_defaults = [
{'version': '>=4.5.4.58', 'pyver_ge': Version('3.10'), 'pyver_lt': Version('4.0')},
{'version': '>=3.4.15.55', 'pyver_ge': Version('3.6'), 'pyver_lt': Version('3.9')},
{'version': '>=3.4.2.16', 'pyver_ge': Version('2.7'), 'pyver_lt': Version('3.6')},
]
return self._build_special_requirements(variant, version_defaults, header_lines)

def build_cv2_headless_requirements_txt(self):
variant = 'opencv-python-headless'
return self._build_cv2_requirements(variant)

def build_cv2_graphics_requirements_txt(self):
variant = 'opencv-python'
return self._build_cv2_requirements(variant)

def build_gdal_requirements_txt(self):
# TODO: make more dynamic
variant = 'GDAL'
header_lines = [
'--find-links https://girder.github.io/large_image_wheels',
]
version_defaults = [
{'version': '>=3.7.2', 'pyver_ge': Version('3.12'), 'pyver_lt': Version('4.0')},
{'version': '>=3.5.2', 'pyver_ge': Version('3.11'), 'pyver_lt': Version('3.12')},
{'version': '>=3.4.1', 'pyver_ge': Version('3.6'), 'pyver_lt': Version('3.11')},
]
return self._build_special_requirements(variant, version_defaults, header_lines)

def build_run_doctests(self):
return ub.codeblock(
f'''
Expand Down
3 changes: 2 additions & 1 deletion xcookie/rc/setup.py.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,8 @@ def parse_requirements(fname='requirements.txt', versions=False):
if plat_deps is not None:
parts.append(';' + plat_deps)
item = ''.join(parts)
yield item
if item:
yield item

packages = list(gen_packages_items())
return packages
Expand Down

0 comments on commit f53d4c8

Please sign in to comment.