Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add blob copy capability #54

Open
wants to merge 19 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 52 additions & 10 deletions docker-registry-show.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,18 +18,27 @@
from __future__ import absolute_import

import argparse
from docker_registry_client import DockerRegistryClient
import json
import logging
import warnings

try:
from urllib.parse import urljoin
except ImportError:
from urlparse import urljoin


import requests

from docker_registry_client import DockerRegistryClient


class CLI(object):
def __init__(self):
self.parser = argparse.ArgumentParser()
excl_group = self.parser.add_mutually_exclusive_group()
excl_group.add_argument("-q", "--quiet", action="store_true")
excl_group.add_argument("-v", "--verbose", action="store_true")
verb_excl_group = self.parser.add_mutually_exclusive_group()
verb_excl_group.add_argument("-q", "--quiet", action="store_true")
verb_excl_group.add_argument("-v", "--verbose", action="store_true")

self.parser.add_argument('--verify-ssl', dest='verify_ssl',
action='store_true')
Expand All @@ -39,8 +48,30 @@ def __init__(self):
self.parser.add_argument('--username', metavar='USERNAME')
self.parser.add_argument('--password', metavar='PASSWORD')

self.parser.add_argument('--authorization-service', metavar='AUTH_SERVICE', type=str,
help='authorization service URL (including scheme) (for registry v2 only)')
auth_excl_group = self.parser.add_mutually_exclusive_group()
auth_excl_group.add_argument(
'--authorization-service-url', metavar='AUTH_SERVICE_URL', type=str,
help=(
'auth service host URL with with scheme and path '
'[e.g. http://foo.com/v2/token] (v2 API only)'
),
)
# DEPRECATED old form of the argument that assumes a path for the URL
auth_excl_group.add_argument(
'--authorization-service', metavar='AUTH_SERVICE', type=str,
help=(
'[DEPRECATED] auth service host URL with scheme, without path '
'[e.g. http://foo.com] (v2 API only)'
),
)

self.parser.add_argument(
'--authorization-service-name', metavar='AUTH_SERVICE_NAME', type=str,
help=(
'auth service URL "service" query parameter for custom auth service names '
'[e.g. container_registry, for GitLab auth] (v2 API only)'
),
)

self.parser.add_argument('registry', metavar='REGISTRY', nargs=1,
help='registry URL (including scheme)')
Expand All @@ -65,15 +96,26 @@ def run(self):
kwargs = {
'username': args.username,
'password': args.password,
'verify_ssl': args.verify_ssl,
'auth_service_name': args.authorization_service_name,
'auth_service_url_full': args.authorization_service_url
}

# Get the URL of the auth service from the command-line flags, accounting for the
# deprecated url flag; only use the deprecated one if the
if args.authorization_service:
warnings.warn(
'The --authorization-service flag is deprecated; '
'use --authorization-service-url instead',
DeprecationWarning
)
kwargs.setdefault('auth_service_url_full',
urljoin(args.authorization_service, 'v2/token'))

if args.api_version:
kwargs['api_version'] = args.api_version

client = DockerRegistryClient(args.registry[0],
auth_service_url=args.authorization_service,
verify_ssl=args.verify_ssl,
**kwargs)
client = DockerRegistryClient(args.registry[0], **kwargs)

if args.repository:
if args.ref:
Expand Down
37 changes: 23 additions & 14 deletions docker_registry_client/AuthorizationService.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
from urllib.parse import urlsplit
except ImportError:
from urlparse import urlsplit
# import urlparse

import requests
import logging

Expand All @@ -19,10 +19,10 @@ class AuthorizationService(object):
authenticate to the registry. Token has to be renew each time we change
"scope".
"""
def __init__(self, registry, url="", auth=None, verify=False,
def __init__(self, service_name, url="", auth=None, verify=False,
api_timeout=None):
# Registry ip:port
self.registry = urlsplit(registry).netloc
# Service name (often, but not always, just the registry host Registry ip:port)
self.service_name = service_name
# Service url, ip:port
self.url = url
# Authentication (user, password) or None. Used by request to do
Expand Down Expand Up @@ -54,15 +54,24 @@ def __init__(self, registry, url="", auth=None, verify=False,
self.token_required = False

def get_new_token(self):
rsp = requests.get("%s/v2/token?service=%s&scope=%s" %
(self.url, self.registry, self.desired_scope),
auth=self.auth, verify=self.verify,
timeout=self.api_timeout)
if not rsp.ok:
rsp = requests.get(
self.url,
params={
'service': self.service_name,
'scope': self.desired_scope,
},
auth=self.auth,
verify=self.verify,
timeout=self.api_timeout
)

if rsp.ok:
self.token = rsp.json()['token']

# We managed to get a new token, update the current scope to the one we
# wanted
self.scope = self.desired_scope
else:
logger.error("Can't get token for authentication")
self.token = ""

self.token = rsp.json()['token']
# We managed to get a new token, update the current scope to the one we
# wanted
self.scope = self.desired_scope
self.scope = None
15 changes: 9 additions & 6 deletions docker_registry_client/DockerRegistryClient.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,26 +5,29 @@


class DockerRegistryClient(object):
def __init__(self, host, verify_ssl=None, api_version=None, username=None,
password=None, auth_service_url="", api_timeout=None):
def __init__(self, host, verify_ssl=None, api_version=None, username=None, password=None,
auth_service_url="", auth_service_url_full="", auth_service_name=None,
api_timeout=None):
"""
Constructor

:param host: str, registry URL including scheme
:param verify_ssl: bool, whether to verify SSL certificate
:param api_version: int, API version to require
:param username: username to use for basic authentication when
connecting to the registry
:param username: username to use for basic authentication when connecting to the registry
:param password: password to use for basic authentication
:param auth_service_url: authorization service URL (including scheme,
for v2 only)
:param auth_service_url_full: authorization service URL with scheme and path (v2),
:param auth_service_url: DEPRECATED authorization service URL with scheme but no path (v2)
:param auth_service_name: service name to use with auth services; defaults to registry host
:param api_timeout: timeout for external request
"""

self._base_client = BaseClient(host, verify_ssl=verify_ssl,
api_version=api_version,
username=username, password=password,
auth_service_url=auth_service_url,
auth_service_url_full=auth_service_url_full,
auth_service_name=auth_service_name,
api_timeout=api_timeout)
self.api_version = self._base_client.version
self._repositories = {}
Expand Down
99 changes: 84 additions & 15 deletions docker_registry_client/_BaseClient.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,24 @@
import json
import logging
from requests import get, put, delete
import warnings

try:
from urllib.parse import urljoin, urlsplit
except ImportError:
from urlparse import urljoin, urlsplit

from requests import delete, get, head, post, put
from requests.exceptions import HTTPError
import json

from .AuthorizationService import AuthorizationService
from .manifest import sign as sign_manifest


# Module setup

# urllib3 throws some ssl warnings with older versions of python
# they're probably ok for the registry client to ignore
import warnings
warnings.filterwarnings("ignore")

warnings.filterwarnings("ignore", module="urllib3", append=True)

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -147,19 +156,51 @@ class BaseClientV2(CommonBaseClient):
LIST_TAGS = '/v2/{name}/tags/list'
MANIFEST = '/v2/{name}/manifests/{reference}'
BLOB = '/v2/{name}/blobs/{digest}'
BLOB_MOUNT = '/v2/{name}/blobs/uploads/?mount={digest}&from={origin}'
schema_1_signed = BASE_CONTENT_TYPE + '.v1+prettyjws'
schema_1 = BASE_CONTENT_TYPE + '.v1+json'
schema_2 = BASE_CONTENT_TYPE + '.v2+json'

def __init__(self, *args, **kwargs):
auth_service_url = kwargs.pop("auth_service_url", "")
host = args[0]

# Default to the main part of the repository hostname if the service name is missing
# or None (the default)
auth_service_name = kwargs.pop("auth_service_name", "") or urlsplit(host).netloc

# Get the URL of the auth service from the args, accounting for the deprecated url arg
auth_service_url = kwargs.pop("auth_service_url_full", "")
deprecated_auth_service_url_arg = kwargs.pop("auth_service_url", "")

if deprecated_auth_service_url_arg:
warnings.warn(
'The auth_service_url argument is deprecated; use auth_service_url_full instead',
DeprecationWarning,
)
if not auth_service_url:
auth_service_url = urljoin(deprecated_auth_service_url_arg, 'v2/token')

super(BaseClientV2, self).__init__(*args, **kwargs)

# If we are using token authentication with v2, we use the username
# and pw only for the authorization service and not for the registry
# itself.
#
# We must pop the auth kwarg so it does not get sent to requests,
# because override the authentication token if it sees the username/password
# provided
# See: http://docs.python-requests.org/en/master/user/quickstart/#custom-headers
if auth_service_url:
auth = self.method_kwargs.pop('auth', None)
else:
auth = self.method_kwargs.get('auth')

self._manifest_digests = {}
self.auth = AuthorizationService(
registry=self.host,
service_name=auth_service_name,
url=auth_service_url,
verify=self.method_kwargs.get('verify', False),
auth=self.method_kwargs.get('auth', None),
auth=auth,
api_timeout=self.method_kwargs.get('api_timeout')
)

Expand All @@ -168,7 +209,7 @@ def version(self):
return 2

def check_status(self):
self.auth.desired_scope = 'registry:catalog:*'
self.auth.desired_scope = ''
return self._http_call('/v2/', get)

def catalog(self):
Expand Down Expand Up @@ -196,11 +237,25 @@ def get_manifest(self, name, reference):
digest=self._manifest_digests[name, reference],
)

def check_manifest(self, name, reference):
self.auth.desired_scope = 'repository:%s:*' % name
response = self._http_response(
self.MANIFEST, head, name=name, reference=reference,
schema=self.schema_1_signed,
)
self._cache_manifest_digest(name, reference, response=response)
return response.ok

def put_manifest(self, name, reference, manifest):
self.auth.desired_scope = 'repository:%s:*' % name
content = {}
content.update(manifest._content)
content.update({'name': name, 'tag': reference})

content['name'] = name

# If reference is a tag, update it; otherwise, leave the tag as is
if not reference.startswith('sha256:'):
content['tag'] = reference

return self._http_call(
self.MANIFEST, put, data=sign_manifest(content),
Expand All @@ -213,6 +268,11 @@ def delete_manifest(self, name, digest):
return self._http_call(self.MANIFEST, delete,
name=name, reference=digest)

def copy_blob(self, origin, digest, destination):
self.auth.desired_scope = ['repository:%s:*' % repo for repo in (origin, destination)]
return self._http_call(self.BLOB_MOUNT, post,
name=destination, digest=digest, origin=origin)

def delete_blob(self, name, digest):
self.auth.desired_scope = 'repository:%s:*' % name
return self._http_call(self.BLOB, delete,
Expand Down Expand Up @@ -264,13 +324,20 @@ def _http_response(self, url, method, data=None, content_type=None,
response = method(self.host + path,
data=data, headers=header, **self.method_kwargs)
logger.debug("%s %s", response.status_code, response.reason)
response.raise_for_status()

try:
response.raise_for_status()
except HTTPError as e:
if e.response.content:
logger.error('Error Response: {}'.format(response.content))
raise

return response


def BaseClient(host, verify_ssl=None, api_version=None, username=None,
password=None, auth_service_url="", api_timeout=None):
def BaseClient(host, verify_ssl=None, api_version=None, username=None, password=None,
auth_service_url="", auth_service_url_full="", auth_service_name=None,
api_timeout=None):
if api_version == 1:
return BaseClientV1(
host, verify_ssl=verify_ssl, username=username, password=password,
Expand All @@ -279,14 +346,16 @@ def BaseClient(host, verify_ssl=None, api_version=None, username=None,
elif api_version == 2:
return BaseClientV2(
host, verify_ssl=verify_ssl, username=username, password=password,
auth_service_url=auth_service_url, api_timeout=api_timeout,
auth_service_url=auth_service_url, auth_service_url_full=auth_service_url_full,
auth_service_name=auth_service_name, api_timeout=api_timeout,
)
elif api_version is None:
# Try V2 first
logger.debug("checking for v2 API")
v2_client = BaseClientV2(
host, verify_ssl=verify_ssl, username=username, password=password,
auth_service_url=auth_service_url, api_timeout=api_timeout,
auth_service_url=auth_service_url, auth_service_url_full=auth_service_url_full,
auth_service_name=auth_service_name, api_timeout=api_timeout,
)
try:
v2_client.check_status()
Expand Down
15 changes: 15 additions & 0 deletions docker_registry_client/manifest.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
# Extracted from python-dxf (https://git.io/vM0EB) used under license (MIT).
import base64
import ecdsa
import hashlib
import jws
import json

Expand Down Expand Up @@ -89,3 +90,17 @@ def sign(manifest, key=None):
json.dumps(signatures) +
format_tail
)


def digest(manifest):
# For manifests, the digest is the manifest body without the signature content, also known as
# the JWS payload - https://docs.docker.com/registry/spec/api/#content-digests
m = assign({}, manifest)

try:
del m['signatures']
except KeyError:
pass

manifest_json = json.dumps(m, sort_keys=True)
return 'sha256:{}'.format(hashlib.sha256(manifest_json.encode('utf-8')).hexdigest())
1 change: 1 addition & 0 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
],
keywords='docker docker-registry REST',
packages=find_packages(),
scripts=['docker-registry-show.py'],
install_requires=[
'requests>=2.4.3, <3.0.0',
'ecdsa>=0.13.0, <0.14.0',
Expand Down
Loading