From 27be9e614bfd94c5d05660c2fc2246e02d32abff Mon Sep 17 00:00:00 2001 From: Joseph Crail Date: Mon, 3 Oct 2016 19:24:03 -0400 Subject: [PATCH] Rename package to azure-datalake-store (#86) --- .coveragerc | 4 +-- .travis.yml | 2 +- MANIFEST.in | 2 +- README.rst | 28 +++++++++---------- appveyor.yml | 2 +- azure/__init__.py | 1 + azure/datalake/__init__.py | 1 + {adlfs => azure/datalake/store}/__init__.py | 0 {adlfs => azure/datalake/store}/cli.py | 8 +++--- {adlfs => azure/datalake/store}/core.py | 0 {adlfs => azure/datalake/store}/exceptions.py | 0 {adlfs => azure/datalake/store}/extern.py | 0 {adlfs => azure/datalake/store}/lib.py | 0 .../datalake/store}/multithread.py | 4 +-- {adlfs => azure/datalake/store}/transfer.py | 4 +-- {adlfs => azure/datalake/store}/utils.py | 4 +-- docs/source/api.rst | 8 +++--- docs/source/conf.py | 18 ++++++------ docs/source/index.rst | 8 +++--- setup.py | 8 ++++-- tests/README.md | 6 ++-- tests/benchmarks.py | 8 +++--- tests/conftest.py | 2 +- tests/settings.py | 2 +- tests/test_cli.py | 2 +- tests/test_core.py | 4 +-- tests/test_lib.py | 5 ++-- tests/test_multithread.py | 8 +++--- tests/test_transfer.py | 4 +-- tests/testing.py | 4 +-- 30 files changed, 77 insertions(+), 70 deletions(-) create mode 100644 azure/__init__.py create mode 100644 azure/datalake/__init__.py rename {adlfs => azure/datalake/store}/__init__.py (100%) rename {adlfs => azure/datalake/store}/cli.py (98%) rename {adlfs => azure/datalake/store}/core.py (100%) rename {adlfs => azure/datalake/store}/exceptions.py (100%) rename {adlfs => azure/datalake/store}/extern.py (100%) rename {adlfs => azure/datalake/store}/lib.py (100%) rename {adlfs => azure/datalake/store}/multithread.py (99%) rename {adlfs => azure/datalake/store}/transfer.py (99%) rename {adlfs => azure/datalake/store}/utils.py (97%) diff --git a/.coveragerc b/.coveragerc index fa8fcd7..561dee7 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,9 +1,9 @@ [run] include = - adlfs/* + azure/datalake/store/* omit = - adlfs/tests/test* + azure/datalake/store/tests/test* [report] show_missing = True diff --git a/.travis.yml b/.travis.yml index baf81c0..1a4baca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,7 @@ install: - python setup.py develop script: - - py.test -x -vvv --doctest-modules --pyargs adlfs tests + - py.test -x -vvv --doctest-modules --pyargs azure.datalake.store tests notifications: email: false diff --git a/MANIFEST.in b/MANIFEST.in index ba6fdcf..ea1585d 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,4 @@ -recursive-include adlfs *.py +recursive-include azure/datalake/store *.py recursive-include docs *.rst include setup.py diff --git a/README.rst b/README.rst index 973e2e5..5051a30 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ -adlfs -==== +azure-datalake-store +==================== .. image:: https://travis-ci.org/Azure/azure-data-lake-store-python.svg?branch=dev :target: https://travis-ci.org/Azure/azure-data-lake-store-python @@ -7,7 +7,7 @@ adlfs NOTE: This management package is currently under active development and should not be consumed until the first version is published. -adlfs is a file-system management system in python for the +azure-datalake-store is a file-system management system in python for the Azure Data-Lake Store. To install: @@ -25,7 +25,7 @@ To play with the code, here is a starting point: .. code-block:: python - from adlfs import core, lib, multithread + from azure.datalake.store import core, lib, multithread token = lib.auth(tenant_id, username, password) adl = core.AzureDLFileSystem(store_name, token) @@ -56,16 +56,16 @@ To play with the code, here is a starting point: To interact with the API at a higher-level, you can use the provided -command-line interface in "adlfs/cli.py". You will need to set the appropriate -environment variables as described above to connect to the Azure Data Lake -Store. +command-line interface in "azure/datalake/store/cli.py". You will need to set +the appropriate environment variables as described above to connect to the +Azure Data Lake Store. -To start the CLI in interactive mode, run "python adlfs/cli.py" and then type -"help" to see all available commands (similiar to Unix utilities): +To start the CLI in interactive mode, run "python azure/datalake/store/cli.py" +and then type "help" to see all available commands (similiar to Unix utilities): .. code-block:: bash - > python adlfs/cli.py + > python azure/datalake/store/cli.py azure> help Documented commands (type help ): @@ -84,7 +84,7 @@ familiar with the Unix/Linux "ls" command, the columns represent 1) permissions, .. code-block:: bash - > python adlfs/cli.py + > python azure/datalake/store/cli.py azure> ls -l drwxrwx--- 0123abcd 0123abcd 0 Aug 02 12:44 azure1 -rwxrwx--- 0123abcd 0123abcd 1048576 Jul 25 18:33 abc.csv @@ -104,7 +104,7 @@ named after the remote file minus the directory path. .. code-block:: bash - > python adlfs/cli.py + > python azure/datalake/store/cli.py azure> ls -l drwxrwx--- 0123abcd 0123abcd 0 Aug 02 12:44 azure1 -rwxrwx--- 0123abcd 0123abcd 1048576 Jul 25 18:33 abc.csv @@ -125,7 +125,7 @@ For example, listing the entries in the home directory: .. code-block:: bash - > python adlfs/cli.py ls -l + > python azure/datalake/store/cli.py ls -l drwxrwx--- 0123abcd 0123abcd 0 Aug 02 12:44 azure1 -rwxrwx--- 0123abcd 0123abcd 1048576 Jul 25 18:33 abc.csv -r-xr-xr-x 0123abcd 0123abcd 36 Jul 22 18:32 xyz.csv @@ -137,7 +137,7 @@ Also, downloading a remote file: .. code-block:: bash - > python adlfs/cli.py get xyz.csv + > python azure/datalake/store/cli.py get xyz.csv 2016-08-04 18:57:48,603 - ADLFS - DEBUG - Creating empty file xyz.csv 2016-08-04 18:57:48,604 - ADLFS - DEBUG - Fetch: xyz.csv, 0-36 2016-08-04 18:57:49,726 - ADLFS - DEBUG - Downloaded to xyz.csv, byte offset 0 diff --git a/appveyor.yml b/appveyor.yml index f7c210a..9c5e505 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -15,4 +15,4 @@ install: build: off test_script: - - "%PYTHON%\\Scripts\\pytest.exe -x -vvv --doctest-modules --pyargs adlfs tests" + - "%PYTHON%\\Scripts\\pytest.exe -x -vvv --doctest-modules --pyargs azure.datalake.store tests" diff --git a/azure/__init__.py b/azure/__init__.py new file mode 100644 index 0000000..de40ea7 --- /dev/null +++ b/azure/__init__.py @@ -0,0 +1 @@ +__import__('pkg_resources').declare_namespace(__name__) diff --git a/azure/datalake/__init__.py b/azure/datalake/__init__.py new file mode 100644 index 0000000..de40ea7 --- /dev/null +++ b/azure/datalake/__init__.py @@ -0,0 +1 @@ +__import__('pkg_resources').declare_namespace(__name__) diff --git a/adlfs/__init__.py b/azure/datalake/store/__init__.py similarity index 100% rename from adlfs/__init__.py rename to azure/datalake/store/__init__.py diff --git a/adlfs/cli.py b/azure/datalake/store/cli.py similarity index 98% rename from adlfs/cli.py rename to azure/datalake/store/cli.py index 6537471..8b50972 100644 --- a/adlfs/cli.py +++ b/azure/datalake/store/cli.py @@ -24,10 +24,10 @@ import stat import sys -from adlfs.core import AzureDLFileSystem -from adlfs.lib import auth -from adlfs.multithread import ADLDownloader, ADLUploader -from adlfs.utils import write_stdout +from azure.datalake.store.core import AzureDLFileSystem +from azure.datalake.store.lib import auth +from azure.datalake.store.multithread import ADLDownloader, ADLUploader +from azure.datalake.store.utils import write_stdout class AzureDataLakeFSCommand(cmd.Cmd, object): diff --git a/adlfs/core.py b/azure/datalake/store/core.py similarity index 100% rename from adlfs/core.py rename to azure/datalake/store/core.py diff --git a/adlfs/exceptions.py b/azure/datalake/store/exceptions.py similarity index 100% rename from adlfs/exceptions.py rename to azure/datalake/store/exceptions.py diff --git a/adlfs/extern.py b/azure/datalake/store/extern.py similarity index 100% rename from adlfs/extern.py rename to azure/datalake/store/extern.py diff --git a/adlfs/lib.py b/azure/datalake/store/lib.py similarity index 100% rename from adlfs/lib.py rename to azure/datalake/store/lib.py diff --git a/adlfs/multithread.py b/azure/datalake/store/multithread.py similarity index 99% rename from adlfs/multithread.py rename to azure/datalake/store/multithread.py index d8d5539..b547dfc 100644 --- a/adlfs/multithread.py +++ b/azure/datalake/store/multithread.py @@ -69,7 +69,7 @@ class ADLDownloader(object): See Also -------- - adlfs.transfer.ADLTransferClient + azure.datalake.store.transfer.ADLTransferClient """ def __init__(self, adlfs, rpath, lpath, nthreads=None, chunksize=2**28, buffersize=2**22, blocksize=2**22, client=None, run=True, @@ -237,7 +237,7 @@ class ADLUploader(object): See Also -------- - adlfs.transfer.ADLTransferClient + azure.datalake.store.transfer.ADLTransferClient """ def __init__(self, adlfs, rpath, lpath, nthreads=None, chunksize=2**28, buffersize=2**22, blocksize=2**22, client=None, run=True, diff --git a/adlfs/transfer.py b/azure/datalake/store/transfer.py similarity index 99% rename from adlfs/transfer.py rename to azure/datalake/store/transfer.py index f790c4a..28eed28 100644 --- a/adlfs/transfer.py +++ b/azure/datalake/store/transfer.py @@ -216,8 +216,8 @@ class ADLTransferClient(object): See Also -------- - adlfs.multithread.ADLDownloader - adlfs.multithread.ADLUploader + azure.datalake.store.multithread.ADLDownloader + azure.datalake.store.multithread.ADLUploader """ def __init__(self, adlfs, name, transfer, merge=None, nthreads=None, diff --git a/adlfs/utils.py b/azure/datalake/store/utils.py similarity index 97% rename from adlfs/utils.py rename to azure/datalake/store/utils.py index 44cc796..796fe20 100644 --- a/adlfs/utils.py +++ b/azure/datalake/store/utils.py @@ -18,9 +18,9 @@ WIN = platform.platform() == 'Windows' if WIN: - datadir = os.path.join(os.environ['APPDATA'], 'adlfs') + datadir = os.path.join(os.environ['APPDATA'], 'azure-datalake-store') else: - datadir = os.sep.join([os.path.expanduser("~"), '.config', 'adlfs']) + datadir = os.sep.join([os.path.expanduser("~"), '.config', 'azure-datalake-store']) try: os.makedirs(datadir) diff --git a/docs/source/api.rst b/docs/source/api.rst index 3ab936a..a8dd3a6 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -1,7 +1,7 @@ API === -.. currentmodule:: adlfs.core +.. currentmodule:: azure.datalake.store.core .. autosummary:: AzureDLFileSystem @@ -31,18 +31,18 @@ API AzureDLFile.tell AzureDLFile.write -.. currentmodule:: adlfs.multithread +.. currentmodule:: azure.datalake.store.multithread .. autosummary:: ADLUploader ADLDownloader -.. currentmodule:: adlfs.core +.. currentmodule:: azure.datalake.store.core .. autoclass:: AzureDLFileSystem :members: -.. currentmodule:: adlfs.multithread +.. currentmodule:: azure.datalake.store.multithread .. autoclass:: AzureDLFile :members: diff --git a/docs/source/conf.py b/docs/source/conf.py index 517490b..68ff56d 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # -# adlfs documentation build configuration file, created by +# azure-datalake-store documentation build configuration file, created by # sphinx-quickstart on Mon Mar 21 15:20:01 2016. # # This file is execfile()d with the current directory set to its @@ -53,7 +53,7 @@ master_doc = 'index' # General information about the project. -project = 'adlfs' +project = 'azure-datalake-store' copyright = 'TBD' author = 'TBD' @@ -62,8 +62,8 @@ # built documents. # # The short X.Y version. -import adlfs -version = adlfs.__version__ +import azure.datalake.store +version = azure.datalake.store.__version__ # The full version, including alpha/beta/rc tags. release = version @@ -213,7 +213,7 @@ #html_search_scorer = 'scorer.js' # Output file base name for HTML help builder. -htmlhelp_basename = 'adlfsdoc' +htmlhelp_basename = 'azure-datalake-store-doc' # -- Options for LaTeX output --------------------------------------------- @@ -235,7 +235,7 @@ # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'adlfs.tex', 'adlfs Documentation', + (master_doc, 'azure-datalake-store.tex', 'azure-datalake-store Documentation', 'TBA', 'manual'), ] @@ -265,7 +265,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - (master_doc, 'adlfs', 'adlfs Documentation', + (master_doc, 'azure-datalake-store', 'azure-datalake-store Documentation', [author], 1) ] @@ -279,8 +279,8 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'adlfs', 'adlfs Documentation', - author, 'adlfs', 'One line description of project.', + (master_doc, 'azure-datalake-store', 'azure-datalake-store Documentation', + author, 'azure-datalake-store', 'One line description of project.', 'Miscellaneous'), ] diff --git a/docs/source/index.rst b/docs/source/index.rst index 1fe4483..a0b74e8 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,5 +1,5 @@ -adlfs -==== +azure-datalake-store +==================== A pure-python interface to the Azure Data-lake Storage system, providing pythonic file-system and file objects, seamless transition between Windows and @@ -12,7 +12,7 @@ Installation ------------ Using ``pip``:: - pip install adlfs + pip install azure-datalake-store Manually (bleeding edge): @@ -90,7 +90,7 @@ forward slashes or backslashes. import pathlib # only >= Python 3.4 from pathlib2 import pathlib # only <= Python 3.3 - from adlfs.core import AzureDLPath + from azure.datalake.store.core import AzureDLPath # possible remote paths to use on API p1 = '\\foo\\bar' diff --git a/setup.py b/setup.py index d34e2af..c59eb99 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ import os from setuptools import setup -setup(name='adlfs', +setup(name='azure-datalake-store', version='0.0.1', description='Convenient Filesystem interface to Azure Data-lake Store', url='https://github.com/Azure/azure-data-lake-store-python', @@ -11,7 +11,11 @@ maintainer_email='', license='', keywords='azure', - packages=['adlfs'], + packages=[ + 'azure', + 'azure.datalake', + 'azure.datalake.store' + ], install_requires=[open('requirements.txt').read().strip().split('\n')], long_description=(open('README.rst').read() if os.path.exists('README.rst') else ''), diff --git a/tests/README.md b/tests/README.md index 46545ce..50baa60 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,6 +1,6 @@ To run the test suite: - py.test -x -vvv --doctest-modules --pyargs adlfs tests + py.test -x -vvv --doctest-modules --pyargs azure-datalake-store tests This test suite uses [VCR.py](https://github.com/kevin1024/vcrpy) to record the responses from Azure. Borrowing from VCR's @@ -11,11 +11,11 @@ invoking the test suite (defaults to `none`). To record responses for a new test without updating previous recordings: - RECORD_MODE=once py.test -x -vvv --doctest-modules --pyargs adlfs tests + RECORD_MODE=once py.test -x -vvv --doctest-modules --pyargs azure-datalake-store tests To record responses for all tests even if previous recordings exist: - RECORD_MODE=all py.test -x -vvv --doctest-modules --pyargs adlfs tests + RECORD_MODE=all py.test -x -vvv --doctest-modules --pyargs azure-datalake-store tests When recording new responses, you will need valid Azure credentials. The following environment variables should be defined: diff --git a/tests/benchmarks.py b/tests/benchmarks.py index 6760f61..a6a539e 100644 --- a/tests/benchmarks.py +++ b/tests/benchmarks.py @@ -6,9 +6,9 @@ import sys import time -from adlfs import core, multithread -from adlfs.transfer import ADLTransferClient -from adlfs.utils import WIN +from azure.datalake.store import core, multithread +from azure.datalake.store.transfer import ADLTransferClient +from azure.datalake.store.utils import WIN from tests.testing import md5sum @@ -158,7 +158,7 @@ def bench_download_50_1gb(adl, lpath, rpath, config): # Log only Azure messages, ignoring 3rd-party libraries logging.basicConfig( format='%(asctime)s %(name)-17s %(levelname)-8s %(message)s') - logger = logging.getLogger('adlfs') + logger = logging.getLogger('azure.datalake.store') logger.setLevel(logging.INFO) # Required setup until outstanding issues are resolved diff --git a/tests/conftest.py b/tests/conftest.py index ff70e75..581ae10 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,7 +8,7 @@ import pytest -from adlfs import AzureDLFileSystem +from azure.datalake.store import AzureDLFileSystem from tests import settings from tests.testing import working_dir diff --git a/tests/settings.py b/tests/settings.py index 978014c..985d8fb 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -10,7 +10,7 @@ import os import time -from adlfs.lib import auth +from azure.datalake.store.lib import auth from tests import fake_settings diff --git a/tests/test_cli.py b/tests/test_cli.py index 7bb7430..d8c089c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -13,7 +13,7 @@ import pytest -from adlfs.cli import AzureDataLakeFSCommand +from azure.datalake.store.cli import AzureDataLakeFSCommand from tests.testing import azure, my_vcr, working_dir diff --git a/tests/test_core.py b/tests/test_core.py index a986219..cf8ae0b 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -628,7 +628,7 @@ def test_chown(azure): @pytest.mark.skipif(sys.platform != 'win32', reason="requires windows") def test_backslash(): - from adlfs.core import AzureDLPath + from azure.datalake.store.core import AzureDLPath posix_abspath = '/foo/bar' posix_relpath = 'foo/bar' @@ -648,7 +648,7 @@ def test_backslash(): def test_forward_slash(): - from adlfs.core import AzureDLPath + from azure.datalake.store.core import AzureDLPath posix_abspath = '/foo/bar' posix_relpath = 'foo/bar' diff --git a/tests/test_lib.py b/tests/test_lib.py index 91ab30a..3a9cafa 100644 --- a/tests/test_lib.py +++ b/tests/test_lib.py @@ -9,8 +9,9 @@ import pytest import time -from adlfs.lib import (refresh_token, DatalakeRESTInterface, - DatalakeRESTException, ManagementRESTInterface) +from azure.datalake.store.lib import ( + refresh_token, DatalakeRESTInterface, DatalakeRESTException, + ManagementRESTInterface) from tests import settings from tests.testing import my_vcr diff --git a/tests/test_multithread.py b/tests/test_multithread.py index a49b7c4..ce2209e 100644 --- a/tests/test_multithread.py +++ b/tests/test_multithread.py @@ -12,10 +12,10 @@ import shutil import tempfile -from adlfs.core import AzureDLPath -from adlfs.multithread import ADLDownloader, ADLUploader +from azure.datalake.store.core import AzureDLPath +from azure.datalake.store.multithread import ADLDownloader, ADLUploader from tests.testing import azure, azure_teardown, md5sum, my_vcr, posix, working_dir -from adlfs.transfer import ADLTransferClient +from azure.datalake.store.transfer import ADLTransferClient test_dir = working_dir() @@ -199,7 +199,7 @@ def test_upload_one(local_files, azure): bigfile, littlefile, a, b, c = local_files # transfer client w/ deterministic temporary directory - from adlfs.multithread import put_chunk + from azure.datalake.store.multithread import put_chunk client = ADLTransferClient(azure, 'foo', transfer=put_chunk, unique_temporary=False) diff --git a/tests/test_transfer.py b/tests/test_transfer.py index aaa849f..e00f9ad 100644 --- a/tests/test_transfer.py +++ b/tests/test_transfer.py @@ -10,8 +10,8 @@ import pytest import time -from adlfs.core import AzureDLPath -from adlfs.transfer import ADLTransferClient +from azure.datalake.store.core import AzureDLPath +from azure.datalake.store.transfer import ADLTransferClient from tests.testing import azure, posix diff --git a/tests/testing.py b/tests/testing.py index 45ac877..e6b4c19 100644 --- a/tests/testing.py +++ b/tests/testing.py @@ -16,7 +16,7 @@ import pytest import vcr -from adlfs.core import AzureDLPath +from azure.datalake.store.core import AzureDLPath from tests import fake_settings, settings @@ -72,7 +72,7 @@ def working_dir(): @pytest.yield_fixture() def azure(): - from adlfs import AzureDLFileSystem + from azure.datalake.store import AzureDLFileSystem fs = AzureDLFileSystem(token=settings.TOKEN, store_name=settings.STORE_NAME) # Clear filesystem cache to ensure we capture all requests from a test