diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..37d811b --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,5 @@ +# Change Log + +## 0.1.0 + +- Initial Release! diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..34b8cc5 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,100 @@ +# So You Want to Contribute to Scrapbook! +We welcome all contributions to Scrapbook both large and small. We encourage you to join our community. + +## Our Community Values + +We are an open and friendly community. Everybody is welcome. + +We encourage friendly discussions and respect for all. There are no exceptions. + +All contributions are equally important. Documentation, answering questions, and fixing bugs are equally as valuable as adding new features. + +Please read our entire code of conduct [here](https://github.com/nteract/nteract/blob/master/CODE_OF_CONDUCT.md). Also, check out the for the [Python](https://github.com/nteract/nteract/blob/master/CODE_OF_CONDUCT.md) code of conduct. + +## Setting up Your Development Environment +Following these instructions should give you an efficient path to opening your first pull-request. + +### Cloning the Scrapbook Repository +Fork the repository to your local Github account. Clone this repository to your local development machine. +``` +git clone https://github.com//scrapbook +cd scrapbook +``` + +### Install an Editable Version +We prefer to use [conda](https://conda.io/docs/user-guide/tasks/manage-environments.html) to manage the development environment. +``` +conda create -n dev +. activate env +``` +or [virtualenv](https://packaging.python.org/guides/installing-using-pip-and-virtualenv/) if you prefer. +``` +python3 -m virtualenv dev +source dev/bin/activate +``` + +Install Scrapbook using: +``` +pip install -e .[dev] +``` + +_Note: When you are finished you can use `source deactivate` to go back to your base environment._ + +### Running Tests Locally + +We need to install the development package before we can run the tests. If anything is confusing below, always resort to the relevant documentation. +``` +pytest --pyargs scrapbook +``` +The `pyargs` option allows `pytest` to interpret arguments as python package names. An advantage is that `pytest` will run in any directory, and this approach follows the `pytest` [best practices](https://docs.pytest.org/en/latest/goodpractices.html#tests-as-part-of-application-code). + +Now there should be a working and editable installation of Scrapbook to start making your own contributions. + +## So You're Ready to Pull Request + +The general workflow for this will be: +1. Run local tests +2. Pushed changes to your forked repository +3. Open pull request to main repository + +### Run Tests Locally + +``` +pytest --pyargs scrapbook +``` + +Run check manifest to ensure all files are accounted for in the repository. +``` +check-manifest +``` +This commands read the `MANIFEST.in` file and explicitly specify the files to include in the source distribution. You can read more about how this works [here](https://docs.python.org/3/distutils/sourcedist.html). + +### Push Changes to Forked Repo + +Your commits should be pushed to the forked repository. To verify this type ```git remote -v``` and ensure the remotes point to your GitHub. Don't work on the master branch! + +1. Commit changes to local repository: + ``` + git checkout -b my-feature + git add + git commit + ``` +2. Push changes to your remote repository: + ``` + git push -u origin my-feature + ``` + +### Create Pull Request + +Follow [these](https://help.github.com/articles/creating-a-pull-request-from-a-fork/) instrucutions to create a pull request from a forked repository. If you are submitting a bug-fix for a specific issue make sure to reference the issue in the pull request. + +There are good references to the [Git documentation](https://git-scm.com/doc) and [Git workflows](https://docs.scipy.org/doc/numpy/dev/gitwash/development_workflow.html) for more information if any of this is unfamiliar. + +_Note: You might want to set a reference to the main repository to fetch/merge from there instead of your forked repository. You can do that using:_ +``` +git remote add upstream https://github.com/nteract/scrapbook +``` + +It's possible you will have conflicts between your repository and master. Here, `master` is meant to be synchronized with the ```upstream``` repository. GitHub has some good [documentation](https://help.github.com/articles/resolving-a-merge-conflict-using-the-command-line/) on merging pull requests from the command line. + +Happy hacking on Scrapbook! diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..7973b16 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,27 @@ +recursive-include papermill *.py +recursive-include papermill *.ipynb +recursive-include papermill *.json +recursive-include papermill *.yaml +recursive-include papermill *.keep +recursive-include papermill *.txt + +include setup.py +include setup.cfg +include requirements.txt +include requirements-dev.txt +include pytest.ini +include LICENSE +include MANIFEST.in +include *.md +include *.toml + +include .coveragerc + +# Documentation +graft docs +# exclude build files +prune docs/_build +# exclude sample notebooks for binder +prune binder/ +# Scripts +graft scripts diff --git a/RELEASING.md b/RELEASING.md new file mode 100644 index 0000000..6f06dac --- /dev/null +++ b/RELEASING.md @@ -0,0 +1,21 @@ +# Releasing + +## Pre-requisites + +- First check that the CHANGELOG is up to date for the next release version +- Ensure `bumpversion`, `wheel>=0.31.0`, `setuptools>=38.6.0`, and `twine>=1.11.0` (or long readme will be malformed) + +## Push to github + +``` +bumpversion patch setup.py +git push && git push --tags +``` + +## Push to PyPi + +``` +rm -rf dist/* +python setup.py sdist bdist_wheel +twine upload dist/* +``` diff --git a/binder/ResultsDemo.ipynb b/binder/ResultsDemo.ipynb new file mode 100644 index 0000000..2a2c27f --- /dev/null +++ b/binder/ResultsDemo.ipynb @@ -0,0 +1,331 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import scrapbook as sb" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "book = sb.read_notebooks('../scrapbook/tests/notebooks/collection')" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/markdown": [ + "### result1" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "#### output" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "'Hello World!'" + ] + }, + "metadata": { + "papermill": { + "name": "output" + } + }, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "#### one_only" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "'Just here!'" + ] + }, + "metadata": { + "scrapbook": { + "name": "one_only" + } + }, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "
" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "### result2" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "#### output" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "'Hello World 2!'" + ] + }, + "metadata": { + "papermill": { + "name": "output" + } + }, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "#### two_only" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "'Just here!'" + ] + }, + "metadata": { + "scrapbook": { + "name": "two_only" + } + }, + "output_type": "display_data" + } + ], + "source": [ + "book.display()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'result1': OrderedDict([('one', 1),\n", + " ('number', 1),\n", + " ('list', [1, 2, 3]),\n", + " ('dict', {'a': 1, 'b': 2})]),\n", + " 'result2': OrderedDict([('two', 2),\n", + " ('number', 2),\n", + " ('list', [4, 5, 6]),\n", + " ('dict', {'a': 3, 'b': 4})])}" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "book.scraps" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "OrderedDict([('one', 1),\n", + " ('number', 2),\n", + " ('list', [4, 5, 6]),\n", + " ('dict', {'a': 3, 'b': 4}),\n", + " ('two', 2)])" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "book.combined_scraps" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'result1': OrderedDict([('output',\n", + " {'data': {'text/plain': \"'Hello World!'\"},\n", + " 'metadata': {'papermill': {'name': 'output'}},\n", + " 'output_type': 'display_data'}),\n", + " ('one_only',\n", + " {'data': {'text/plain': \"'Just here!'\"},\n", + " 'metadata': {'scrapbook': {'name': 'one_only'}},\n", + " 'output_type': 'display_data'})]),\n", + " 'result2': OrderedDict([('output',\n", + " {'data': {'text/plain': \"'Hello World 2!'\"},\n", + " 'metadata': {'papermill': {'name': 'output'}},\n", + " 'output_type': 'display_data'}),\n", + " ('two_only',\n", + " {'data': {'text/plain': \"'Just here!'\"},\n", + " 'metadata': {'scrapbook': {'name': 'two_only'}},\n", + " 'output_type': 'display_data'})])}" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "book.frames" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "OrderedDict([('output',\n", + " {'data': {'text/plain': \"'Hello World 2!'\"},\n", + " 'metadata': {'papermill': {'name': 'output'}},\n", + " 'output_type': 'display_data'}),\n", + " ('one_only',\n", + " {'data': {'text/plain': \"'Just here!'\"},\n", + " 'metadata': {'scrapbook': {'name': 'one_only'}},\n", + " 'output_type': 'display_data'}),\n", + " ('two_only',\n", + " {'data': {'text/plain': \"'Just here!'\"},\n", + " 'metadata': {'scrapbook': {'name': 'two_only'}},\n", + " 'output_type': 'display_data'})])" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "book.combined_frames" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "application/scrapbook.scrap+json": { + "collection_keys": [ + "one", + "number", + "list", + "dict", + "two" + ] + } + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "sb.glue('collection_keys', list(book.combined_scraps.keys()))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/binder/requirements.txt b/binder/requirements.txt new file mode 100644 index 0000000..9f689cf --- /dev/null +++ b/binder/requirements.txt @@ -0,0 +1 @@ +-e git://github.com/nteract/scrapbook.git#egg=master diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..0996ab8 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,4 @@ +[pytest] +env = + AWS_SECRET_ACCESS_KEY=foobar_secret + AWS_ACCESS_KEY_ID=foobar_key diff --git a/requirements-dev.txt b/requirements-dev.txt index e69de29..8b13789 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -0,0 +1 @@ + diff --git a/requirements.txt b/requirements.txt index e69de29..c716265 100644 --- a/requirements.txt +++ b/requirements.txt @@ -0,0 +1,6 @@ +pandas +six +papermill +future +futures ; python_version < "3.0" +ipython >= 5.0 diff --git a/scrapbook/__init__.py b/scrapbook/__init__.py index e69de29..613ae91 100644 --- a/scrapbook/__init__.py +++ b/scrapbook/__init__.py @@ -0,0 +1,5 @@ +from __future__ import absolute_import, division, print_function + +from .version import version as __version__ + +from .api import frame, glue, read_notebook, read_notebooks diff --git a/scrapbook/_version.py b/scrapbook/_version.py deleted file mode 100644 index 4e91b3b..0000000 --- a/scrapbook/_version.py +++ /dev/null @@ -1,544 +0,0 @@ - -# This file helps to compute a version number in source trees obtained from -# git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (built by setup.py sdist) and build -# directories (produced by setup.py build) will contain a much shorter file -# that just contains the computed version number. - -# This file is released into the public domain. Generated by -# versioneer-0.18 (https://github.com/warner/python-versioneer) - -"""Git implementation of _version.py.""" - -import errno -import os -import re -import subprocess -import sys - - -def get_keywords(): - """Get the keywords needed to look up the version information.""" - # these strings will be replaced by git during git-archive. - # setup.py/versioneer.py will grep for the variable names, so they must - # each be defined on a line of their own. _version.py will just call - # get_keywords(). - git_refnames = "$Format:%d$" - git_full = "$Format:%H$" - git_date = "$Format:%ci$" - keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} - return keywords - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_config(): - """Create, populate and return the VersioneerConfig() object.""" - # these strings are filled in when 'setup.py versioneer' creates - # _version.py - cfg = VersioneerConfig() - cfg.VCS = "git" - cfg.style = "pep440" - cfg.tag_prefix = "" - cfg.parentdir_prefix = "scrapbook-" - cfg.versionfile_source = "scrapbook/_version.py" - cfg.verbose = False - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -LONG_VERSION_PY = {} -HANDLERS = {} - - -def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" - - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - - return decorate - - -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=None): - """Call the given command(s).""" - assert isinstance(commands, list) - p = None - for c in commands: - try: - dispcmd = str([c] + args) - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen( - [c] + args, - cwd=cwd, - env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr else None), - ) - break - except EnvironmentError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %s" % dispcmd) - print(e) - return None, None - else: - if verbose: - print("unable to find command, tried %s" % (commands,)) - return None, None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: - if verbose: - print("unable to run %s (error)" % dispcmd) - print("stdout was %s" % stdout) - return None, p.returncode - return stdout, p.returncode - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes both - the project name and a version string. We will also support searching up - two directory levels for an appropriately named parent directory - """ - rootdirs = [] - - for i in range(3): - dirname = os.path.basename(root) - if dirname.startswith(parentdir_prefix): - return { - "version": dirname[len(parentdir_prefix) :], - "full-revisionid": None, - "dirty": False, - "error": None, - "date": None, - } - else: - rootdirs.append(root) - root = os.path.dirname(root) # up a level - - if verbose: - print( - "Tried directories %s but none started with prefix %s" - % (str(rootdirs), parentdir_prefix) - ) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - f = open(versionfile_abs, "r") - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - f.close() - except EnvironmentError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") - date = keywords.get("date") - if date is not None: - # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant - # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 - # -like" string, which we must then edit to make compliant), because - # it's been around since git-1.5.3, and it's too difficult to - # discover which version we're using, or to work around using an - # older one. - date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)]) - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r'\d', r)]) - if verbose: - print("discarding '%s', no digits" % ",".join(refs - tags)) - if verbose: - print("likely tags: %s" % ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix) :] - if verbose: - print("picking %s" % r) - return { - "version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": None, - "date": date, - } - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return { - "version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": "no suitable tags", - "date": None, - } - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - - out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True) - if rc != 0: - if verbose: - print("Directory %s not under git control" % root) - raise NotThisMethod("'git rev-parse --git-dir' returned error") - - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command( - GITS, - ["describe", "--tags", "--dirty", "--always", "--long", "--match", "%s*" % tag_prefix], - cwd=root, - ) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[: git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) - if not mo: - # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = "unable to parse git-describe output: '%s'" % describe_out - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%s' doesn't start with prefix '%s'" - print(fmt % (full_tag, tag_prefix)) - pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % (full_tag, tag_prefix) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix) :] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], cwd=root) - pieces["distance"] = int(count_out) # total number of commits - - # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[0].strip() - pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - - return pieces - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post.devDISTANCE - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += ".post.dev%d" % pieces["distance"] - else: - # exception #1 - rendered = "0.post.dev%d" % pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%s" % pieces["short"] - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%s" % pieces["short"] - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Eexceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return { - "version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None, - } - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%s'" % style) - - return { - "version": rendered, - "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], - "error": None, - "date": pieces.get("date"), - } - - -def get_versions(): - """Get version information or return default if unable to do so.""" - # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have - # __file__, we can work backwards from there to the root. Some - # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which - # case we can only use expanded keywords. - - cfg = get_config() - verbose = cfg.verbose - - try: - return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, verbose) - except NotThisMethod: - pass - - try: - root = os.path.realpath(__file__) - # versionfile_source is the relative path from the top of the source - # tree (where the .git directory might live) to this file. Invert - # this to find the root from __file__. - for i in cfg.versionfile_source.split('/'): - root = os.path.dirname(root) - except NameError: - return { - "version": "0+unknown", - "full-revisionid": None, - "dirty": None, - "error": "unable to find root of source tree", - "date": None, - } - - try: - pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) - return render(pieces, cfg.style) - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - except NotThisMethod: - pass - - return { - "version": "0+unknown", - "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", - "date": None, - } diff --git a/scrapbook/api.py b/scrapbook/api.py new file mode 100644 index 0000000..6182077 --- /dev/null +++ b/scrapbook/api.py @@ -0,0 +1,132 @@ +# -*- coding: utf-8 -*- +""" +api.py + +Provides the base API calls for scrapbook +""" +from __future__ import unicode_literals +import os + +import IPython + +from six import string_types +from IPython.display import display as ip_display +# We lean on papermill's readers to connect to remote stores +from papermill.iorw import load_notebook_node, list_notebook_files + +from .models import Notebook, Scrapbook, GLUE_OUTPUT_PREFIX +from .translators import registry as translator_registry + + +def glue(name, scrap, storage=None): + """ + Record a value in the output notebook when a cell is executed. + + The recorded value can be retrieved during later inspection of the + output notebook. + + Example + ------- + `record` provides a handy way for data to be stored with a notebook to + be used later:: + + sb.glue("hello", "world") + sb.glue("number", 123) + sb.glue("some_list", [1, 3, 5]) + sb.glue("some_dict", {"a": 1, "b": 2}) + sb.glue("non_json", df, 'arrow') + + scrapbook can be used later to recover recorded values by + reading the output notebook + + nb = sb.read_notebook('notebook.ipynb') + nb.scraps + + Parameters + ---------- + name: str + Name of the value to record. + scrap: any + The value to record. + storage: str (optional) + The data protocol name to respect in persisting data + """ + + # TODO: Implement the cool stuff. Remote storage indicators?!? Maybe remote media type?!? + # TODO: Make this more modular + # TODO: Use translators to determine best storage type + # ... + if not storage: + if isinstance(scrap, string_types): + storage = 'unicode' + elif isinstance(scrap, (list, dict)): + storage = 'json' + else: + # This may be more complex in the future + storage = 'json' + data = { + GLUE_OUTPUT_PREFIX + storage: { + name: translator_registry.translate_data(storage, scrap) + } + } + + # IPython.display.display takes a tuple of objects as first parameter + # `http://ipython.readthedocs.io/en/stable/api/generated/IPython.display.html#IPython.display.display` + ip_display(data, raw=True) + + +def frame(name, obj): + """ + Display an object with the reference `name` in a retrievable manner. + + Parameters + ---------- + name : str + Name of the output. + obj : object + An object that can be displayed in the notebook. + + """ + data, metadata = IPython.core.formatters.format_display_data(obj) + metadata['scrapbook'] = dict(name=name) + ip_display(data, metadata=metadata, raw=True) + + +def read_notebook(path): + """ + Returns a Notebook object loaded from the location specified at `path`. + + Parameters + ---------- + path : str + Path to a notebook `.ipynb` file. + + Returns + ------- + notebook : object + A Notebook object. + + """ + return Notebook(path) + +def read_notebooks(path): + """ + Returns a Scrapbook including the notebooks read from the + directory specified by `path`. + + Parameters + ---------- + path : str + Path to directory containing notebook `.ipynb` files. + + Returns + ------- + scrapbook : object + A `Scrapbook` object. + + """ + scrapbook = Scrapbook() + for notebook_path in list_notebook_files(path): + fn = os.path.splitext(os.path.basename(notebook_path))[0] + scrapbook[fn] = read_notebook(notebook_path) + return scrapbook diff --git a/scrapbook/exceptions.py b/scrapbook/exceptions.py new file mode 100644 index 0000000..383ef96 --- /dev/null +++ b/scrapbook/exceptions.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- + +class ScrapbookException(ValueError): + """Raised when an exception is encountered when operating on a notebook.""" diff --git a/scrapbook/log.py b/scrapbook/log.py new file mode 100644 index 0000000..0230c7f --- /dev/null +++ b/scrapbook/log.py @@ -0,0 +1,3 @@ +import logging + +logger = logging.getLogger('scrapbook') diff --git a/scrapbook/models.py b/scrapbook/models.py new file mode 100644 index 0000000..7e490a3 --- /dev/null +++ b/scrapbook/models.py @@ -0,0 +1,312 @@ +# -*- coding: utf-8 -*- +""" +notebook.py + +Provides the Notebook wrapper objects for scrapbook +""" +from __future__ import unicode_literals +import os +import operator +import itertools +import collections +import pandas as pd + +from six import string_types +from IPython.display import display as ip_display, Markdown +# We lean on papermill's readers to connect to remote stores +from papermill.iorw import load_notebook_node + +from .translators import registry as translator_registry +from .exceptions import ScrapbookException + + +GLUE_OUTPUT_PREFIX = 'application/scrapbook.scrap+' +RECORD_OUTPUT_PREFIX = 'application/papermill.record+' +DATA_OUTPUT_PREFIXES = [ + GLUE_OUTPUT_PREFIX, + # Backwards compatibility + RECORD_OUTPUT_PREFIX +] + + +def merge_dicts(dicts): + iterdicts = iter(dicts) + outcome = next(iterdicts).copy() + boom = False + for d in iterdicts: + outcome.update(d) + return outcome + + +class Notebook(object): + """ + Representation of a notebook. + + Parameters + ---------- + node : `nbformat.NotebookNode`, str + a notebook object, or a path to a notebook object + """ + + def __init__(self, node_or_path, translators=None): + if isinstance(node_or_path, string_types): + if not node_or_path.endswith(".ipynb"): + raise ValueError("Requires an '.ipynb' file extension. Provided path: '{}'".format(node_or_path)) + self.path = node_or_path + self.node = load_notebook_node(node_or_path) + else: + self.path = '' + self.node = node_or_path + self.translators = translators or translator_registry + + # Memoized traits + self._scraps = None + self._frames = None + + @property + def filename(self): + """str: filename found a the specified path""" + return os.path.basename(self.path) + + @property + def directory(self): + """str: directory name at the specified path""" + return os.path.dirname(self.path) + + @property + def parameters(self): + """dict: parameters stored in the notebook metadata""" + return self.node.metadata.get('papermill', {}).get('parameters', {}) + + def _fetch_scraps(self): + """Returns a dictionary of the data recorded in a notebook.""" + scraps = collections.OrderedDict() + for cell in self.node.cells: + for output in cell.get('outputs', []): + for sig, payload in output.get('data', {}).items(): + for prefix in DATA_OUTPUT_PREFIXES: + if sig.startswith(prefix): + data_type = sig.split(prefix, 1)[1] + scraps.update(self.translators.load_data(data_type, payload)) + return scraps + + @property + def scraps(self): + """dict: a dictionary of data found in the notebook""" + if self._scraps is None: + self._scraps = self._fetch_scraps() + return self._scraps + + @property + def cell_timing(self): + """list: a list of cell execution timings in cell order""" + return [ + # TODO: Other timing conventions? + cell.metadata.get('papermill', {}).get('duration', 0.0) + if cell.get("execution_count") else None + for cell in self.node.cells + ] + + @property + def execution_counts(self): + """list: a list of cell execution counts in cell order""" + return [ + cell.get("execution_count") for cell in self.node.cells + ] + + @property + def papermill_metrics(self): + """pandas dataframe: dataframe of cell execution counts and times""" + df = pd.DataFrame(columns=['filename', 'cell', 'value', 'type']) + + for i, cell in enumerate(self.node.cells): + execution_count = cell.get("execution_count") + if not execution_count: + continue + name = "Out [{}]".format(str(execution_count)) + value = cell.metadata.get('papermill', {}).get('duration', 0.0) + df.loc[i] = self.filename, name, value, "time (s)" + return df + + @property + def parameter_dataframe(self): + """pandas dataframe: dataframe of notebook parameters""" + # Meant for backwards compatibility to papermill's dataframe method + return pd.DataFrame( + [[name, self.parameters[name], 'parameter', self.filename] + for name in sorted(self.parameters.keys())], + columns=['name', 'value', 'type', 'filename']) + + @property + def scrap_dataframe(self): + """pandas dataframe: dataframe of cell scraps""" + # Meant for backwards compatibility to papermill's dataframe method + return pd.DataFrame( + [[name, self.scraps[name], 'record', self.filename] + for name in sorted(self.scraps.keys())], + columns=['name', 'value', 'type', 'filename']) + + @property + def papermill_dataframe(self): + """pandas dataframe: dataframe of notebook parameters and cell scraps""" + # Meant for backwards compatibility to papermill's dataframe method + return self.parameter_dataframe.append(self.scrap_dataframe, ignore_index=True) + + def _fetch_frames(self): + outputs = collections.OrderedDict() + for cell in self.node.cells: + for output in cell.get('outputs', []): + if 'scrapbook' in output.get('metadata', {}): + output_name = output.metadata.scrapbook.get('name') + if output_name: + outputs[output_name] = output + # Backwards compatibility + if 'papermill' in output.get('metadata', {}): + output_name = output.metadata.papermill.get('name') + if output_name: + outputs[output_name] = output + return outputs + + @property + def frames(self): + """dict: a dictionary of the notebook display outputs.""" + if self._frames is None: + self._frames = self._fetch_frames() + return self._frames + + def reframe(self, name, raise_error=True): + """ + Display output from a named source of the notebook. + + Parameters + ---------- + name : str + name of framed object + raise_error : bool + indicator for if the reframe should print a message or error on missing frame + + """ + if name not in self.frames: + if raise_error: + raise ScrapbookException("Frame '{}' is not available in this notebook.".format(name)) + else: + ip_display("No frame available for {}".format(name)) + else: + output = self.frames[name] + ip_display(output.data, metadata=output.metadata, raw=True) + + +class Scrapbook(collections.MutableMapping): + """ + Represents a collection of notebooks as a dictionary of notebooks. + """ + + def __init__(self): + self._notebooks = {} + + def __setitem__(self, key, value): + # If notebook is a path str then load the notebook. + if isinstance(value, string_types): + value = Notebook(value) + self._notebooks.__setitem__(key, value) + + def __getitem__(self, key): + return self._notebooks.__getitem__(key) + + def __delitem__(self, key): + return self._notebooks.__delitem__(key) + + def __iter__(self): + return self._notebooks.__iter__() + + def __len__(self): + return self._notebooks.__len__() + + @property + def papermill_dataframe(self): + """list: a list of dataframes from a collection of notebooks""" + # Backwards compatible dataframe interface + df_list = [] + for key in sorted(self._notebooks): + nb = self._notebooks[key] + df = nb.papermill_dataframe + df['key'] = key + df_list.append(df) + return pd.concat(df_list).reset_index(drop=True) + + @property + def papermill_metrics(self): + """list: a list of metrics from a collection of notebooks""" + df_list = [] + for key in sorted(self._notebooks): + nb = self._notebooks[key] + df = nb.papermill_metrics + df['key'] = key + df_list.append(df) + return pd.concat(df_list).reset_index(drop=True) + + @property + def sorted_notebooks(self): + """list: a list of the notebooks in key order.""" + return map(operator.itemgetter(1), + sorted(self._notebooks.items(), + key=operator.itemgetter(0))) + + @property + def scraps(self): + """dict: a dictionary of the notebook scraps by key.""" + return { key: nb.scraps for key, nb in self._notebooks.items() } + + @property + def combined_scraps(self): + """dict: a dictionary of the merged notebook scraps.""" + return merge_dicts(nb.scraps for nb in self.sorted_notebooks) + + @property + def frames(self): + """dict: a dictionary of the notebook display outputs by key.""" + return { key: nb.frames for key, nb in self._notebooks.items() } + + @property + def combined_frames(self): + """dict: a dictionary of the merged notebook display outputs.""" + return merge_dicts(nb.frames for nb in self.sorted_notebooks) + + def display(self, frames=None, keys=None, header=True, raise_error=False): + """ + Display frames as markdown structed outputs. + + Parameters + ---------- + frames : str or iterable[str] (optional) + the frames to display as outputs + keys : str or iterable[str] (optional) + notebook keys to use in framing the scrapbook displays + header : bool (default: True) + indicator for if the frames should have headers + raise_error : bool (default: False) + flag for if errors should be raised on missing output_names + """ + if isinstance(frames, string_types): + frames = [frames] + + if keys is None: + keys = self._notebooks.keys() + elif isinstance(keys, string_types): + keys = [keys] + + for i, k in enumerate(keys): + if header: + if i > 0: + ip_display(Markdown("
")) # tag between outputs + ip_display(Markdown("### {}".format(k))) + + if frames is None: + names = self[k].frames.keys() + else: + names = frames + + for name in names: + if header: + ip_display(Markdown("#### {}".format(name))) + self[k].reframe(name, raise_error=raise_error) diff --git a/scrapbook/tests/__init__.py b/scrapbook/tests/__init__.py new file mode 100644 index 0000000..0538243 --- /dev/null +++ b/scrapbook/tests/__init__.py @@ -0,0 +1,10 @@ +import os + +def get_fixture_path(*args): + return os.path.join(os.path.dirname(os.path.abspath(__file__)), 'fixtures', *args) + +def get_notebook_path(*args): + return os.path.join(os.path.dirname(os.path.abspath(__file__)), 'notebooks', *args) + +def get_notebook_dir(*args): + return os.path.dirname(get_notebook_path(*args)) diff --git a/scrapbook/tests/fixtures/tiny.png b/scrapbook/tests/fixtures/tiny.png new file mode 100644 index 0000000..3463f3c Binary files /dev/null and b/scrapbook/tests/fixtures/tiny.png differ diff --git a/scrapbook/tests/notebooks/collection/result1.ipynb b/scrapbook/tests/notebooks/collection/result1.ipynb new file mode 100644 index 0000000..fd3ee93 --- /dev/null +++ b/scrapbook/tests/notebooks/collection/result1.ipynb @@ -0,0 +1,143 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "foo = 1\n", + "bar = \"hello\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "papermill": {"duration": 0.123}, + "tags": [] + }, + "outputs": [ + { + "data": { + "application/scrapbook.scrap+json": { + "one": 1 + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/papermill.record+json": { + "number": 1 + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/papermill.record+json": { + "list": [ + 1, + 2, + 3 + ] + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/papermill.record+json": { + "dict": { + "a": 1, + "b": 2 + } + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "'Hello World!'" + ] + }, + "metadata": { + "papermill": { + "name": "output" + } + }, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "'Just here!'" + ] + }, + "metadata": { + "scrapbook": { + "name": "one_only" + } + }, + "output_type": "display_data" + } + ], + "source": [ + "import scrapbook as sb\n", + "\n", + "sb.glue(\"one\", 1)\n", + "sb.glue(\"number\", 1)\n", + "sb.glue(\"list\", [1,2,3])\n", + "sb.glue(\"dict\", dict(a=1, b=2))\n", + "\n", + "sb.frame(\"output\", \"Hello World!\")", + "sb.frame(\"one_only\", \"Just here!\")" + ] + } + ], + "metadata": { + "celltoolbar": "Tags", + "hide_input": false, + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2.0 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.12" + }, + "papermill": { + "environment_variables": {}, + "metrics": { + "duration": 2.250469923019409 + }, + "parameters": { + "bar": "hello", + "foo": 1 + }, + "version": "0.4+2.ge10f94c.dirty" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/scrapbook/tests/notebooks/collection/result2.ipynb b/scrapbook/tests/notebooks/collection/result2.ipynb new file mode 100644 index 0000000..e45ae51 --- /dev/null +++ b/scrapbook/tests/notebooks/collection/result2.ipynb @@ -0,0 +1,143 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "foo = 2\n", + "bar = \"world\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "papermill": {"duration": 0.456}, + "tags": [] + }, + "outputs": [ + { + "data": { + "application/scrapbook.scrap+json": { + "two": 2 + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/papermill.record+json": { + "number": 2 + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/papermill.record+json": { + "list": [ + 4, + 5, + 6 + ] + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/papermill.record+json": { + "dict": { + "a": 3, + "b": 4 + } + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "'Hello World 2!'" + ] + }, + "metadata": { + "papermill": { + "name": "output" + } + }, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "'Just here!'" + ] + }, + "metadata": { + "scrapbook": { + "name": "two_only" + } + }, + "output_type": "display_data" + } + ], + "source": [ + "import scrapbook as sb\n", + "\n", + "sb.glue(\"two\", 2)\n", + "sb.glue(\"number\", 2)\n", + "sb.glue(\"list\", [4,5,6])\n", + "sb.glue(\"dict\", dict(a=3, b=4))\n", + "\n", + "sb.frame(\"output\", \"Hello World 2!\")", + "sb.frame(\"two_only\", \"Just here!\")" + ] + } + ], + "metadata": { + "celltoolbar": "Tags", + "hide_input": false, + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2.0 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.12" + }, + "papermill": { + "environment_variables": {}, + "metrics": { + "duration": 2.2486860752105713 + }, + "parameters": { + "bar": "world", + "foo": 2 + }, + "version": "0.4+2.ge10f94c.dirty" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/scrapbook/tests/notebooks/result_no_exec.ipynb b/scrapbook/tests/notebooks/result_no_exec.ipynb new file mode 100644 index 0000000..aff2405 --- /dev/null +++ b/scrapbook/tests/notebooks/result_no_exec.ipynb @@ -0,0 +1,53 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "foo = 1\n", + "bar = \"hello\"\n" + ] + } + ], + "metadata": { + "celltoolbar": "Tags", + "hide_input": false, + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2.0 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.12" + }, + "papermill": { + "environment_variables": {}, + "metrics": { + "duration": 2.250469923019409 + }, + "parameters": { + "bar": "hello", + "foo": 1 + }, + "version": "0.4+2.ge10f94c.dirty" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/scrapbook/tests/test_api.py b/scrapbook/tests/test_api.py new file mode 100644 index 0000000..b782c92 --- /dev/null +++ b/scrapbook/tests/test_api.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +import mock +import pytest +import collections + +from IPython.display import Image + +from . import get_fixture_path +from ..models import GLUE_OUTPUT_PREFIX +from ..api import glue, frame + +@pytest.mark.parametrize( + "name,scrap,storage,data", + [ + ( + 'foobarbaz', + {"foo":"bar","baz":1}, + None, + { + GLUE_OUTPUT_PREFIX + 'json': { + 'foobarbaz': {"foo":"bar","baz":1} + } + } + ), + ( + 'foobarbaz', + '{"foo":"bar","baz":1}', + None, + { + GLUE_OUTPUT_PREFIX + 'unicode': { + 'foobarbaz': '{"foo":"bar","baz":1}' + } + } + ), + ( + 'foobarbaz', + '{"foo":"bar","baz":1}', + 'json', + { + GLUE_OUTPUT_PREFIX + 'json': { + 'foobarbaz': {"foo":"bar","baz":1} + } + } + ), + ( + 'foobarbaz', + # Pick something we don't match normally + collections.OrderedDict({"foo":"bar","baz":1}), + 'json', + { + GLUE_OUTPUT_PREFIX + 'json': { + 'foobarbaz': {"foo":"bar","baz":1} + } + } + ), + ], +) +@mock.patch('scrapbook.api.ip_display') +def test_glue(mock_display, name, scrap, storage, data): + glue(name, scrap, storage) + mock_display.assert_called_once_with(data, raw=True) + +@pytest.mark.parametrize( + "name,obj,data,metadata", + [ + ( + 'foobarbaz', + 'foo,bar,baz', + {'text/plain': "'foo,bar,baz'"}, + {'scrapbook': {'name': 'foobarbaz'}}, + ), + ( + 'tinypng', + Image(filename=get_fixture_path('tiny.png')), + { + 'image/png': 'iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAIAAAD91JpzAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAB3RJTUUH4gwRBREo2qqE0wAAAB1pVFh0Q29tbWVudAAAAAAAQ3JlYXRlZCB3aXRoIEdJTVBkLmUHAAAAFklEQVQI12P8//8/AwMDEwMDAwMDAwAkBgMBvR7jugAAAABJRU5ErkJggg==\n', + 'text/plain': '' + }, + {'scrapbook': {'name': 'tinypng'}}, + ), + ], +) +@mock.patch('scrapbook.api.ip_display') +def test_frame(mock_display, name, obj, data, metadata): + frame(name, obj) + mock_display.assert_called_once_with(data, metadata=metadata, raw=True) diff --git a/scrapbook/tests/test_notebooks.py b/scrapbook/tests/test_notebooks.py new file mode 100644 index 0000000..27da3c5 --- /dev/null +++ b/scrapbook/tests/test_notebooks.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +import mock +import pytest +import collections + +import pandas as pd + +from pandas.util.testing import assert_frame_equal +from nbformat.v4 import new_notebook, new_code_cell, new_markdown_cell, new_output + +from . import get_notebook_path, get_notebook_dir +from .. import read_notebook +from ..models import Notebook +from ..exceptions import ScrapbookException + + +@pytest.fixture +def notebook_result(): + path = get_notebook_path('collection/result1.ipynb') + return read_notebook(path) + +def test_bad_path(): + with pytest.raises(FileNotFoundError): + Notebook('not/a/valid/path.ipynb') + +def test_bad_ext(): + with pytest.raises(ValueError): + Notebook('not/a/valid/extension.py') + +def test_filename(notebook_result): + assert notebook_result.filename == 'result1.ipynb' + +def test_directory(notebook_result): + assert notebook_result.directory == get_notebook_dir('collection/result1.ipynb') + +def test_parameters(notebook_result): + assert notebook_result.parameters == dict(foo=1, bar="hello") + +def test_scraps(notebook_result): + assert notebook_result.scraps == { + 'dict': {u'a': 1, u'b': 2}, + 'list': [1, 2, 3], + 'number': 1, + 'one': 1 + } + +def test_frames(notebook_result): + assert notebook_result.frames == { + 'output': { + 'data': {'text/plain': "'Hello World!'"}, + 'metadata': {'papermill': {'name': 'output'}}, + 'output_type': 'display_data' + }, + 'one_only': { + 'data': {'text/plain': "'Just here!'"}, + 'metadata': {'scrapbook': {'name': 'one_only'}}, + 'output_type': 'display_data' + } + } + +@mock.patch('scrapbook.models.ip_display') +def test_reframe(mock_display, notebook_result): + notebook_result.reframe('output') + mock_display.assert_called_once_with( + {'text/plain': "'Hello World!'"}, + # We don't re-translate the metadata from older messages + metadata={'papermill': {'name': 'output'}}, + raw=True) + +def test_missing_reframe(notebook_result): + with pytest.raises(ScrapbookException): + notebook_result.reframe('foo') + +@mock.patch('scrapbook.models.ip_display') +def test_missing_reframe_no_error(mock_display, notebook_result): + notebook_result.reframe('foo', raise_error=False) + mock_display.assert_called_once_with('No frame available for foo') + +@pytest.fixture +def no_exec_result(): + path = get_notebook_path('result_no_exec.ipynb') + return read_notebook(path) + +def test_cell_timing(notebook_result): + assert notebook_result.cell_timing == [0.0, 0.123] + +def test_malformed_cell_timing(no_exec_result): + assert no_exec_result.cell_timing == [None] + +def test_execution_counts(notebook_result): + assert notebook_result.execution_counts == [1, 2] + +def test_malformed_execution_counts(no_exec_result): + assert no_exec_result.execution_counts == [None] + +def test_papermill_metrics(notebook_result): + expected_df = pd.DataFrame( + [ + ('result1.ipynb', 'Out [1]', 0.000, 'time (s)'), + ('result1.ipynb', 'Out [2]', 0.123, 'time (s)'), + ], + columns=['filename', 'cell', 'value', 'type'], + ) + assert_frame_equal(notebook_result.papermill_metrics, expected_df) + +def test_malformed_execution_metrics(no_exec_result): + expected_df = pd.DataFrame( + [], + columns=['filename', 'cell', 'value', 'type'], + ) + assert_frame_equal(no_exec_result.papermill_metrics, expected_df) + +def test_papermill_dataframe(notebook_result): + expected_df = pd.DataFrame( + [ + ('bar', 'hello', 'parameter', 'result1.ipynb'), + ('foo', 1, 'parameter', 'result1.ipynb'), + ('dict', {u'a': 1, u'b': 2}, 'record', 'result1.ipynb'), + ('list', [1, 2, 3], 'record', 'result1.ipynb'), + ('number', 1, 'record', 'result1.ipynb'), + ('one', 1, 'record', 'result1.ipynb'), + ], + columns=['name', 'value', 'type', 'filename'], + ) + assert_frame_equal(notebook_result.papermill_dataframe, expected_df) + +def test_no_cells(): + nb = Notebook(new_notebook(cells=[])) + assert nb.scraps == collections.OrderedDict() + assert nb.frames == collections.OrderedDict() + +def test_no_outputs(): + nb = Notebook(new_notebook(cells=[new_code_cell('test', outputs=[])])) + assert nb.scraps == collections.OrderedDict() + assert nb.frames == collections.OrderedDict() + +def test_empty_metadata(): + output = new_output(output_type='display_data', data={}, metadata={}) + raw_nb = new_notebook(cells=[new_code_cell('test', outputs=[output])]) + nb = Notebook(raw_nb) + assert nb.scraps == collections.OrderedDict() + assert nb.frames == collections.OrderedDict() + +def test_metadata_but_empty_content(): + output = new_output(output_type='display_data', metadata={'scrapbook': {}}) + raw_nb = new_notebook(cells=[new_code_cell('test', outputs=[output])]) + nb = Notebook(raw_nb) + assert nb.scraps == collections.OrderedDict() + assert nb.frames == collections.OrderedDict() + +def test_markdown(): + nb = Notebook(new_notebook(cells=[new_markdown_cell('this is a test.')])) + assert nb.scraps == collections.OrderedDict() + assert nb.frames == collections.OrderedDict() diff --git a/scrapbook/tests/test_scrapbooks.py b/scrapbook/tests/test_scrapbooks.py new file mode 100644 index 0000000..120a4d4 --- /dev/null +++ b/scrapbook/tests/test_scrapbooks.py @@ -0,0 +1,264 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +import mock +import pytest + +import pandas as pd + +from IPython.display import Markdown +from pandas.util.testing import assert_frame_equal + +from . import get_notebook_path, get_notebook_dir +from .. import read_notebooks +from ..exceptions import ScrapbookException + +@pytest.fixture +def notebook_collection(): + path = get_notebook_path('collection') + return read_notebooks(path) + +def test_assign_from_path(notebook_collection): + notebook_collection['result_no_exec.ipynb'] = get_notebook_path('result_no_exec.ipynb') + +def test_scraps(notebook_collection): + assert notebook_collection.scraps == { + 'result1': { + 'dict': {u'a': 1, u'b': 2}, + 'list': [1, 2, 3], + 'number': 1, + 'one': 1 + }, + 'result2': { + 'dict': {u'a': 3, u'b': 4}, + 'list': [4, 5, 6], + 'number': 2, + 'two': 2 + } + } + +def test_combined_scraps(notebook_collection): + assert notebook_collection.combined_scraps == { + 'dict': {u'a': 3, u'b': 4}, + 'list': [4, 5, 6], + 'number': 2, + 'one': 1, + 'two': 2 + } + +def test_frames(notebook_collection): + assert notebook_collection.frames == { + 'result1': { + 'output': { + 'data': {'text/plain': "'Hello World!'"}, + 'metadata': {'papermill': {'name': 'output'}}, + 'output_type': 'display_data' + }, + 'one_only': { + 'data': {'text/plain': "'Just here!'"}, + 'metadata': {'scrapbook': {'name': 'one_only'}}, + 'output_type': 'display_data' + } + }, + 'result2': { + 'output': { + 'data': {'text/plain': "'Hello World 2!'"}, + 'metadata': {'papermill': {'name': 'output'}}, + 'output_type': 'display_data' + }, + 'two_only': { + 'data': {'text/plain': "'Just here!'"}, + 'metadata': {'scrapbook': {'name': 'two_only'}}, + 'output_type': 'display_data' + } + }, + } + +def test_combined_frames(notebook_collection): + assert notebook_collection.combined_frames == { + 'output': { + 'data': {'text/plain': "'Hello World 2!'"}, + 'metadata': {'papermill': {'name': 'output'}}, + 'output_type': 'display_data' + }, + 'one_only': { + 'data': {'text/plain': "'Just here!'"}, + 'metadata': {'scrapbook': {'name': 'one_only'}}, + 'output_type': 'display_data' + }, + 'two_only': { + 'data': {'text/plain': "'Just here!'"}, + 'metadata': {'scrapbook': {'name': 'two_only'}}, + 'output_type': 'display_data' + } + } + +def test_papermill_metrics(notebook_collection): + expected_df = pd.DataFrame( + [ + ('result1.ipynb', 'Out [1]', 0.0, 'time (s)', 'result1'), + ('result1.ipynb', 'Out [2]', 0.123, 'time (s)', 'result1'), + ('result2.ipynb', 'Out [1]', 0.0, 'time (s)', 'result2'), + ('result2.ipynb', 'Out [2]', 0.456, 'time (s)', 'result2'), + ], + columns=['filename', 'cell', 'value', 'type', 'key'], + ) + assert_frame_equal(notebook_collection.papermill_metrics, expected_df) + +def test_papermill_dataframe(notebook_collection): + expected_df = pd.DataFrame( + [ + ('bar', 'hello', 'parameter', 'result1.ipynb', 'result1'), + ('foo', 1, 'parameter', 'result1.ipynb', 'result1'), + ('dict', {u'a': 1, u'b': 2}, 'record', 'result1.ipynb', 'result1'), + ('list', [1, 2, 3], 'record', 'result1.ipynb', 'result1'), + ('number', 1, 'record', 'result1.ipynb', 'result1'), + ('one', 1, 'record', 'result1.ipynb', 'result1'), + ('bar', 'world', 'parameter', 'result2.ipynb', 'result2'), + ('foo', 2, 'parameter', 'result2.ipynb', 'result2'), + ('dict', {u'a': 3, u'b': 4}, 'record', 'result2.ipynb', 'result2'), + ('list', [4, 5, 6], 'record', 'result2.ipynb', 'result2'), + ('number', 2, 'record', 'result2.ipynb', 'result2'), + ('two', 2, 'record', 'result2.ipynb', 'result2'), + ], + columns=['name', 'value', 'type', 'filename', 'key'], + ) + assert_frame_equal(notebook_collection.papermill_dataframe, expected_df) + +class AnyMarkdownWith(Markdown): + def __eq__(self, other): + try: + return self.data == other.data + except AttributeError: + return False + +@mock.patch('scrapbook.models.ip_display') +def test_display(mock_display, notebook_collection): + notebook_collection.display() + mock_display.assert_has_calls([ + mock.call(AnyMarkdownWith("### result1")), + mock.call(AnyMarkdownWith("#### output")), + mock.call( + {'text/plain': "'Hello World!'"}, + # We don't re-translate the metadata from older messages + metadata={'papermill': {'name': 'output'}}, + raw=True + ), + mock.call(AnyMarkdownWith("#### one_only")), + mock.call( + {'text/plain': "'Just here!'"}, + metadata={'scrapbook': {'name': 'one_only'}}, + raw=True + ), + mock.call(AnyMarkdownWith("
")), + mock.call(AnyMarkdownWith("### result2")), + mock.call(AnyMarkdownWith("#### output")), + mock.call( + {'text/plain': "'Hello World 2!'"}, + # We don't re-translate the metadata from older messages + metadata={'papermill': {'name': 'output'}}, + raw=True + ), + mock.call(AnyMarkdownWith("#### two_only")), + mock.call( + {'text/plain': "'Just here!'"}, + metadata={'scrapbook': {'name': 'two_only'}}, + raw=True + ), + ]) + +@mock.patch('scrapbook.models.ip_display') +def test_display_no_header(mock_display, notebook_collection): + notebook_collection.display(header=None) + mock_display.assert_has_calls([ + mock.call( + {'text/plain': "'Hello World!'"}, + # We don't re-translate the metadata from older messages + metadata={'papermill': {'name': 'output'}}, + raw=True + ), + mock.call( + {'text/plain': "'Just here!'"}, + metadata={'scrapbook': {'name': 'one_only'}}, + raw=True + ), + mock.call( + {'text/plain': "'Hello World 2!'"}, + # We don't re-translate the metadata from older messages + metadata={'papermill': {'name': 'output'}}, + raw=True + ), + mock.call( + {'text/plain': "'Just here!'"}, + metadata={'scrapbook': {'name': 'two_only'}}, + raw=True + ), + ]) + +@pytest.mark.parametrize( + "keys", + [ + ('result2',), + (['result2'],), + ], +) +@mock.patch('scrapbook.models.ip_display') +def test_display_specific_notebook(mock_display, keys, notebook_collection): + for key in keys: + notebook_collection.display(keys=key) + mock_display.assert_has_calls([ + mock.call(AnyMarkdownWith("### result2")), + mock.call(AnyMarkdownWith("#### output")), + mock.call( + {'text/plain': "'Hello World 2!'"}, + # We don't re-translate the metadata from older messages + metadata={'papermill': {'name': 'output'}}, + raw=True + ), + mock.call(AnyMarkdownWith("#### two_only")), + mock.call( + {'text/plain': "'Just here!'"}, + metadata={'scrapbook': {'name': 'two_only'}}, + raw=True + ), + ]) + +@pytest.mark.parametrize( + "frames", + [ + ('one_only',), + (['one_only'],), + ], +) +@mock.patch('scrapbook.models.ip_display') +def test_display_specific_frame(mock_display, frames, notebook_collection): + for frame in frames: + notebook_collection.display(frames=frame, header=False) + mock_display.assert_has_calls([ + mock.call( + {'text/plain': "'Just here!'"}, + metadata={'scrapbook': {'name': 'one_only'}}, + raw=True + ), + ]) + +@mock.patch('scrapbook.models.ip_display') +def test_display_frame_key_mismatches(mock_display, notebook_collection): + notebook_collection.display(frames='one_only') + mock_display.assert_has_calls([ + mock.call(AnyMarkdownWith("### result1")), + mock.call(AnyMarkdownWith("#### one_only")), + mock.call( + {'text/plain': "'Just here!'"}, + metadata={'scrapbook': {'name': 'one_only'}}, + raw=True + ), + mock.call(AnyMarkdownWith("
")), + mock.call(AnyMarkdownWith("### result2")), + mock.call(AnyMarkdownWith("#### one_only")), + mock.call('No frame available for one_only'), + ]) + +def test_display_missing_frame_error(notebook_collection): + with pytest.raises(ScrapbookException): + notebook_collection.display(frames='one_only', raise_error=True) + diff --git a/scrapbook/tests/test_translators.py b/scrapbook/tests/test_translators.py new file mode 100644 index 0000000..a0c66f5 --- /dev/null +++ b/scrapbook/tests/test_translators.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import pytest +import json + +from ..translators import (DataTranslatorRegistry, + JsonTranslator, + UnicodeTranslator) +from ..exceptions import ScrapbookException + +@pytest.mark.parametrize( + "test_input,expected", + [ + ({"foo":"bar","baz":1}, {"foo":"bar","baz":1}), + ('{"foo":"bar","baz":1}', {"foo":"bar","baz":1}), + (["foo","bar",1,2,3], ["foo","bar",1,2,3]), + ('["foo","bar",1,2,3]', ["foo","bar",1,2,3]), + (u'["😍"]', [u"😍"]), + ], +) +def test_json_load(test_input, expected): + assert JsonTranslator().load(test_input) == expected + +@pytest.mark.parametrize( + "test_input", + [ + (""), + ('{"inavlid","json"}'), + (u"😍"), + ], +) +def test_json_load_failures(test_input): + with pytest.raises(json.decoder.JSONDecodeError): + JsonTranslator().load(test_input) + +@pytest.mark.parametrize( + "test_input,expected", + [ + ({"foo":"bar","baz":1}, {"foo":"bar","baz":1}), + ('{"foo":"bar","baz":1}', {"foo":"bar","baz":1}), + (["foo","bar",1,2,3], ["foo","bar",1,2,3]), + ('["foo","bar",1,2,3]', ["foo","bar",1,2,3]), + (u'["😍"]', [u"😍"]), + ], +) +def test_json_translate(test_input, expected): + assert JsonTranslator().translate(test_input) == expected + +@pytest.mark.parametrize( + "test_input", + [ + (""), + ('{"inavlid","json"}'), + (u"😍"), + ], +) +def test_json_translate_failures(test_input): + with pytest.raises(json.decoder.JSONDecodeError): + JsonTranslator().translate(test_input) + +class Dummy(object): + def __str__(self): + return "foo" + +@pytest.mark.parametrize( + "test_input,expected", + [ + ({"foo":"bar","baz":1}, "{'foo': 'bar', 'baz': 1}"), + ('{"foo":"bar","baz":1}', '{"foo":"bar","baz":1}'), + (["foo","bar",1,2,3], "['foo', 'bar', 1, 2, 3]"), + ('["foo","bar",1,2,3]', '["foo","bar",1,2,3]'), + (Dummy(), 'foo'), + (u"😍", u"😍"), + ], +) +def test_unicode_load(test_input, expected): + assert UnicodeTranslator().load(test_input) == expected + +@pytest.mark.parametrize( + "test_input,expected", + [ + ({"foo":"bar","baz":1}, "{'foo': 'bar', 'baz': 1}"), + ('{"foo":"bar","baz":1}', '{"foo":"bar","baz":1}'), + (["foo","bar",1,2,3], "['foo', 'bar', 1, 2, 3]"), + ('["foo","bar",1,2,3]', '["foo","bar",1,2,3]'), + (Dummy(), 'foo'), + (u"😍", u"😍"), + ], +) +def test_unicode_translate(test_input, expected): + assert UnicodeTranslator().translate(test_input) == expected + +@pytest.fixture +def registry(): + registry = DataTranslatorRegistry() + registry.register('json', JsonTranslator()) + return registry + +def test_registry_register(registry): + registry.register('unicode', UnicodeTranslator()) + assert 'unicode' in registry + +def test_registry_invalid_register(registry): + with pytest.raises(ScrapbookException): + registry.register('unicode', 'not a translator') + +def test_registry_deregister(registry): + registry.deregister('json') + assert 'json' not in registry + +def test_registry_missing_deregister(registry): + with pytest.raises(KeyError): + registry.deregister('unicode') + +def test_registry_reset(registry): + registry.register('unicode', UnicodeTranslator()) + registry.reset() + assert 'json' not in registry + assert 'unicode' not in registry + assert len(registry) == 0 + +def test_load_data(registry): + # Test that it can select and execute the qualified translator + assert registry.load_data('json', '["foobar"]') == ['foobar'] + +def test_translate_data(registry): + # Test that it can select and execute the qualified translator + assert registry.translate_data('json', '["foobar"]') == ['foobar'] diff --git a/scrapbook/translators.py b/scrapbook/translators.py new file mode 100644 index 0000000..20394d9 --- /dev/null +++ b/scrapbook/translators.py @@ -0,0 +1,137 @@ +# -*- coding: utf-8 -*- +""" +notebook.py + +Provides the translators for various data types to be persistable +""" +import six +import json +import collections + +from .exceptions import ScrapbookException + +class DataTranslatorRegistry(collections.MutableMapping): + def __init__(self): + self._translators = {} + + def __getitem__(self, key): + return self._translators.__getitem__(key) + + def __setitem__(self, key, value): + if not (getattr(value, "translate", None) and + callable(value.translate)): + raise ScrapbookException("Can't register object without 'translate' method.") + if not (getattr(value, "load", None) and + callable(value.translate)): + raise ScrapbookException("Can't register object without 'load' method.") + return self._translators.__setitem__(key, value) + + def __delitem__(self, key): + return self._translators.__delitem__(key) + + def __iter__(self): + return self._translators.__iter__() + + def __len__(self): + return self._translators.__len__() + + def register(self, storage_type, translator): + """ + Registers a new storage_type to a particular translator + + Parameters + ---------- + storage_type: str + Name of the mime subtype parsed by the translator. + translator: obj + The object which implements the required functions. + """ + # TODO: Make the translators specify what types they can store? + self[storage_type] = translator + + def deregister(self, storage_type): + """ + Removes a particular translator from the registry + + Parameters + ---------- + storage_type: str + Name of the mime subtype parsed by the translator. + """ + del self[storage_type] + + def reset(self): + """ + Resets the registry to have no translators. + """ + self._translators = {} + + def load_data(self, storage_type, scrap): + """ + Finds the register for the given storage_type and loads the scrap into + a JSON or string object. + + Parameters + ---------- + storage_type: str + Name of the mime subtype parsed by the translator. + scrap: obj + Object to be converted from JSON or string format to the original value. + """ + loader = self._translators.get(storage_type) + if not loader: + raise ScrapbookException('No translator found for "{}" data type!'.format(data_type)) + return loader.load(scrap) + + def translate_data(self, storage_type, scrap): + """ + Finds the register for the given storage_type and translates the scrap into + an object of the translator output type. + + Parameters + ---------- + storage_type: str + Name of the mime subtype parsed by the translator. + scrap: obj + Object to be converted to JSON or string format for storage in an output + """ + translator = self._translators.get(storage_type) + if not translator: + raise ScrapbookException('No translator found for "{}" data type!'.format(data_type)) + return translator.translate(scrap) + +class JsonTranslator(object): + def translate(self, scrap): + if isinstance(scrap, six.string_types): + return json.loads(scrap) + return scrap + + def load(self, scrap): + # Just in case we somehow got a valid JSON string pushed + if isinstance(scrap, six.string_types): + return json.loads(scrap) + return scrap + +class UnicodeTranslator(object): + def translate(self, scrap): + if not isinstance(scrap, six.string_types): + return str(scrap) + return scrap + + def load(self, scrap): + # Just in case we somehow got a non-string saved?! + if not isinstance(scrap, six.string_types): + return str(scrap) + return scrap + +class ArrowDataframeTranslator(object): + def translate(self, scrap): + pass # TODO: Implement + + def load(self, scrap): + pass # TODO: Implement + +registry = DataTranslatorRegistry() +registry.register('unicode', UnicodeTranslator()) +registry.register('json', JsonTranslator()) +# registry.register('arrow', ArrowDataframeTranslator()) diff --git a/scrapbook/version.py b/scrapbook/version.py new file mode 100644 index 0000000..75b8a57 --- /dev/null +++ b/scrapbook/version.py @@ -0,0 +1 @@ +version = '0.1.0' diff --git a/setup.cfg b/setup.cfg index f228440..7d08a96 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,13 +24,13 @@ max-line-length = 120 [bdist_wheel] universal=1 -[versioneer] -VCS = git -style = pep440 -versionfile_source = scrapbook/_version.py -versionfile_build = scrapbook/_version.py -tag_prefix = -parentdir_prefix = scrapbook- +[bumpversion] +current_version = 0.0.0 +commit = False +tag = True +tag_name = {new_version} + +[bumpversion:file:scrapbook/version.py] [coverage:run] branch = False diff --git a/setup.py b/setup.py index 1c856ab..053f2d2 100644 --- a/setup.py +++ b/setup.py @@ -1,18 +1,5 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -"""" -setup.py - -Note: Do a version check for IPython. - IPython v6+ no longer supports Python 2. - If Python 2, intall ipython 5.x. - -See: -https://packaging.python.org/tutorials/packaging-projects/ -https://packaging.python.org/en/latest/distributing.html -https://github.com/pypa/sampleproject - -""" from __future__ import print_function import os import sys @@ -25,8 +12,6 @@ # Python 3 only projects can skip this import from io import open -import versioneer - python_2 = sys.version_info[0] == 2 @@ -35,6 +20,20 @@ def read(fname): return fhandle.read() +local_path = os.path.dirname(__file__) +# Fix for tox which manipulates execution pathing +if not local_path: + local_path = '.' + + +def version(): + with open(local_path + '/scrapbook/version.py', 'r') as ver: + for line in ver.readlines(): + if line.startswith('version ='): + return line.split(' = ')[-1].strip()[1:-1] + raise ValueError('No version found in scrapbook/version.py') + + req_path = os.path.join(os.path.dirname('__file__'), 'requirements.txt') required = [req.strip() for req in read(req_path).splitlines() if req.strip()] @@ -76,8 +75,7 @@ def read(fname): setup( name='scrapbook', - version=versioneer.get_version(), - cmdclass=versioneer.get_cmdclass(), + version=version(), description='A library for recording and reading data in Jupyter and nteract Notebooks', author='nteract contributors', author_email='nteract@googlegroups.com', diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..566f6b8 --- /dev/null +++ b/tox.ini @@ -0,0 +1,12 @@ +[tox] +envlist = py27, py36 + +[testenv] +# disable Python's hash randomization for tests that stringify dicts, etc +setenv = + PYTHONHASHSEED = 0 +passenv = * +commands = pytest -v --maxfail=2 --cov=scrapbook --pyargs scrapbook -W always +deps = + pytest + pytest-cov diff --git a/versioneer.py b/versioneer.py deleted file mode 100644 index 8eee493..0000000 --- a/versioneer.py +++ /dev/null @@ -1,1858 +0,0 @@ - -# Version: 0.18 - -"""The Versioneer - like a rocketeer, but for versions. - -The Versioneer -============== - -* like a rocketeer, but for versions! -* https://github.com/warner/python-versioneer -* Brian Warner -* License: Public Domain -* Compatible With: python2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6, and pypy -* [![Latest Version] -(https://pypip.in/version/versioneer/badge.svg?style=flat) -](https://pypi.python.org/pypi/versioneer/) -* [![Build Status] -(https://travis-ci.org/warner/python-versioneer.png?branch=master) -](https://travis-ci.org/warner/python-versioneer) - -This is a tool for managing a recorded version number in distutils-based -python projects. The goal is to remove the tedious and error-prone "update -the embedded version string" step from your release process. Making a new -release should be as easy as recording a new tag in your version-control -system, and maybe making new tarballs. - - -## Quick Install - -* `pip install versioneer` to somewhere to your $PATH -* add a `[versioneer]` section to your setup.cfg (see below) -* run `versioneer install` in your source tree, commit the results - -## Version Identifiers - -Source trees come from a variety of places: - -* a version-control system checkout (mostly used by developers) -* a nightly tarball, produced by build automation -* a snapshot tarball, produced by a web-based VCS browser, like github's - "tarball from tag" feature -* a release tarball, produced by "setup.py sdist", distributed through PyPI - -Within each source tree, the version identifier (either a string or a number, -this tool is format-agnostic) can come from a variety of places: - -* ask the VCS tool itself, e.g. "git describe" (for checkouts), which knows - about recent "tags" and an absolute revision-id -* the name of the directory into which the tarball was unpacked -* an expanded VCS keyword ($Id$, etc) -* a `_version.py` created by some earlier build step - -For released software, the version identifier is closely related to a VCS -tag. Some projects use tag names that include more than just the version -string (e.g. "myproject-1.2" instead of just "1.2"), in which case the tool -needs to strip the tag prefix to extract the version identifier. For -unreleased software (between tags), the version identifier should provide -enough information to help developers recreate the same tree, while also -giving them an idea of roughly how old the tree is (after version 1.2, before -version 1.3). Many VCS systems can report a description that captures this, -for example `git describe --tags --dirty --always` reports things like -"0.7-1-g574ab98-dirty" to indicate that the checkout is one revision past the -0.7 tag, has a unique revision id of "574ab98", and is "dirty" (it has -uncommitted changes. - -The version identifier is used for multiple purposes: - -* to allow the module to self-identify its version: `myproject.__version__` -* to choose a name and prefix for a 'setup.py sdist' tarball - -## Theory of Operation - -Versioneer works by adding a special `_version.py` file into your source -tree, where your `__init__.py` can import it. This `_version.py` knows how to -dynamically ask the VCS tool for version information at import time. - -`_version.py` also contains `$Revision$` markers, and the installation -process marks `_version.py` to have this marker rewritten with a tag name -during the `git archive` command. As a result, generated tarballs will -contain enough information to get the proper version. - -To allow `setup.py` to compute a version too, a `versioneer.py` is added to -the top level of your source tree, next to `setup.py` and the `setup.cfg` -that configures it. This overrides several distutils/setuptools commands to -compute the version when invoked, and changes `setup.py build` and `setup.py -sdist` to replace `_version.py` with a small static file that contains just -the generated version data. - -## Installation - -See [INSTALL.md](./INSTALL.md) for detailed installation instructions. - -## Version-String Flavors - -Code which uses Versioneer can learn about its version string at runtime by -importing `_version` from your main `__init__.py` file and running the -`get_versions()` function. From the "outside" (e.g. in `setup.py`), you can -import the top-level `versioneer.py` and run `get_versions()`. - -Both functions return a dictionary with different flavors of version -information: - -* `['version']`: A condensed version string, rendered using the selected - style. This is the most commonly used value for the project's version - string. The default "pep440" style yields strings like `0.11`, - `0.11+2.g1076c97`, or `0.11+2.g1076c97.dirty`. See the "Styles" section - below for alternative styles. - -* `['full-revisionid']`: detailed revision identifier. For Git, this is the - full SHA1 commit id, e.g. "1076c978a8d3cfc70f408fe5974aa6c092c949ac". - -* `['date']`: Date and time of the latest `HEAD` commit. For Git, it is the - commit date in ISO 8601 format. This will be None if the date is not - available. - -* `['dirty']`: a boolean, True if the tree has uncommitted changes. Note that - this is only accurate if run in a VCS checkout, otherwise it is likely to - be False or None - -* `['error']`: if the version string could not be computed, this will be set - to a string describing the problem, otherwise it will be None. It may be - useful to throw an exception in setup.py if this is set, to avoid e.g. - creating tarballs with a version string of "unknown". - -Some variants are more useful than others. Including `full-revisionid` in a -bug report should allow developers to reconstruct the exact code being tested -(or indicate the presence of local changes that should be shared with the -developers). `version` is suitable for display in an "about" box or a CLI -`--version` output: it can be easily compared against release notes and lists -of bugs fixed in various releases. - -The installer adds the following text to your `__init__.py` to place a basic -version in `YOURPROJECT.__version__`: - - from ._version import get_versions - __version__ = get_versions()['version'] - del get_versions - -## Styles - -The setup.cfg `style=` configuration controls how the VCS information is -rendered into a version string. - -The default style, "pep440", produces a PEP440-compliant string, equal to the -un-prefixed tag name for actual releases, and containing an additional "local -version" section with more detail for in-between builds. For Git, this is -TAG[+DISTANCE.gHEX[.dirty]] , using information from `git describe --tags ---dirty --always`. For example "0.11+2.g1076c97.dirty" indicates that the -tree is like the "1076c97" commit but has uncommitted changes (".dirty"), and -that this commit is two revisions ("+2") beyond the "0.11" tag. For released -software (exactly equal to a known tag), the identifier will only contain the -stripped tag, e.g. "0.11". - -Other styles are available. See [details.md](details.md) in the Versioneer -source tree for descriptions. - -## Debugging - -Versioneer tries to avoid fatal errors: if something goes wrong, it will tend -to return a version of "0+unknown". To investigate the problem, run `setup.py -version`, which will run the version-lookup code in a verbose mode, and will -display the full contents of `get_versions()` (including the `error` string, -which may help identify what went wrong). - -## Known Limitations - -Some situations are known to cause problems for Versioneer. This details the -most significant ones. More can be found on Github -[issues page](https://github.com/warner/python-versioneer/issues). - -### Subprojects - -Versioneer has limited support for source trees in which `setup.py` is not in -the root directory (e.g. `setup.py` and `.git/` are *not* siblings). The are -two common reasons why `setup.py` might not be in the root: - -* Source trees which contain multiple subprojects, such as - [Buildbot](https://github.com/buildbot/buildbot), which contains both - "master" and "slave" subprojects, each with their own `setup.py`, - `setup.cfg`, and `tox.ini`. Projects like these produce multiple PyPI - distributions (and upload multiple independently-installable tarballs). -* Source trees whose main purpose is to contain a C library, but which also - provide bindings to Python (and perhaps other langauges) in subdirectories. - -Versioneer will look for `.git` in parent directories, and most operations -should get the right version string. However `pip` and `setuptools` have bugs -and implementation details which frequently cause `pip install .` from a -subproject directory to fail to find a correct version string (so it usually -defaults to `0+unknown`). - -`pip install --editable .` should work correctly. `setup.py install` might -work too. - -Pip-8.1.1 is known to have this problem, but hopefully it will get fixed in -some later version. - -[Bug #38](https://github.com/warner/python-versioneer/issues/38) is tracking -this issue. The discussion in -[PR #61](https://github.com/warner/python-versioneer/pull/61) describes the -issue from the Versioneer side in more detail. -[pip PR#3176](https://github.com/pypa/pip/pull/3176) and -[pip PR#3615](https://github.com/pypa/pip/pull/3615) contain work to improve -pip to let Versioneer work correctly. - -Versioneer-0.16 and earlier only looked for a `.git` directory next to the -`setup.cfg`, so subprojects were completely unsupported with those releases. - -### Editable installs with setuptools <= 18.5 - -`setup.py develop` and `pip install --editable .` allow you to install a -project into a virtualenv once, then continue editing the source code (and -test) without re-installing after every change. - -"Entry-point scripts" (`setup(entry_points={"console_scripts": ..})`) are a -convenient way to specify executable scripts that should be installed along -with the python package. - -These both work as expected when using modern setuptools. When using -setuptools-18.5 or earlier, however, certain operations will cause -`pkg_resources.DistributionNotFound` errors when running the entrypoint -script, which must be resolved by re-installing the package. This happens -when the install happens with one version, then the egg_info data is -regenerated while a different version is checked out. Many setup.py commands -cause egg_info to be rebuilt (including `sdist`, `wheel`, and installing into -a different virtualenv), so this can be surprising. - -[Bug #83](https://github.com/warner/python-versioneer/issues/83) describes -this one, but upgrading to a newer version of setuptools should probably -resolve it. - -### Unicode version strings - -While Versioneer works (and is continually tested) with both Python 2 and -Python 3, it is not entirely consistent with bytes-vs-unicode distinctions. -Newer releases probably generate unicode version strings on py2. It's not -clear that this is wrong, but it may be surprising for applications when then -write these strings to a network connection or include them in bytes-oriented -APIs like cryptographic checksums. - -[Bug #71](https://github.com/warner/python-versioneer/issues/71) investigates -this question. - - -## Updating Versioneer - -To upgrade your project to a new release of Versioneer, do the following: - -* install the new Versioneer (`pip install -U versioneer` or equivalent) -* edit `setup.cfg`, if necessary, to include any new configuration settings - indicated by the release notes. See [UPGRADING](./UPGRADING.md) for details. -* re-run `versioneer install` in your source tree, to replace - `SRC/_version.py` -* commit any changed files - -## Future Directions - -This tool is designed to make it easily extended to other version-control -systems: all VCS-specific components are in separate directories like -src/git/ . The top-level `versioneer.py` script is assembled from these -components by running make-versioneer.py . In the future, make-versioneer.py -will take a VCS name as an argument, and will construct a version of -`versioneer.py` that is specific to the given VCS. It might also take the -configuration arguments that are currently provided manually during -installation by editing setup.py . Alternatively, it might go the other -direction and include code from all supported VCS systems, reducing the -number of intermediate scripts. - - -## License - -To make Versioneer easier to embed, all its code is dedicated to the public -domain. The `_version.py` that it creates is also in the public domain. -Specifically, both are released under the Creative Commons "Public Domain -Dedication" license (CC0-1.0), as described in -https://creativecommons.org/publicdomain/zero/1.0/ . - -""" - -from __future__ import print_function - -try: - import configparser -except ImportError: - import ConfigParser as configparser -import errno -import json -import os -import re -import subprocess -import sys - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_root(): - """Get the project root directory. - - We require that all commands are run from the project root, i.e. the - directory that contains setup.py, setup.cfg, and versioneer.py . - """ - root = os.path.realpath(os.path.abspath(os.getcwd())) - setup_py = os.path.join(root, "setup.py") - versioneer_py = os.path.join(root, "versioneer.py") - if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): - # allow 'python path/to/setup.py COMMAND' - root = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0]))) - setup_py = os.path.join(root, "setup.py") - versioneer_py = os.path.join(root, "versioneer.py") - if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): - err = ( - "Versioneer was unable to run the project root directory. " - "Versioneer requires setup.py to be executed from " - "its immediate directory (like 'python setup.py COMMAND'), " - "or in a way that lets it use sys.argv[0] to find the root " - "(like 'python path/to/setup.py COMMAND')." - ) - raise VersioneerBadRootError(err) - try: - # Certain runtime workflows (setup.py install/develop in a setuptools - # tree) execute all dependencies in a single python process, so - # "versioneer" may be imported multiple times, and python's shared - # module-import table will cache the first one. So we can't use - # os.path.dirname(__file__), as that will find whichever - # versioneer.py was first imported, even in later projects. - me = os.path.realpath(os.path.abspath(__file__)) - me_dir = os.path.normcase(os.path.splitext(me)[0]) - vsr_dir = os.path.normcase(os.path.splitext(versioneer_py)[0]) - if me_dir != vsr_dir: - print( - "Warning: build in %s is using versioneer.py from %s" - % (os.path.dirname(me), versioneer_py) - ) - except NameError: - pass - return root - - -def get_config_from_root(root): - """Read the project setup.cfg file to determine Versioneer config.""" - # This might raise EnvironmentError (if setup.cfg is missing), or - # configparser.NoSectionError (if it lacks a [versioneer] section), or - # configparser.NoOptionError (if it lacks "VCS="). See the docstring at - # the top of versioneer.py for instructions on writing your setup.cfg . - setup_cfg = os.path.join(root, "setup.cfg") - parser = configparser.SafeConfigParser() - with open(setup_cfg, "r") as f: - parser.readfp(f) - VCS = parser.get("versioneer", "VCS") # mandatory - - def get(parser, name): - if parser.has_option("versioneer", name): - return parser.get("versioneer", name) - return None - - cfg = VersioneerConfig() - cfg.VCS = VCS - cfg.style = get(parser, "style") or "" - cfg.versionfile_source = get(parser, "versionfile_source") - cfg.versionfile_build = get(parser, "versionfile_build") - cfg.tag_prefix = get(parser, "tag_prefix") - if cfg.tag_prefix in ("''", '""'): - cfg.tag_prefix = "" - cfg.parentdir_prefix = get(parser, "parentdir_prefix") - cfg.verbose = get(parser, "verbose") - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -# these dictionaries contain VCS-specific tools -LONG_VERSION_PY = {} -HANDLERS = {} - - -def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" - - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - - return decorate - - -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=None): - """Call the given command(s).""" - assert isinstance(commands, list) - p = None - for c in commands: - try: - dispcmd = str([c] + args) - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen( - [c] + args, - cwd=cwd, - env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr else None), - ) - break - except EnvironmentError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %s" % dispcmd) - print(e) - return None, None - else: - if verbose: - print("unable to find command, tried %s" % (commands,)) - return None, None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: - if verbose: - print("unable to run %s (error)" % dispcmd) - print("stdout was %s" % stdout) - return None, p.returncode - return stdout, p.returncode - - -LONG_VERSION_PY[ - 'git' -] = ''' -# This file helps to compute a version number in source trees obtained from -# git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (built by setup.py sdist) and build -# directories (produced by setup.py build) will contain a much shorter file -# that just contains the computed version number. - -# This file is released into the public domain. Generated by -# versioneer-0.18 (https://github.com/warner/python-versioneer) - -"""Git implementation of _version.py.""" - -import errno -import os -import re -import subprocess -import sys - - -def get_keywords(): - """Get the keywords needed to look up the version information.""" - # these strings will be replaced by git during git-archive. - # setup.py/versioneer.py will grep for the variable names, so they must - # each be defined on a line of their own. _version.py will just call - # get_keywords(). - git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s" - git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s" - git_date = "%(DOLLAR)sFormat:%%ci%(DOLLAR)s" - keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} - return keywords - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_config(): - """Create, populate and return the VersioneerConfig() object.""" - # these strings are filled in when 'setup.py versioneer' creates - # _version.py - cfg = VersioneerConfig() - cfg.VCS = "git" - cfg.style = "%(STYLE)s" - cfg.tag_prefix = "%(TAG_PREFIX)s" - cfg.parentdir_prefix = "%(PARENTDIR_PREFIX)s" - cfg.versionfile_source = "%(VERSIONFILE_SOURCE)s" - cfg.verbose = False - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -LONG_VERSION_PY = {} -HANDLERS = {} - - -def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - return decorate - - -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, - env=None): - """Call the given command(s).""" - assert isinstance(commands, list) - p = None - for c in commands: - try: - dispcmd = str([c] + args) - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen([c] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None)) - break - except EnvironmentError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %%s" %% dispcmd) - print(e) - return None, None - else: - if verbose: - print("unable to find command, tried %%s" %% (commands,)) - return None, None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: - if verbose: - print("unable to run %%s (error)" %% dispcmd) - print("stdout was %%s" %% stdout) - return None, p.returncode - return stdout, p.returncode - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes both - the project name and a version string. We will also support searching up - two directory levels for an appropriately named parent directory - """ - rootdirs = [] - - for i in range(3): - dirname = os.path.basename(root) - if dirname.startswith(parentdir_prefix): - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None, "date": None} - else: - rootdirs.append(root) - root = os.path.dirname(root) # up a level - - if verbose: - print("Tried directories %%s but none started with prefix %%s" %% - (str(rootdirs), parentdir_prefix)) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - f = open(versionfile_abs, "r") - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - f.close() - except EnvironmentError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") - date = keywords.get("date") - if date is not None: - # git-2.2.0 added "%%cI", which expands to an ISO-8601 -compliant - # datestamp. However we prefer "%%ci" (which expands to an "ISO-8601 - # -like" string, which we must then edit to make compliant), because - # it's been around since git-1.5.3, and it's too difficult to - # discover which version we're using, or to work around using an - # older one. - date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %%d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r'\d', r)]) - if verbose: - print("discarding '%%s', no digits" %% ",".join(refs - tags)) - if verbose: - print("likely tags: %%s" %% ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] - if verbose: - print("picking %%s" %% r) - return {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None, - "date": date} - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return {"version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags", "date": None} - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - - out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=True) - if rc != 0: - if verbose: - print("Directory %%s not under git control" %% root) - raise NotThisMethod("'git rev-parse --git-dir' returned error") - - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%%s*" %% tag_prefix], - cwd=root) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[:git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) - if not mo: - # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = ("unable to parse git-describe output: '%%s'" - %% describe_out) - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%%s' doesn't start with prefix '%%s'" - print(fmt %% (full_tag, tag_prefix)) - pieces["error"] = ("tag '%%s' doesn't start with prefix '%%s'" - %% (full_tag, tag_prefix)) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], - cwd=root) - pieces["distance"] = int(count_out) # total number of commits - - # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%%ci", "HEAD"], - cwd=root)[0].strip() - pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - - return pieces - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%%d.g%%s" %% (pieces["distance"], - pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post.devDISTANCE - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += ".post.dev%%d" %% pieces["distance"] - else: - # exception #1 - rendered = "0.post.dev%%d" %% pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%%s" %% pieces["short"] - else: - # exception #1 - rendered = "0.post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%%s" %% pieces["short"] - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Eexceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return {"version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None} - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%%s'" %% style) - - return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None, - "date": pieces.get("date")} - - -def get_versions(): - """Get version information or return default if unable to do so.""" - # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have - # __file__, we can work backwards from there to the root. Some - # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which - # case we can only use expanded keywords. - - cfg = get_config() - verbose = cfg.verbose - - try: - return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, - verbose) - except NotThisMethod: - pass - - try: - root = os.path.realpath(__file__) - # versionfile_source is the relative path from the top of the source - # tree (where the .git directory might live) to this file. Invert - # this to find the root from __file__. - for i in cfg.versionfile_source.split('/'): - root = os.path.dirname(root) - except NameError: - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to find root of source tree", - "date": None} - - try: - pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) - return render(pieces, cfg.style) - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - except NotThisMethod: - pass - - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", "date": None} -''' - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - f = open(versionfile_abs, "r") - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - f.close() - except EnvironmentError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") - date = keywords.get("date") - if date is not None: - # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant - # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 - # -like" string, which we must then edit to make compliant), because - # it's been around since git-1.5.3, and it's too difficult to - # discover which version we're using, or to work around using an - # older one. - date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)]) - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r'\d', r)]) - if verbose: - print("discarding '%s', no digits" % ",".join(refs - tags)) - if verbose: - print("likely tags: %s" % ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix) :] - if verbose: - print("picking %s" % r) - return { - "version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": None, - "date": date, - } - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return { - "version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": "no suitable tags", - "date": None, - } - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - - out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True) - if rc != 0: - if verbose: - print("Directory %s not under git control" % root) - raise NotThisMethod("'git rev-parse --git-dir' returned error") - - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command( - GITS, - ["describe", "--tags", "--dirty", "--always", "--long", "--match", "%s*" % tag_prefix], - cwd=root, - ) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[: git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) - if not mo: - # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = "unable to parse git-describe output: '%s'" % describe_out - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%s' doesn't start with prefix '%s'" - print(fmt % (full_tag, tag_prefix)) - pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % (full_tag, tag_prefix) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix) :] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], cwd=root) - pieces["distance"] = int(count_out) # total number of commits - - # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[0].strip() - pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - - return pieces - - -def do_vcs_install(manifest_in, versionfile_source, ipy): - """Git-specific installation logic for Versioneer. - - For Git, this means creating/changing .gitattributes to mark _version.py - for export-subst keyword substitution. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - files = [manifest_in, versionfile_source] - if ipy: - files.append(ipy) - try: - me = __file__ - if me.endswith(".pyc") or me.endswith(".pyo"): - me = os.path.splitext(me)[0] + ".py" - versioneer_file = os.path.relpath(me) - except NameError: - versioneer_file = "versioneer.py" - files.append(versioneer_file) - present = False - try: - f = open(".gitattributes", "r") - for line in f.readlines(): - if line.strip().startswith(versionfile_source): - if "export-subst" in line.strip().split()[1:]: - present = True - f.close() - except EnvironmentError: - pass - if not present: - f = open(".gitattributes", "a+") - f.write("%s export-subst\n" % versionfile_source) - f.close() - files.append(".gitattributes") - run_command(GITS, ["add", "--"] + files) - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes both - the project name and a version string. We will also support searching up - two directory levels for an appropriately named parent directory - """ - rootdirs = [] - - for i in range(3): - dirname = os.path.basename(root) - if dirname.startswith(parentdir_prefix): - return { - "version": dirname[len(parentdir_prefix) :], - "full-revisionid": None, - "dirty": False, - "error": None, - "date": None, - } - else: - rootdirs.append(root) - root = os.path.dirname(root) # up a level - - if verbose: - print( - "Tried directories %s but none started with prefix %s" - % (str(rootdirs), parentdir_prefix) - ) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - - -SHORT_VERSION_PY = """ -# This file was generated by 'versioneer.py' (0.18) from -# revision-control system data, or from the parent directory name of an -# unpacked source archive. Distribution tarballs contain a pre-generated copy -# of this file. - -import json - -version_json = ''' -%s -''' # END VERSION_JSON - - -def get_versions(): - return json.loads(version_json) -""" - - -def versions_from_file(filename): - """Try to determine the version from _version.py if present.""" - try: - with open(filename) as f: - contents = f.read() - except EnvironmentError: - raise NotThisMethod("unable to read _version.py") - mo = re.search(r"version_json = '''\n(.*)''' # END VERSION_JSON", contents, re.M | re.S) - if not mo: - mo = re.search(r"version_json = '''\r\n(.*)''' # END VERSION_JSON", contents, re.M | re.S) - if not mo: - raise NotThisMethod("no version_json in _version.py") - return json.loads(mo.group(1)) - - -def write_to_version_file(filename, versions): - """Write the given version number to the given _version.py file.""" - os.unlink(filename) - contents = json.dumps(versions, sort_keys=True, indent=1, separators=(",", ": ")) - with open(filename, "w") as f: - f.write(SHORT_VERSION_PY % contents) - - print("set %s to '%s'" % (filename, versions["version"])) - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post.devDISTANCE - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += ".post.dev%d" % pieces["distance"] - else: - # exception #1 - rendered = "0.post.dev%d" % pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%s" % pieces["short"] - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%s" % pieces["short"] - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Eexceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return { - "version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None, - } - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%s'" % style) - - return { - "version": rendered, - "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], - "error": None, - "date": pieces.get("date"), - } - - -class VersioneerBadRootError(Exception): - """The project root directory is unknown or missing key files.""" - - -def get_versions(verbose=False): - """Get the project version from whatever source is available. - - Returns dict with two keys: 'version' and 'full'. - """ - if "versioneer" in sys.modules: - # see the discussion in cmdclass.py:get_cmdclass() - del sys.modules["versioneer"] - - root = get_root() - cfg = get_config_from_root(root) - - assert cfg.VCS is not None, "please set [versioneer]VCS= in setup.cfg" - handlers = HANDLERS.get(cfg.VCS) - assert handlers, "unrecognized VCS '%s'" % cfg.VCS - verbose = verbose or cfg.verbose - assert cfg.versionfile_source is not None, "please set versioneer.versionfile_source" - assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" - - versionfile_abs = os.path.join(root, cfg.versionfile_source) - - # extract version from first of: _version.py, VCS command (e.g. 'git - # describe'), parentdir. This is meant to work for developers using a - # source checkout, for users of a tarball created by 'setup.py sdist', - # and for users of a tarball/zipball created by 'git archive' or github's - # download-from-tag feature or the equivalent in other VCSes. - - get_keywords_f = handlers.get("get_keywords") - from_keywords_f = handlers.get("keywords") - if get_keywords_f and from_keywords_f: - try: - keywords = get_keywords_f(versionfile_abs) - ver = from_keywords_f(keywords, cfg.tag_prefix, verbose) - if verbose: - print("got version from expanded keyword %s" % ver) - return ver - except NotThisMethod: - pass - - try: - ver = versions_from_file(versionfile_abs) - if verbose: - print("got version from file %s %s" % (versionfile_abs, ver)) - return ver - except NotThisMethod: - pass - - from_vcs_f = handlers.get("pieces_from_vcs") - if from_vcs_f: - try: - pieces = from_vcs_f(cfg.tag_prefix, root, verbose) - ver = render(pieces, cfg.style) - if verbose: - print("got version from VCS %s" % ver) - return ver - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - ver = versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - if verbose: - print("got version from parentdir %s" % ver) - return ver - except NotThisMethod: - pass - - if verbose: - print("unable to compute version") - - return { - "version": "0+unknown", - "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", - "date": None, - } - - -def get_version(): - """Get the short version string for this project.""" - return get_versions()["version"] - - -def get_cmdclass(): - """Get the custom setuptools/distutils subclasses used by Versioneer.""" - if "versioneer" in sys.modules: - del sys.modules["versioneer"] - # this fixes the "python setup.py develop" case (also 'install' and - # 'easy_install .'), in which subdependencies of the main project are - # built (using setup.py bdist_egg) in the same python process. Assume - # a main project A and a dependency B, which use different versions - # of Versioneer. A's setup.py imports A's Versioneer, leaving it in - # sys.modules by the time B's setup.py is executed, causing B to run - # with the wrong versioneer. Setuptools wraps the sub-dep builds in a - # sandbox that restores sys.modules to it's pre-build state, so the - # parent is protected against the child's "import versioneer". By - # removing ourselves from sys.modules here, before the child build - # happens, we protect the child from the parent's versioneer too. - # Also see https://github.com/warner/python-versioneer/issues/52 - - cmds = {} - - # we add "version" to both distutils and setuptools - from distutils.core import Command - - class cmd_version(Command): - description = "report generated version string" - user_options = [] - boolean_options = [] - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - vers = get_versions(verbose=True) - print("Version: %s" % vers["version"]) - print(" full-revisionid: %s" % vers.get("full-revisionid")) - print(" dirty: %s" % vers.get("dirty")) - print(" date: %s" % vers.get("date")) - if vers["error"]: - print(" error: %s" % vers["error"]) - - cmds["version"] = cmd_version - - # we override "build_py" in both distutils and setuptools - # - # most invocation pathways end up running build_py: - # distutils/build -> build_py - # distutils/install -> distutils/build ->.. - # setuptools/bdist_wheel -> distutils/install ->.. - # setuptools/bdist_egg -> distutils/install_lib -> build_py - # setuptools/install -> bdist_egg ->.. - # setuptools/develop -> ? - # pip install: - # copies source tree to a tempdir before running egg_info/etc - # if .git isn't copied too, 'git describe' will fail - # then does setup.py bdist_wheel, or sometimes setup.py install - # setup.py egg_info -> ? - - # we override different "build_py" commands for both environments - if "setuptools" in sys.modules: - from setuptools.command.build_py import build_py as _build_py - else: - from distutils.command.build_py import build_py as _build_py - - class cmd_build_py(_build_py): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - _build_py.run(self) - # now locate _version.py in the new build/ directory and replace - # it with an updated value - if cfg.versionfile_build: - target_versionfile = os.path.join(self.build_lib, cfg.versionfile_build) - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - cmds["build_py"] = cmd_build_py - - if "cx_Freeze" in sys.modules: # cx_freeze enabled? - from cx_Freeze.dist import build_exe as _build_exe - - # nczeczulin reports that py2exe won't like the pep440-style string - # as FILEVERSION, but it can be used for PRODUCTVERSION, e.g. - # setup(console=[{ - # "version": versioneer.get_version().split("+", 1)[0], # FILEVERSION - # "product_version": versioneer.get_version(), - # ... - - class cmd_build_exe(_build_exe): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - target_versionfile = cfg.versionfile_source - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - _build_exe.run(self) - os.unlink(target_versionfile) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write( - LONG - % { - "DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - } - ) - - cmds["build_exe"] = cmd_build_exe - del cmds["build_py"] - - if 'py2exe' in sys.modules: # py2exe enabled? - try: - from py2exe.distutils_buildexe import py2exe as _py2exe # py3 - except ImportError: - from py2exe.build_exe import py2exe as _py2exe # py2 - - class cmd_py2exe(_py2exe): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - target_versionfile = cfg.versionfile_source - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - _py2exe.run(self) - os.unlink(target_versionfile) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write( - LONG - % { - "DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - } - ) - - cmds["py2exe"] = cmd_py2exe - - # we override different "sdist" commands for both environments - if "setuptools" in sys.modules: - from setuptools.command.sdist import sdist as _sdist - else: - from distutils.command.sdist import sdist as _sdist - - class cmd_sdist(_sdist): - def run(self): - versions = get_versions() - self._versioneer_generated_versions = versions - # unless we update this, the command will keep using the old - # version - self.distribution.metadata.version = versions["version"] - return _sdist.run(self) - - def make_release_tree(self, base_dir, files): - root = get_root() - cfg = get_config_from_root(root) - _sdist.make_release_tree(self, base_dir, files) - # now locate _version.py in the new base_dir directory - # (remembering that it may be a hardlink) and replace it with an - # updated value - target_versionfile = os.path.join(base_dir, cfg.versionfile_source) - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, self._versioneer_generated_versions) - - cmds["sdist"] = cmd_sdist - - return cmds - - -CONFIG_ERROR = """ -setup.cfg is missing the necessary Versioneer configuration. You need -a section like: - - [versioneer] - VCS = git - style = pep440 - versionfile_source = src/myproject/_version.py - versionfile_build = myproject/_version.py - tag_prefix = - parentdir_prefix = myproject- - -You will also need to edit your setup.py to use the results: - - import versioneer - setup(version=versioneer.get_version(), - cmdclass=versioneer.get_cmdclass(), ...) - -Please read the docstring in ./versioneer.py for configuration instructions, -edit setup.cfg, and re-run the installer or 'python versioneer.py setup'. -""" - -SAMPLE_CONFIG = """ -# See the docstring in versioneer.py for instructions. Note that you must -# re-run 'versioneer.py setup' after changing this section, and commit the -# resulting files. - -[versioneer] -#VCS = git -#style = pep440 -#versionfile_source = -#versionfile_build = -#tag_prefix = -#parentdir_prefix = - -""" - -INIT_PY_SNIPPET = """ -from ._version import get_versions -__version__ = get_versions()['version'] -del get_versions -""" - - -def do_setup(): - """Main VCS-independent setup function for installing Versioneer.""" - root = get_root() - try: - cfg = get_config_from_root(root) - except (EnvironmentError, configparser.NoSectionError, configparser.NoOptionError) as e: - if isinstance(e, (EnvironmentError, configparser.NoSectionError)): - print("Adding sample versioneer config to setup.cfg", file=sys.stderr) - with open(os.path.join(root, "setup.cfg"), "a") as f: - f.write(SAMPLE_CONFIG) - print(CONFIG_ERROR, file=sys.stderr) - return 1 - - print(" creating %s" % cfg.versionfile_source) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write( - LONG - % { - "DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - } - ) - - ipy = os.path.join(os.path.dirname(cfg.versionfile_source), "__init__.py") - if os.path.exists(ipy): - try: - with open(ipy, "r") as f: - old = f.read() - except EnvironmentError: - old = "" - if INIT_PY_SNIPPET not in old: - print(" appending to %s" % ipy) - with open(ipy, "a") as f: - f.write(INIT_PY_SNIPPET) - else: - print(" %s unmodified" % ipy) - else: - print(" %s doesn't exist, ok" % ipy) - ipy = None - - # Make sure both the top-level "versioneer.py" and versionfile_source - # (PKG/_version.py, used by runtime code) are in MANIFEST.in, so - # they'll be copied into source distributions. Pip won't be able to - # install the package without this. - manifest_in = os.path.join(root, "MANIFEST.in") - simple_includes = set() - try: - with open(manifest_in, "r") as f: - for line in f: - if line.startswith("include "): - for include in line.split()[1:]: - simple_includes.add(include) - except EnvironmentError: - pass - # That doesn't cover everything MANIFEST.in can do - # (http://docs.python.org/2/distutils/sourcedist.html#commands), so - # it might give some false negatives. Appending redundant 'include' - # lines is safe, though. - if "versioneer.py" not in simple_includes: - print(" appending 'versioneer.py' to MANIFEST.in") - with open(manifest_in, "a") as f: - f.write("include versioneer.py\n") - else: - print(" 'versioneer.py' already in MANIFEST.in") - if cfg.versionfile_source not in simple_includes: - print(" appending versionfile_source ('%s') to MANIFEST.in" % cfg.versionfile_source) - with open(manifest_in, "a") as f: - f.write("include %s\n" % cfg.versionfile_source) - else: - print(" versionfile_source already in MANIFEST.in") - - # Make VCS-specific changes. For git, this means creating/changing - # .gitattributes to mark _version.py for export-subst keyword - # substitution. - do_vcs_install(manifest_in, cfg.versionfile_source, ipy) - return 0 - - -def scan_setup_py(): - """Validate the contents of setup.py against Versioneer's expectations.""" - found = set() - setters = False - errors = 0 - with open("setup.py", "r") as f: - for line in f.readlines(): - if "import versioneer" in line: - found.add("import") - if "versioneer.get_cmdclass()" in line: - found.add("cmdclass") - if "versioneer.get_version()" in line: - found.add("get_version") - if "versioneer.VCS" in line: - setters = True - if "versioneer.versionfile_source" in line: - setters = True - if len(found) != 3: - print("") - print("Your setup.py appears to be missing some important items") - print("(but I might be wrong). Please make sure it has something") - print("roughly like the following:") - print("") - print(" import versioneer") - print(" setup( version=versioneer.get_version(),") - print(" cmdclass=versioneer.get_cmdclass(), ...)") - print("") - errors += 1 - if setters: - print("You should remove lines like 'versioneer.VCS = ' and") - print("'versioneer.versionfile_source = ' . This configuration") - print("now lives in setup.cfg, and should be removed from setup.py") - print("") - errors += 1 - return errors - - -if __name__ == "__main__": - cmd = sys.argv[1] - if cmd == "setup": - errors = do_setup() - errors += scan_setup_py() - if errors: - sys.exit(1)