How to run python in multiple versions with tox, pip and pip-tools

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

concurrent-api-client
├── .python-version
├── README.md
├── MANIFEST.in
├── requirements.in
├── requirements.testing.in
├── requirements.txt
├── setup.py
├── setup.cfg
├── tox.ini
├── src
│   └── api_client
│       ├── __init__.py
│       ├── api_client.py
│       └── ...
└── tests
    ├── __init__.py
    ├── test_api_formatter.py
    └── ...

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42

from glob import glob
from os.path import basename
from os.path import splitext

from setuptools import find_packages
from setuptools import setup

def parse_requirements(filename):
    """ Given a filename, strip empty lines and those beginning with # """
    output = []
    with open(filename, 'r') as f:
        for line in f:
            sline = line.strip()
            if sline and not line.startswith('#'):
                output.append(sline)
    return output

def find_modules():
    found = []
    for pattern in ['src/*.py']:
#    for pattern in ['src/*.py', 'src/*.json']:
        found.extend([splitext(basename(path))[0] for path in glob(pattern)])
    return found

setup(
    name='concurrent-api-client',
    version='1.0',
    license='MIT',
    description='An example for API client using python request library',
    long_description="\n" + get_readme(),
    author='tatoflam',
    author_email='tatoflam@gamil.com',
    url='https://github.com/tatoflam/concurrent-api-client',
    packages=find_packages('src'),
    package_dir={'': 'src'},
    py_modules=find_modules(),
    include_package_data=True,
    python_requires='>=3.7',
    install_requires=parse_requirements("requirements.txt"),
    tests_require=parse_requirements("requirements.testing.txt"),
    setup_requires=['pytest-runner'],
)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

[tox]
envlist = py37, py38, py39
indexserver =
    default = https://pypi.org/simple

[testenv]
deps=
    -rrequirements.txt
commands=
    pytest -rsfp

[testenv:pip-compile]
deps=
    pip-tools==6.4.0
    pip==21.2.3

commands=
    pip-compile -i https://pypi.org/simple requirements.in requirements.testing.in -o requirements.txt -v

1
2
3
4
5

$ cd concurrent-api-client
$ pyenv local 3.7.8, 3.8.6, 3.9.7 # this set multiple python interpreters
$ pip-compile requirements.testing.in # this output requirements.testing.txt
$ tox -e pip-compile # this generates requirements.txt
$ tox -r # -r option force recreation of virtual environments, then run test in tox virtual environment per python versions

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

$ tox -rvv
using tox.ini: /Users/tatsuro.homma/repo/tatoflam/concurrent-api-client/tox.ini (pid 68529)
...
    ERROR: Command errored out with exit status 1:
     command: /Users/a-user/repo/tatoflam/concurrent-api-client/.tox/py39/bin/python -c 'import io, os, sys, setuptools, tokenize; sys.argv[0] = '"'"'/private/var/folders/q0/g44k3tvx0t7cz98pw1f3tfhw0000gq/T/pip-req-build-pchbcgxt/setup.py'"'"'; __file__='"'"'/private/var/folders/q0/g44k3tvx0t7cz98pw1f3tfhw0000gq/T/pip-req-build-pchbcgxt/setup.py'"'"';f = getattr(tokenize, '"'"'open'"'"', open)(__file__) if os.path.exists(__file__) else io.StringIO('"'"'from setuptools import setup; setup()'"'"');code = f.read().replace('"'"'\r\n'"'"', '"'"'\n'"'"');f.close();exec(compile(code, __file__, '"'"'exec'"'"'))' egg_info --egg-base /private/var/folders/q0/g44k3tvx0t7cz98pw1f3tfhw0000gq/T/pip-pip-egg-info-jlxkkci2
         cwd: /private/var/folders/q0/g44k3tvx0t7cz98pw1f3tfhw0000gq/T/pip-req-build-pchbcgxt/
    Complete output (7 lines):
    Traceback (most recent call last):
      File "<string>", line 1, in <module>
      File "/private/var/folders/q0/g44k3tvx0t7cz98pw1f3tfhw0000gq/T/pip-req-build-pchbcgxt/setup.py", line 71, in <module>
        install_requires=parse_requirements("requirements.txt"),
      File "/private/var/folders/q0/g44k3tvx0t7cz98pw1f3tfhw0000gq/T/pip-req-build-pchbcgxt/setup.py", line 32, in parse_requirements
        with open(filename, 'r') as f:
    FileNotFoundError: [Errno 2] No such file or directory: 'requirements.txt'
    ----------------------------------------

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

# content of: tox.ini , put in same dir as setup.py
[tox]
envlist = py39
indexserver =
    default = https://pypi.org/simple

[testenv]
# install pytest in the virtualenv where commands will be executed
deps=
    -rrequirements.txt
commands=
    {envbindir}/pytest

1
2
3

concurrent-api-client
├── MANIFEST.in
├──...

1

include requirements.txt

1
2
3
4
5

$ tox
...
    distutils.errors.DistutilsError: Command '['/a-user/repo/tatoflam/concurrent-api-client/.tox/py39/bin/python', '-m', 'pip', '--disable-pip-version-check', 'wheel', '--no-deps', '-w', '/var/folders/q0/g44k3tvx0t7cz98pw1f3tfhw0000gq/T/tmppg3oyf_8', '--quiet', 'pytest-runner']' returned non-zero exit status 1.
...
ERROR: invocation failed (exit code 1), logfile: /Users/a-user/repo/tatoflam/concurrent-api-client/.tox/py39/log/py39-2.log

1
2

commands=
    pip-compile requirements.in requirements.testing.in -o requirements.txt -v

1
2
3
4
5

setup(
    setup_requires=[
        'pytest-runner',
    ]
)

1
2

commands=
    pip-compile -i https://pypi.org/simple requirements.in requirements.testing.in -o requirements.txt -v

1
2

[easy_install]
index-url = https://pypi.org/simple

1
2

$ tox -e pip-compile  # this creates requirements.txt from requirements.in and requirements.testing.in. 
$ tox -r

1
2
3

$ tox -r
...
E   FileNotFoundError: [Errno 2] No such file or directory: '/Users/tatsuro.homma/repo/tatoflam/concurrent-api-client/.tox/py39/lib/python3.9/site-packages/api_client/config/logging.json'

1
2
3
4

include requirements.txt
include requirements.testing.txt
include README*
include src/api_client/config/*.json