Add machine parallelism in circle for win (4), mac (4) (wandb#2373)

.circleci/config.yml

@@ -172,6 +172,7 @@ jobs:
             toxenv:
               type: string
         executor: win/default
+        parallelism: 4
         steps:
             - checkout
             - run:
@@ -188,7 +189,7 @@ jobs:
                   name: Run tests
                   shell: bash.exe
                   command: |
-                      tox -v -e << parameters.toxenv >>
+                      CI_PYTEST_PARALLEL=3 CI_PYTEST_SPLIT_ARGS="--splits $CIRCLE_NODE_TOTAL --group $(( $CIRCLE_NODE_INDEX + 1 ))" tox -v -e << parameters.toxenv >>
                   no_output_timeout: 10m
             - save-tox-cache
             - save-test-results
@@ -199,6 +200,7 @@ jobs:
               type: string
         macos:
             xcode: 11.4.1
+        parallelism: 4
         steps:
             - checkout
             - run:
@@ -211,7 +213,7 @@ jobs:
                   # Tests failed with Too many open files, so added ulimit
                   command: |
                       ulimit -n 4096
-                      python3 -m tox -v -e << parameters.toxenv >>
+                      CI_PYTEST_PARALLEL=3 CI_PYTEST_SPLIT_ARGS="--splits $CIRCLE_NODE_TOTAL --group $(( $CIRCLE_NODE_INDEX + 1 ))" python3 -m tox -v -e << parameters.toxenv >>
                   no_output_timeout: 10m
             - save-tox-cache
             - save-test-results

.codecov.yml

@@ -1,7 +1,10 @@
 codecov:
   require_ci_to_pass: no
   notify:
-    after_n_builds: 3
+    # after_n_builds calculation
+    # 6 = 1 (lin-py36) + 4 (win-py37) + 1 (func-lin-py37)
+    # also change comment section
+    after_n_builds: 6
     wait_for_ci: no
 
 ignore:
@@ -50,7 +53,7 @@ comment:
   layout: "reach, diff, flags, files"
   behavior: default
   require_changes: no
-  after_n_builds: 3
+  after_n_builds: 6
 
 github_checks:
   annotations: true

CONTRIBUTING.md

@@ -212,12 +212,19 @@ We currently have 8 categories of test coverage:
 1. project: main coverage numbers, i dont think it can drop by more than a few percent or you will get a failure
 2. patch/tests: must be 100%, if you are writing code for tests, it needs to be executed, if you are planning for the future, comment out your lines
 3. patch/tests-utils: tests/conftest.py and supporting fixtures at tests/utils/, no coverage requirements
-4. patch/sdk: anything that matches wandb/sdk/*.py (so top level sdk files).   These have lots of ways to test, so it should be high coverage.  currently target is ~80% (but it is dynamic)
+4. patch/sdk: anything that matches `wandb/sdk/*.py` (so top level sdk files).   These have lots of ways to test, so it should be high coverage.  currently target is ~80% (but it is dynamic)
 5. patch/sdk-internal: should be covered very high target is around 80% (also dynamic)
 6. patch/sdk-other: will be a catch all for other stuff in wandb/sdk/ target around 75% (dynamic)
 7. patch/apis: we have no good fixtures for this, so until we do, this will get a waiver
 8. patch/other: everything else, we have lots of stuff that isnt easy to test, so it is in this category, currently the requirement is ~60%
 
+### Test parallelism
+
+The circleci uses pytest-split to balance unittest load on multiple nodes.  In order to do this efficiently every once in a while the test timing file (`.test_durations`) needs to be updated with:
+```shell
+CI_PYTEST_SPLIT_ARGS="--store-durations" tox -e py37
+```
+
 ### Regression Testing
 
 TODO(jhr): describe how regression works, how to run them, where they're located etc.

tox.ini

@@ -10,12 +10,16 @@ deps =
     pytest-cov
     pytest-xdist
     pytest-flask
+    pytest-split
     pytest-mock<=3.2.0
     pytest-timeout
     pytest-openfiles
     pytest-flakefinder
 install_command = pip install -f https://download.pytorch.org/whl/torch_stable.html {opts} {packages}
-passenv = USERNAME
+passenv =
+    USERNAME
+    CI_PYTEST_SPLIT_ARGS
+    CI_PYTEST_PARALLEL
 setenv =
     py{27,35,36,37,38,39}: COVERAGE_FILE={envdir}/.coverage
     py{37}: WINDIR=C:\\Windows
@@ -28,7 +32,7 @@ commands_pre =
 commands =
     py{35,36,37,38,39}: ipython kernel install --user --name=wandb_python
     mkdir -p test-results
-    python -m pytest -n=3 --durations=20 --junitxml=test-results/junit.xml --cov-config=.coveragerc --cov --cov-report= --no-cov-on-fail --ignore=wandb/sweeps --ignore=build/ {posargs:tests/ wandb/sweeps/}
+    python -m pytest {env:CI_PYTEST_SPLIT_ARGS:} -n={env:CI_PYTEST_PARALLEL:4} --durations=20 --junitxml=test-results/junit.xml --cov-config=.coveragerc --cov --cov-report= --no-cov-on-fail --ignore=wandb/sweeps --ignore=build/ {posargs:tests/ wandb/sweeps/}
 
 [testenv:dev]
 usedevelop = true