Postgresql ansible (#182)

* Create postgresql.yaml

* Update postgresql.yaml

* Create postgresqlconf.yaml

* Add postgresql ansible setup POC

* Make the ansible script work for remote server

* Ansible tweaks

* Ansible works on remote VM

* split the cli entry point

* Add reverse proxy utility to test database

* Publish to db worked

* Fixx cleo issues

* update argument names

* Add missing constraint file

* Fix unit tests

* fix reporting command line

* Fix failing test

---------

Co-authored-by: Pierre Delaunay <[email protected]>

.gitignore

@@ -24,3 +24,5 @@ sqlite.db
 trash/
 workspace/
 slurm-*
+scripts/inventory.yaml
+out.txt

.pin/constraints-cuda-torch.txt

@@ -0,0 +1,379 @@
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    pip-compile --config=pyproject.toml --output-file=.pin/constraints-cuda-torch.txt --resolver=backtracking .pin/tmp-constraints.txt benchmarks/accelerate_opt/requirements.in benchmarks/dlrm/requirements.in benchmarks/flops/requirements.in benchmarks/huggingface/requirements.in benchmarks/rwkv/requirements.in benchmarks/stargan/requirements.in benchmarks/super-slomo/requirements.in benchmarks/timm/requirements.in benchmarks/torchvision/requirements.in
+#
+--extra-index-url https://download.pytorch.org/whl/cu118
+
+absl-py==2.0.0
+    # via tensorboard
+accelerate==0.24.1
+    # via -r benchmarks/accelerate_opt/requirements.in
+aiohttp==3.8.6
+    # via
+    #   datasets
+    #   fsspec
+aiosignal==1.3.1
+    # via aiohttp
+antlr4-python3-runtime==4.9.3
+    # via omegaconf
+asttokens==2.4.1
+    # via giving
+async-timeout==4.0.3
+    # via aiohttp
+attrs==23.1.0
+    # via aiohttp
+cachetools==5.3.2
+    # via google-auth
+certifi==2023.7.22
+    # via requests
+charset-normalizer==3.3.2
+    # via
+    #   aiohttp
+    #   requests
+codefind==0.1.3
+    # via ptera
+datasets==2.14.6
+    # via
+    #   -r benchmarks/accelerate_opt/requirements.in
+    #   evaluate
+deepspeed==0.11.1
+    # via
+    #   -r benchmarks/accelerate_opt/requirements.in
+    #   -r benchmarks/rwkv/requirements.in
+dill==0.3.7
+    # via
+    #   datasets
+    #   evaluate
+    #   multiprocess
+docker==6.1.3
+    # via torchx
+docstring-parser==0.15
+    # via torchx
+evaluate==0.4.1
+    # via -r benchmarks/accelerate_opt/requirements.in
+executing==1.2.0
+    # via varname
+fbgemm-gpu==0.5.0+cu118
+    # via torchrec
+filelock==3.13.1
+    # via
+    #   huggingface-hub
+    #   torch
+    #   torchx
+    #   transformers
+    #   triton
+frozenlist==1.4.0
+    # via
+    #   aiohttp
+    #   aiosignal
+fsspec[http]==2023.1.0
+    # via
+    #   datasets
+    #   evaluate
+    #   huggingface-hub
+    #   pytorch-lightning
+    #   torch
+    #   torchx
+future==0.18.3
+    # via -r benchmarks/dlrm/requirements.in
+giving==0.4.2
+    # via
+    #   ptera
+    #   voir
+google-auth==2.23.4
+    # via
+    #   google-auth-oauthlib
+    #   tensorboard
+google-auth-oauthlib==1.1.0
+    # via tensorboard
+graphviz==0.20.1
+    # via torchviz
+grpcio==1.59.2
+    # via tensorboard
+hjson==3.1.0
+    # via deepspeed
+huggingface-hub==0.17.3
+    # via
+    #   -r benchmarks/timm/requirements.in
+    #   accelerate
+    #   datasets
+    #   evaluate
+    #   tokenizers
+    #   transformers
+idna==3.4
+    # via
+    #   requests
+    #   yarl
+importlib-metadata==6.8.0
+    # via torchx
+jinja2==3.1.2
+    # via torch
+joblib==1.3.2
+    # via scikit-learn
+lightning-utilities==0.9.0
+    # via
+    #   pytorch-lightning
+    #   torchmetrics
+markdown==3.5.1
+    # via tensorboard
+markdown-it-py==3.0.0
+    # via rich
+markupsafe==2.1.3
+    # via
+    #   jinja2
+    #   werkzeug
+mdurl==0.1.2
+    # via markdown-it-py
+mpmath==1.3.0
+    # via sympy
+multidict==6.0.4
+    # via
+    #   aiohttp
+    #   yarl
+multiprocess==0.70.15
+    # via
+    #   datasets
+    #   evaluate
+mypy-extensions==1.0.0
+    # via typing-inspect
+networkx==3.2.1
+    # via torch
+ninja==1.11.1.1
+    # via
+    #   -r benchmarks/rwkv/requirements.in
+    #   deepspeed
+numpy==1.26.1
+    # via
+    #   -r benchmarks/stargan/requirements.in
+    #   -r benchmarks/super-slomo/requirements.in
+    #   accelerate
+    #   datasets
+    #   deepspeed
+    #   evaluate
+    #   onnx
+    #   opencv-python
+    #   pandas
+    #   pyarrow
+    #   pytorch-lightning
+    #   scikit-learn
+    #   scipy
+    #   tensorboard
+    #   torchmetrics
+    #   torchvision
+    #   transformers
+oauthlib==3.2.2
+    # via requests-oauthlib
+omegaconf==2.3.0
+    # via voir
+onnx==1.15.0
+    # via -r benchmarks/dlrm/requirements.in
+opencv-python==4.8.1.78
+    # via -r benchmarks/super-slomo/requirements.in
+ovld==0.3.2
+    # via voir
+packaging==23.2
+    # via
+    #   accelerate
+    #   datasets
+    #   deepspeed
+    #   docker
+    #   evaluate
+    #   huggingface-hub
+    #   lightning-utilities
+    #   pytorch-lightning
+    #   torchmetrics
+    #   transformers
+pandas==2.1.2
+    # via
+    #   datasets
+    #   evaluate
+pillow==10.1.0
+    # via torchvision
+protobuf==4.23.4
+    # via
+    #   onnx
+    #   tensorboard
+psutil==5.9.6
+    # via
+    #   accelerate
+    #   deepspeed
+ptera==1.4.1
+    # via voir
+py-cpuinfo==9.0.0
+    # via deepspeed
+pyarrow==14.0.0
+    # via datasets
+pyasn1==0.5.0
+    # via
+    #   pyasn1-modules
+    #   rsa
+pyasn1-modules==0.3.0
+    # via google-auth
+pydantic==1.10.13
+    # via
+    #   -r benchmarks/rwkv/requirements.in
+    #   deepspeed
+pydot==1.4.2
+    # via -r benchmarks/dlrm/requirements.in
+pygments==2.16.1
+    # via rich
+pynvml==11.5.0
+    # via voir
+pyparsing==3.1.1
+    # via pydot
+pyre-extensions==0.0.30
+    # via torchx
+python-dateutil==2.8.2
+    # via pandas
+pytorch-lightning==1.9.5
+    # via -r benchmarks/rwkv/requirements.in
+pytz==2023.3.post1
+    # via pandas
+pyyaml==6.0.1
+    # via
+    #   -r benchmarks/timm/requirements.in
+    #   accelerate
+    #   datasets
+    #   huggingface-hub
+    #   omegaconf
+    #   pytorch-lightning
+    #   torchx
+    #   transformers
+reactivex==4.0.4
+    # via giving
+regex==2023.10.3
+    # via transformers
+requests==2.31.0
+    # via
+    #   datasets
+    #   docker
+    #   evaluate
+    #   fsspec
+    #   huggingface-hub
+    #   requests-oauthlib
+    #   responses
+    #   tensorboard
+    #   torchvision
+    #   transformers
+requests-oauthlib==1.3.1
+    # via google-auth-oauthlib
+responses==0.18.0
+    # via evaluate
+rich==13.6.0
+    # via
+    #   -r benchmarks/accelerate_opt/requirements.in
+    #   voir
+rsa==4.9
+    # via google-auth
+safetensors==0.4.0
+    # via
+    #   -r benchmarks/timm/requirements.in
+    #   transformers
+scikit-learn==1.3.2
+    # via -r benchmarks/dlrm/requirements.in
+scipy==1.11.3
+    # via scikit-learn
+six==1.16.0
+    # via
+    #   asttokens
+    #   python-dateutil
+    #   tensorboard
+sympy==1.12
+    # via torch
+tabulate==0.9.0
+    # via torchx
+tensorboard==2.15.0
+    # via -r benchmarks/dlrm/requirements.in
+tensorboard-data-server==0.7.2
+    # via tensorboard
+threadpoolctl==3.2.0
+    # via scikit-learn
+tokenizers==0.14.1
+    # via transformers
+torch==2.1.0+cu118
+    # via
+    #   -r benchmarks/stargan/requirements.in
+    #   -r benchmarks/super-slomo/requirements.in
+    #   accelerate
+    #   deepspeed
+    #   pytorch-lightning
+    #   torchaudio
+    #   torchmetrics
+    #   torchvision
+    #   torchviz
+torchaudio==2.1.0+cu118
+    # via -r benchmarks/accelerate_opt/requirements.in
+torchmetrics==1.0.3
+    # via
+    #   pytorch-lightning
+    #   torchrec
+torchrec==0.5.0+cu118
+    # via -r benchmarks/dlrm/requirements.in
+torchvision==0.16.0+cu118
+    # via
+    #   -r benchmarks/stargan/requirements.in
+    #   -r benchmarks/super-slomo/requirements.in
+torchviz==0.0.2
+    # via -r benchmarks/dlrm/requirements.in
+torchx==0.6.0
+    # via -r benchmarks/dlrm/requirements.in
+tqdm==4.66.1
+    # via
+    #   -r benchmarks/super-slomo/requirements.in
+    #   -r benchmarks/torchvision/requirements.in
+    #   datasets
+    #   deepspeed
+    #   evaluate
+    #   huggingface-hub
+    #   pytorch-lightning
+    #   torchrec
+    #   transformers
+transformers==4.34.1
+    # via
+    #   -r benchmarks/accelerate_opt/requirements.in
+    #   -r benchmarks/huggingface/requirements.in
+triton==2.1.0
+    # via torch
+typing-extensions==4.8.0
+    # via
+    #   huggingface-hub
+    #   lightning-utilities
+    #   pydantic
+    #   pyre-extensions
+    #   pytorch-lightning
+    #   reactivex
+    #   torch
+    #   typing-inspect
+typing-inspect==0.9.0
+    # via pyre-extensions
+tzdata==2023.3
+    # via pandas
+urllib3==1.26.18
+    # via
+    #   docker
+    #   requests
+    #   responses
+    #   torchx
+varname==0.10.0
+    # via giving
+voir==0.2.11
+    # via
+    #   -r benchmarks/stargan/requirements.in
+    #   -r benchmarks/super-slomo/requirements.in
+websocket-client==1.6.4
+    # via docker
+werkzeug==3.0.1
+    # via tensorboard
+xxhash==3.4.1
+    # via
+    #   datasets
+    #   evaluate
+yarl==1.9.2
+    # via aiohttp
+zipp==3.17.0
+    # via importlib-metadata
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools

benchmarks/llama/benchfile.py

@@ -1,4 +1,3 @@
-
 from milabench.commands import CmdCommand
 from milabench.pack import Package
 

benchmarks/stargan/stargan/synth.py

@@ -1,5 +1,3 @@
-
-
 class SyntheticData:
     def __init__(self, generators, n, repeat):
         self.n = n