-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathMakefile
141 lines (120 loc) · 4.49 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
.DEFAULT_GOAL := help
###########################
# HELP
###########################
include *.mk
###########################
# VARIABLES
###########################
PROJECTNAME := selfclean
GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD | tr / _)
PROJECT_DIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))/)
COMMA := ,
DASH := -
EMPTY :=
SPACE := $(EMPTY) $(EMPTY)
# check if `netstat` is installed
ifeq (, $(shell which netstat))
$(error "Netstat executable not found, install it with `apt-get install net-tools`")
endif
# Check if Jupyter Port is already use and define an alternative
ifeq ($(origin PORT), undefined)
PORT_USED = $(shell netstat -tl | grep -E '(tcp|tcp6)' | grep -Eo '8888' | tail -n 1)
# Will fail if both ports 9999 and 10000 are used, I am sorry for that
NEXT_TCP_PORT = $(shell netstat -tl | grep -E '(tcp|tcp6)' | grep -Eo '[0-9]{4}' | sort | tail -n 1 | xargs -I '{}' expr {} + 1)
ifeq ($(PORT_USED), 8888)
PORT = $(NEXT_TCP_PORT)
else
PORT = 8888
endif
endif
# docker
ifeq ($(origin CONTAINER_NAME), undefined)
CONTAINER_NAME := default
endif
ifeq ($(origin LOCAL_DATA_DIR), undefined)
LOCAL_DATA_DIR := $$PWD/data/
endif
ifeq ($(origin GPU_ID), undefined)
GPU_ID := all
GPU_NAME := $(GPU_ID)
else
GPU_NAME = $(subst $(COMMA),$(DASH),$(GPU_ID))
endif
ifeq ("$(GPU)", "false")
ifeq (, $(shell which nvidia-smi))
GPU_ARGS := --shm-size 64G
else
GPU_ARGS := --gpus '"device="' --shm-size 64G
endif
DOCKER_CONTAINER_NAME := --name $(PROJECTNAME)_$(CONTAINER_NAME)
else
GPU_ARGS := --gpus '"device=$(GPU_ID)"' --shm-size 64G --ipc=host
DOCKER_CONTAINER_NAME := --name $(PROJECTNAME)_gpu_$(GPU_NAME)_$(CONTAINER_NAME)
endif
# count elements in comma-seperated GPU list
count = $(words $1)$(if $2,$(call count,$(wordlist 2,$(words $1),$1),$2))
GPU_LIST := $(subst $(COMMA),$(SPACE),$(GPU_ID))
NUM_GPUS := $(call count,$(GPU_LIST))
UNAME_S := $(shell uname -s)
ifeq ($(UNAME_S),Linux)
NUM_CORES := $(shell nproc)
else ifeq ($(UNAME_S),Darwin)
NUM_CORES := $(shell sysctl -n hw.ncpu)
else
NUM_CORES := 1
endif
# optimal number of threads is #cores/#gpus
NUM_THREADS := $(shell expr $(NUM_CORES) / $(NUM_GPUS))
DOCKER_ARGS := -v $$PWD:/workspace/ -v $(LOCAL_DATA_DIR):/data/ -p $(PORT):8888 --rm
DOCKER_CMD := docker run $(DOCKER_ARGS) $(GPU_ARGS) $(DOCKER_CONTAINER_NAME) -it $(PROJECTNAME):$(GIT_BRANCH)
###########################
# PROJECT UTILS
###########################
.PHONY: init
init: ##@Utils initializes the project and pulls all the nessecary data
@git submodule update --init --recursive
.PHONY: install
install: ##@Utils install the dependencies for the project
@python3 -m pip install -r requirements.txt
@pre-commit install
.PHONY: update_dependencies
update_dependencies: ##@Utils updates the dependencies for the project in the toml file
@python3.9 -m update_dependencies
.PHONY: clean
clean: ##@Utils clean the project
@black .
@find . -name '*.pyc' -delete
@find . -name '__pycache__' -type d | xargs rm -fr
@rm -f .DS_Store
@rm -f .coverage coverage.xml report.xml
@rm -f -R .pytest_cache
@rm -f -R .idea
@rm -f -R tmp/
@rm -f -R cov_html/
@rm -f -R build/
@rm -f -R dist/
@rm -f -R selfclean.egg-info/
###########################
# DOCKER
###########################
_build:
@echo "Build image $(GIT_BRANCH)..."
@docker build -f Dockerfile -t $(PROJECTNAME):$(GIT_BRANCH) .
run_bash: _build ##@Docker run an interactive bash inside the docker image (default: GPU=true)
@echo "Running bash with GPU being $(GPU) and GPU_ID $(GPU_ID)"
$(DOCKER_CMD) /bin/bash; \
start_jupyter: _build ##@Docker start a jupyter notebook inside the docker image
@echo "Starting jupyter notebook"
@-docker rm $(DOCKER_CONTAINER_NAME)
$(DOCKER_CMD) /bin/bash -c "jupyter notebook --allow-root --ip 0.0.0.0 --port 8888"
.DEFAULT_GOAL := help
###########################
# TESTS
###########################
.PHONY: test
test: _build ##@Test run all tests in the project
$(DOCKER_CMD) /bin/bash -c "python3 -m coverage run -m pytest tests --junitxml=report.xml; coverage report -i --include=selfclean/* --omit="selfclean/ssl_library/*"; coverage xml -i --include=selfclean/* --omit="selfclean/ssl_library/*";"
.PHONY: unittest
unittest: _build ##@Test run all unittests in the project
$(DOCKER_CMD) /bin/bash -c "python3 -m coverage run -m pytest tests --junitxml=report.xml --ignore=tests/integration_tests; coverage report -i --include=selfclean/* --omit="selfclean/ssl_library/*"; coverage xml -i --include=selfclean/* --omit="selfclean/ssl_library/*";"