add dockerfiles

Dockerfile

@@ -0,0 +1,13 @@
+FROM ubuntu:22.04
+
+RUN apt-get update && apt-get install -y python3 python3-pip
+
+COPY . /app
+
+WORKDIR /app
+
+RUN test -f /app/generations.json && rm /app/generations.json || true
+
+RUN pip3 install .
+
+CMD ["python3", "main.py"]

Dockerfile-multiple

@@ -0,0 +1,56 @@
+FROM ubuntu:22.04
+RUN apt-get update -yqq && apt-get install -yqq curl build-essential python3-pip python3-tqdm
+RUN apt-get install racket -yqq
+ARG DEBIAN_FRONTEND=noninteractive
+ENV TZ=Etc/UTC
+RUN apt-get install -yqq \
+    default-jdk-headless \
+    golang-go \
+    php-cli \
+    ruby \
+    lua5.3 \
+    r-base \
+    rustc \
+    scala
+
+RUN apt-get install -yqq libtest-deep-perl 
+RUN apt-get install -yqq wget 
+
+# JS/TS
+RUN curl -fsSL https://deb.nodesource.com/setup_current.x | bash - 
+RUN apt-get install -y nodejs
+RUN npm install -g typescript
+
+# Dlang
+RUN wget https://netcologne.dl.sourceforge.net/project/d-apt/files/d-apt.list -O /etc/apt/sources.list.d/d-apt.list
+RUN apt-get update --allow-insecure-repositories
+RUN apt-get -y --allow-unauthenticated install --reinstall d-apt-keyring
+RUN apt-get update && apt-get install -yqq dmd-compiler dub
+
+# C#
+RUN apt install gnupg ca-certificates
+RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
+RUN echo "deb https://download.mono-project.com/repo/ubuntu stable-focal main" | tee /etc/apt/sources.list.d/mono-official-stable.list
+RUN apt update
+RUN apt install -yqq mono-devel
+
+# Post-processing
+
+# Julia
+RUN curl https://julialang-s3.julialang.org/bin/linux/x64/1.8/julia-1.8.2-linux-x86_64.tar.gz | tar xz
+ENV PATH="/julia-1.8.2/bin:${PATH}"
+# Swift
+RUN curl https://download.swift.org/swift-5.7-release/ubuntu2204/swift-5.7-RELEASE/swift-5.7-RELEASE-ubuntu22.04.tar.gz | tar xz
+ENV PATH="/swift-5.7-RELEASE-ubuntu22.04/usr/bin:${PATH}"
+# Javatuples
+RUN mkdir /usr/multiple && wget https://repo.mavenlibs.com/maven/org/javatuples/javatuples/1.2/javatuples-1.2.jar -O /usr/multiple/javatuples-1.2.jar
+# Luaunit
+RUN apt-get update -yqq && apt-get install -yqq lua-unit
+
+# Standard requirements
+COPY . /app
+WORKDIR /app
+RUN test -f /app/generations.json && rm /app/generations.json || true
+
+RUN pip3 install .
+CMD ["python3", "main.py"]

makefile

@@ -0,0 +1,20 @@
+# There are two dockerfiles: for all benchmarks, and for MultiPL-E
+DOCKERFILE=Dockerfile
+
+ifeq ($(DOCKERFILE), Dockerfile)
+	IMAGE_NAME=evaluation-harness
+else
+	IMAGE_NAME=evaluation-harness-multiple
+endif
+
+build:
+	docker build -f $(DOCKERFILE) -t $(IMAGE_NAME) .
+
+test:
+	docker run -v $(CURDIR)/tests/docker_test/test_generations.json:/app/test_generations.json:ro \
+	-it $(IMAGE_NAME) python3 main.py --model dummy_model --tasks humaneval --limit 4 \
+	--generations_path /app/test_generations.json --allow_code_execution 
+
+	@echo "If pass@1 is 0.25 then your configuration for standard benchmarks is correct"
+
+all: build test