Merge pull request #167 from andromeda/web-index-input

Add inputs for web index
binpash · Mar 18, 2021 · 58ffe04 · 58ffe04
2 parents fce117a + 76121f3
commit 58ffe04
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 0 deletions.
diff --git a/evaluation/scripts/web-index/setup.sh b/evaluation/scripts/web-index/setup.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+mkdir input
+wget https://dumps.wikimedia.org/other/static_html_dumps/current/en/wikipedia-en-html.tar.7z
+7za x wikipedia-en-html.tar.7z
+tar -xvf wikipedia-en-html.tar
+wget http://ndr.md/data/wikipedia/index.txt
diff --git a/scripts/install.sh b/scripts/install.sh
@@ -101,6 +101,7 @@ python3 -m pip install -U PyYAML &> $LOG_DIR/pip_install_pyyaml.log
 python3 -m pip install numpy &> $LOG_DIR/pip_install_numpy.log
 python3 -m pip install matplotlib &> $LOG_DIR/pip_install_matplotlib.log
 
+sudo apt-get install -y p7zip-full
 echo "Installing web-index dependencies..."
 # pandoc v.2.2.1
 wget https://github.com/jgm/pandoc/releases/download/2.2.1/pandoc-2.2.1-1-$(dpkg --print-architecture).deb