diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml deleted file mode 100644 index 5294083d0..000000000 --- a/.github/workflows/cmake.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: CMake - -on: [push, pull_request] - -env: - # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.) - BUILD_TYPE: Release - -jobs: - build: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - - name: Install Dependencies - run: sudo apt-get update && sudo apt-get install -yq gfortran libboost-dev libboost-serialization-dev liblapack-dev - - - name: Create Build Environment - shell: bash - run: | - cd cxx - mkdir build - - - name: Configure CMake - shell: bash - working-directory: ${{runner.workspace}}/mspass/cxx/build - # Note the current convention is to use the -S and -B options here to specify source - # and build directories, but this is only available with CMake 3.13 and higher. - # The CMake binaries on the Github Actions machines are (as of this writing) 3.12 - run: cmake $GITHUB_WORKSPACE/cxx -DCMAKE_BUILD_TYPE=$BUILD_TYPE - - - name: Build - working-directory: ${{runner.workspace}}/mspass/cxx/build - shell: bash - # Execute the build. You can specify a specific target with "--target " - run: cmake --build . --config $BUILD_TYPE - - - name: Test - working-directory: ${{runner.workspace}}/mspass/cxx/build - shell: bash - run: ctest --output-on-failure -C $BUILD_TYPE diff --git a/.github/workflows/code-format.yml b/.github/workflows/code-format.yml deleted file mode 100644 index a2b466526..000000000 --- a/.github/workflows/code-format.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: code-format - -on: [pull_request] - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Check files using the black formatter - uses: rickstaa/action-black@v1 - id: action_black - with: - black_args: "." - - name: Create Pull Request - if: steps.action_black.outputs.is_formatted == 'true' - uses: peter-evans/create-pull-request@v3 - with: - token: ${{ secrets.GITHUB_TOKEN }} - title: "Format Python code with psf/black push" - commit-message: ":art: Format Python code with psf/black" - body: | - There appear to be some python formatting errors in ${{ github.sha }}. This pull request - uses the [psf/black](https://github.com/psf/black) formatter to fix these issues. - base: ${{ github.head_ref }} # Creates pull request onto pull request or commit branch - branch: actions/black - #- name: Annotate diff changes using reviewdog - # if: steps.action_black.outputs.is_formatted == 'true' - # uses: reviewdog/action-suggester@v1 - # with: - # tool_name: blackfmt diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml deleted file mode 100644 index f6217dd46..000000000 --- a/.github/workflows/docker-publish.yml +++ /dev/null @@ -1,282 +0,0 @@ -name: Docker - -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. - -on: - schedule: - - cron: '21 21 * * 5' - push: - branches: [ master ] - # Publish semver tags as releases. - tags: [ 'v*.*.*' ] - pull_request: - branches: [ master ] - -env: - # Use docker.io for Docker Hub if empty - REGISTRY: ghcr.io - # github.repository as / - IMAGE_NAME: ${{ github.repository }} - - -jobs: - build-latest: - - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - # Setup for Multi-platform image - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Cache Docker layers - uses: actions/cache@v3 - with: - path: | - /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - # Login against a Docker registry - # https://github.com/docker/login-action - - name: Log into registry ${{ env.REGISTRY }} - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Log in to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - # Extract metadata (tags, labels) for Docker - # https://github.com/docker/metadata-action - - name: Extract Docker metadata - id: meta - uses: docker/metadata-action@v4 - with: - flavor: | - latest=true - images: | - mspass/mspass - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - - # Build and push Docker image with Buildx (don't push on PR) - # https://github.com/docker/build-push-action - - name: Build and push Docker image (amd64 only) - uses: docker/build-push-action@v4 - if: ${{ github.event_name != 'schedule' }} - with: - context: . - platforms: linux/amd64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new - - - name: Build and push Docker image (amd64 and arm64) - uses: docker/build-push-action@v4 - if: ${{ github.event_name == 'schedule' }} - with: - context: . - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new - - # Temp fix - # https://github.com/docker/build-push-action/issues/252 - # https://github.com/moby/buildkit/issues/1896 - - name: Move cache - run: | - rm -rf /tmp/.buildx-cache - mv /tmp/.buildx-cache-new /tmp/.buildx-cache - - build-dev: - - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - # Setup for Multi-platform image - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Cache Docker layers - uses: actions/cache@v3 - with: - path: | - /tmp/.buildx-cache-dev - key: ${{ runner.os }}-buildx.dev-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx.dev- - - # Login against a Docker registry - # https://github.com/docker/login-action - - name: Log into registry ${{ env.REGISTRY }} - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Log in to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - # Extract metadata (tags, labels) for Docker - # https://github.com/docker/metadata-action - - name: Extract Docker metadata - id: meta - uses: docker/metadata-action@v4 - with: - flavor: | - latest=true - images: | - mspass/mspass - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - - # Build and push dev image - - name: Build and push dev image (amd64 only) - uses: docker/build-push-action@v4 - if: ${{ github.event_name != 'schedule' }} - with: - context: . - file: Dockerfile_dev - platforms: linux/amd64 - push: true - tags: | - mspass/mspass:dev - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=local,src=/tmp/.buildx-cache-dev - cache-to: type=local,dest=/tmp/.buildx-cache-dev-new - - - name: Build and push dev image (amd64 amd arm64) - uses: docker/build-push-action@v4 - if: ${{ github.event_name == 'schedule' }} - with: - context: . - file: Dockerfile_dev - platforms: linux/amd64,linux/arm64 - push: true - tags: | - mspass/mspass:dev - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=local,src=/tmp/.buildx-cache-dev - cache-to: type=local,dest=/tmp/.buildx-cache-dev-new - - # Temp fix - # https://github.com/docker/build-push-action/issues/252 - # https://github.com/moby/buildkit/issues/1896 - - name: Move cache - run: | - rm -rf /tmp/.buildx-cache-dev - mv /tmp/.buildx-cache-dev-new /tmp/.buildx-cache-dev - - build-mpi: - - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - # Setup for Multi-platform image - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Cache Docker layers - uses: actions/cache@v3 - with: - path: | - /tmp/.buildx-cache-mpi - key: ${{ runner.os }}-buildx.mpi-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx.mpi- - - # Login against a Docker registry - # https://github.com/docker/login-action - - name: Log into registry ${{ env.REGISTRY }} - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Log in to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - # Extract metadata (tags, labels) for Docker - # https://github.com/docker/metadata-action - - name: Extract Docker metadata - id: meta - uses: docker/metadata-action@v4 - with: - flavor: | - latest=true - images: | - mspass/mspass - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - - # Build and push mpi image - - name: Build and push mpi image (amd64 only) - uses: docker/build-push-action@v4 - with: - context: . - file: Dockerfile_mpi - platforms: linux/amd64 - push: ${{ github.event_name != 'pull_request' }} - tags: | - mspass/mspass:mpi - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:mpi - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=local,src=/tmp/.buildx-cache-mpi - cache-to: type=local,dest=/tmp/.buildx-cache-mpi-new - - # Temp fix - # https://github.com/docker/build-push-action/issues/252 - # https://github.com/moby/buildkit/issues/1896 - - name: Move cache - run: | - rm -rf /tmp/.buildx-cache-mpi - mv /tmp/.buildx-cache-mpi-new /tmp/.buildx-cache-mpi diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml deleted file mode 100644 index e33cadb93..000000000 --- a/.github/workflows/python-package.yml +++ /dev/null @@ -1,80 +0,0 @@ -name: Python package - -on: [push, pull_request] - -jobs: - build: - - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ['3.8', '3.10'] - - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - uses: actions/cache@v2 - with: - path: | - ~/.cache/pip - ${{ env.pythonLocation }}/bin/* - ${{ env.pythonLocation }}/include - ${{ env.pythonLocation }}/lib/python*/site-packages/* - !${{ env.pythonLocation }}/bin/pip* - !${{ env.pythonLocation }}/lib/python*/site-packages/pip* - !${{ env.pythonLocation }}/lib/python*/site-packages/setuptools* - key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ hashFiles('requirements.txt') }} - - name: Install CXX Dependencies - run: sudo apt-get update && sudo apt-get install -yq gfortran libboost-dev libboost-serialization-dev liblapack-dev - - name: Install MongoDB - uses: supercharge/mongodb-github-action@1.3.0 - - name: Install Python Dependencies - run: | - python -m pip install --upgrade pip - python -m pip install --upgrade --upgrade-strategy eager pytest pytest-cov - if [ -f requirements.txt ]; then pip install --upgrade --upgrade-strategy eager -r requirements.txt; fi - - name: Install Apache Spark - run: | - mkdir -p /opt - wget -q -O /opt/spark.tgz https://archive.apache.org/dist/spark/spark-3.0.0/spark-3.0.0-bin-hadoop2.7.tgz - tar xzf /opt/spark.tgz -C /opt/ - rm /opt/spark.tgz - export SPARK_HOME=/opt/spark-3.0.0-bin-hadoop2.7 - echo "SPARK_HOME=/opt/spark-3.0.0-bin-hadoop2.7" >> $GITHUB_ENV - export PATH=$PATH:/opt/spark-3.0.0-bin-hadoop2.7/bin - echo "/opt/spark-3.0.0-bin-hadoop2.7/bin" >> $GITHUB_PATH - python -m pip install --upgrade --upgrade-strategy eager pyspark pytest-spark - - name: Install - run: python -m pip install -C--build-option=build -C--build-option=--debug -v . - - name: Test with pytest - run: | - export MSPASS_HOME=$(pwd) - export SPARK_LOCAL_IP=127.0.0.1 - pytest --cov=mspasspy python/tests/ - make test -C build/temp.*/ - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v1 - with: - env_vars: OS,PYTHON - verbose: true - - name: Build Documentation - if: ${{ matrix.python-version == '3.8' }} - run: | - export MSPASS_HOME=$(pwd) - sudo apt-get update && sudo apt-get install -yq doxygen pandoc - pip install -r docs/requirements.txt - cd docs - ls - make html - - name: Deploy to GitHub Pages - if: ${{ github.ref == 'refs/heads/master' && matrix.python-version == '3.8' }} - uses: JamesIves/github-pages-deploy-action@3.7.1 - with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - BRANCH: gh-pages - FOLDER: docs/build/html - CLEAN: false diff --git a/.gitignore b/.gitignore deleted file mode 100644 index f2730f28c..000000000 --- a/.gitignore +++ /dev/null @@ -1,16 +0,0 @@ -.env -.vscode -cxx/build/ -docs/build/ -docs/source/doxygen -docs/source/mspass_schema/*.csv -**/_build/ -**/__pycache__/ -.idea -misc -.coverage* -build/ -python/mspasspy/data/ -python/mspasspy.egg-info/ -dask-worker-space/ -dist/ diff --git a/Dockerfile b/Dockerfile index fd2c92337..33983c5d9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -172,6 +172,7 @@ RUN pip3 --no-cache-dir install --upgrade setuptools \ && docker-clean # Add cxx library +RUN apt-get update && apt-get install -y libyaml-cpp-dev ADD cxx /mspass/cxx RUN ln -s /opt/conda/include/yaml-cpp /usr/include/yaml-cpp && cd /mspass/cxx \ && mkdir build && cd build \ diff --git a/Dockerfile_dev b/Dockerfile_dev index 69a582615..656317252 100644 --- a/Dockerfile_dev +++ b/Dockerfile_dev @@ -172,6 +172,7 @@ RUN pip3 --no-cache-dir install --upgrade setuptools \ && docker-clean # Add cxx library +RUN apt-get update && apt-get install -y libyaml-cpp-dev ADD cxx /mspass/cxx RUN ln -s /opt/conda/include/yaml-cpp /usr/include/yaml-cpp && cd /mspass/cxx \ && mkdir build && cd build \ diff --git a/Dockerfile_mpi b/Dockerfile_mpi index 8e9f07bfb..8c53b24d1 100644 --- a/Dockerfile_mpi +++ b/Dockerfile_mpi @@ -169,6 +169,7 @@ RUN pip3 --no-cache-dir install --upgrade setuptools \ && docker-clean # Add cxx library +RUN apt-get update && apt-get install -y libyaml-cpp-dev ADD cxx /mspass/cxx RUN ln -s /opt/conda/include/yaml-cpp /usr/include/yaml-cpp && cd /mspass/cxx \ && mkdir build && cd build \ diff --git a/Dockerfile_tapis b/Dockerfile_tapis new file mode 100644 index 000000000..7dd1dc135 --- /dev/null +++ b/Dockerfile_tapis @@ -0,0 +1,231 @@ +#Image: jthet/mspass-tapis +#Version: 1.0.0 + +FROM ghcr.io/seisscoped/container-base:ubuntu22.04_jupyterlab + +LABEL maintainer="Ian Wang , Jackson Thetford " + +# add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added +RUN set -eux; \ + groupadd --gid 999 --system mongodb; \ + useradd --uid 999 --system --gid mongodb --home-dir /data/db mongodb; \ + mkdir -p /data/db /data/configdb; \ + chown -R mongodb:mongodb /data/db /data/configdb \ + && docker-clean + +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + dirmngr \ + gnupg \ + jq \ + numactl \ + procps \ + ; \ + rm -rf /var/lib/apt/lists/* \ + && docker-clean + +# grab "js-yaml" for parsing mongod's YAML config files (https://github.com/nodeca/js-yaml/releases) +ENV JSYAML_VERSION 3.13.1 + +RUN set -ex; \ + \ + savedAptMark="$(apt-mark showmanual)"; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + wget \ + ; \ + rm -rf /var/lib/apt/lists/*; \ + \ + dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')"; \ + export GNUPGHOME="$(mktemp -d)"; \ + gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4; \ + gpgconf --kill all; \ + \ + wget -O /js-yaml.js "https://github.com/nodeca/js-yaml/raw/${JSYAML_VERSION}/dist/js-yaml.js"; \ +# TODO some sort of download verification here + \ + apt-mark auto '.*' > /dev/null; \ + apt-mark manual $savedAptMark > /dev/null; \ + apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \ + && docker-clean + +RUN mkdir /docker-entrypoint-initdb.d + +RUN set -ex; \ + export GNUPGHOME="$(mktemp -d)"; \ + set -- '39BD841E4BE5FB195A65400E6A26B1AE64C3C388'; \ + for key; do \ + gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "$key"; \ + done; \ + mkdir -p /etc/apt/keyrings; \ + gpg --batch --export "$@" > /etc/apt/keyrings/mongodb.gpg; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME" \ + && docker-clean + +# Allow build-time overrides (eg. to build image with MongoDB Enterprise version) +# Options for MONGO_PACKAGE: mongodb-org OR mongodb-enterprise +# Options for MONGO_REPO: repo.mongodb.org OR repo.mongodb.com +# Example: docker build --build-arg MONGO_PACKAGE=mongodb-enterprise --build-arg MONGO_REPO=repo.mongodb.com . +ARG MONGO_PACKAGE=mongodb-org +ARG MONGO_REPO=repo.mongodb.org +ENV MONGO_PACKAGE=${MONGO_PACKAGE} MONGO_REPO=${MONGO_REPO} + +ENV MONGO_MAJOR 6.0 +RUN echo "deb [ signed-by=/etc/apt/keyrings/mongodb.gpg ] http://$MONGO_REPO/apt/ubuntu jammy/${MONGO_PACKAGE%-unstable}/$MONGO_MAJOR multiverse" | tee "/etc/apt/sources.list.d/${MONGO_PACKAGE%-unstable}.list" + +# https://docs.mongodb.org/master/release-notes/6.0/ +ENV MONGO_VERSION 6.0.5 +# 03/08/2023, https://github.com/mongodb/mongo/tree/c9a99c120371d4d4c52cbb15dac34a36ce8d3b1d + +RUN set -x \ +# installing "mongodb-enterprise" pulls in "tzdata" which prompts for input + && export DEBIAN_FRONTEND=noninteractive \ + && apt-get update \ + && apt-get install -y \ + ${MONGO_PACKAGE}=$MONGO_VERSION \ + ${MONGO_PACKAGE}-server=$MONGO_VERSION \ + ${MONGO_PACKAGE}-shell=$MONGO_VERSION \ + ${MONGO_PACKAGE}-mongos=$MONGO_VERSION \ + ${MONGO_PACKAGE}-tools=$MONGO_VERSION \ + && rm -rf /var/lib/apt/lists/* \ + && rm -rf /var/lib/mongodb \ + && mv /etc/mongod.conf /etc/mongod.conf.orig \ + && docker-clean + +VOLUME /data/db /data/configdb + +# ensure that if running as custom user that "mongosh" has a valid "HOME" +# https://github.com/docker-library/mongo/issues/524 +ENV HOME /data/db + +COPY docker-entrypoint.sh /usr/local/bin/ + +EXPOSE 27017 + +RUN apt-get update \ + && apt-get install -y wget ssh rsync vim-tiny less \ + build-essential python3-setuptools \ + python3-dev python3-pip \ + openjdk-8-jdk \ + git cmake gfortran gdb \ + liblapack-dev libboost-dev libboost-serialization-dev libyaml-dev \ + zip unzip \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + && docker-clean + +ARG TARGETARCH + +# Prepare the environment +ARG SPARK_VERSION=3.0.0 + +ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-${TARGETARCH} +ENV SPARK_HOME /usr/local/spark +ENV PYSPARK_PYTHON python3 + +ARG APACHE_MIRROR=https://archive.apache.org/dist +ARG SPARK_URL=${APACHE_MIRROR}/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop2.7.tgz + +# Download & install Spark +RUN wget -qO - ${SPARK_URL} | tar -xz -C /usr/local/ \ + && cd /usr/local && ln -s spark-${SPARK_VERSION}-bin-hadoop2.7 spark \ + && docker-clean +RUN ln -s /usr/local/spark/bin/pyspark /usr/bin/pyspark +RUN python -c "import site; print(site.getsitepackages()[0])" > site_packages_path.txt && \ + PYTHON_SITE_PACKAGES_PATH=$(cat site_packages_path.txt) && \ + ln -s /usr/local/spark/python/pyspark ${PYTHON_SITE_PACKAGES_PATH}/pyspark && \ + unzip /usr/local/spark/python/lib/py4j-0.10.9-src.zip -d ${PYTHON_SITE_PACKAGES_PATH}/ \ + && docker-clean + +# Patch pyspark for machines don't have localhost defined in /etc/hosts +RUN sed -i 's/localhost/127.0.0.1/' /usr/local/spark/python/pyspark/accumulators.py +RUN unzip /usr/local/spark/python/lib/pyspark.zip \ + && sed -i 's/localhost/127.0.0.1/' ./pyspark/accumulators.py \ + && zip /usr/local/spark/python/lib/pyspark.zip pyspark/accumulators.py \ + && rm -r ./pyspark \ + && docker-clean + +# Install Python dependencies through pip +ADD requirements.txt requirements.txt +RUN pip3 --no-cache-dir install --upgrade pip \ + && docker-clean +RUN if [ "$TARGETARCH" == "arm64" ]; then export CFLAGS="-O3" \ + && export DISABLE_NUMCODECS_SSE2=true && export DISABLE_NUMCODECS_AVX2=true; fi\ + && pip3 --no-cache-dir install numpy \ + && pip3 --no-cache-dir install -r requirements.txt \ + && rm -f requirements.txt \ + && docker-clean + +# Download & install pybind11 +ARG PYBIND11_VERSION=2.6.0 +ARG PYBIND11_URL=https://github.com/pybind/pybind11/archive/v${PYBIND11_VERSION}.tar.gz +RUN wget -qO - ${PYBIND11_URL} | tar -xz -C /usr/local/ \ + && cd /usr/local/pybind11-${PYBIND11_VERSION} \ + && mkdir build && cd build && cmake .. -DPYBIND11_TEST=OFF && make install && docker-clean +RUN rm -r /usr/local/pybind11-${PYBIND11_VERSION} + +# Upgrade setuptools to enable namespace package +RUN pip3 --no-cache-dir install --upgrade setuptools \ + && docker-clean + + + +RUN apt-get update && apt-get install -y libyaml-cpp-dev +# Add cxx library +ADD cxx /mspass/cxx +RUN ln -s /opt/conda/include/yaml-cpp /usr/include/yaml-cpp && cd /mspass/cxx \ + && mkdir build && cd build \ + && cmake .. \ + && make \ + && make install \ + && rm -rf ../build \ + && docker-clean + +# Add data and env variable for the MetadataDefinition class +ADD data /mspass/data +ENV MSPASS_HOME /mspass + +# Add setup.py to install python components +ADD setup.py /mspass/setup.py +ADD pyproject.toml /mspass/pyproject.toml +ADD requirements.txt /mspass/requirements.txt +ADD python /mspass/python +RUN pip3 install /mspass -v \ + && rm -rf /mspass/build && docker-clean + +# Install jedi +RUN pip3 --no-cache-dir install jedi==0.17.2 && docker-clean + +# Tini operates as a process subreaper for jupyter. +ARG TINI_VERSION=v0.19.0 +ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-${TARGETARCH} /usr/sbin/tini +RUN chmod +x /usr/sbin/tini + +# Add startup script +ADD scripts/start-mspass-tacc.sh /usr/sbin/start-mspass-tacc.sh +RUN chmod +x /usr/local/bin/docker-entrypoint.sh +RUN sed -i '/set -- mongod "$@"/i [[ -d data ]] || mkdir data' /usr/local/bin/docker-entrypoint.sh + +# replace localhost to 127.0.0.1 in pymongo to run on HPC +RUN python -c "import site; print(site.getsitepackages()[0])" > site_packages_path.txt && \ + PYTHON_SITE_PACKAGES_PATH=$(cat site_packages_path.txt) && \ + sed -i "s/localhost:27020,/127.0.0.1:27020,/g" "${PYTHON_SITE_PACKAGES_PATH}/pymongo/encryption_options.py" && \ + sed -i 's/HOST = "localhost"/HOST = "127.0.0.1"/g' "${PYTHON_SITE_PACKAGES_PATH}/pymongo/mongo_client.py" && \ + sed -i "s/'localhost'/'127.0.0.1'/g" "${PYTHON_SITE_PACKAGES_PATH}/pymongo/settings.py" && \ + sed -i "s/'localhost'/'127.0.0.1'/g" "${PYTHON_SITE_PACKAGES_PATH}/pymongo/pool.py" && \ + rm site_packages_path.txt + +ENV PATH="${SPARK_HOME}/bin:${SPARK_HOME}/sbin:${PATH}" + +# Set the default behavior of this container +ENV SPARK_MASTER_PORT 7077 +ENV DASK_SCHEDULER_PORT 8786 +ENV MONGODB_PORT 27017 +ENV JUPYTER_PORT 8888 +ENV MSPASS_ROLE all + +# ENV MSPASS_SCHEDULER dask +ENTRYPOINT ["/usr/sbin/tini", "-s", "-g", "--", "/usr/sbin/start-mspass-tacc.sh"] diff --git a/cxx/include/misc/base64.h b/cxx/include/misc/base64.h deleted file mode 100644 index 47907cfce..000000000 --- a/cxx/include/misc/base64.h +++ /dev/null @@ -1,130 +0,0 @@ -/* - base64.cpp and base64.h - - Copyright (C) 2004-2008 René Nyffenegger - - This source code is provided 'as-is', without any express or implied - warranty. In no event will the author be held liable for any damages - arising from the use of this software. - - Permission is granted to anyone to use this software for any purpose, - including commercial applications, and to alter it and redistribute it - freely, subject to the following restrictions: - - 1. The origin of this source code must not be misrepresented; you must not - claim that you wrote the original source code. If you use this source code - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original source code. - - 3. This notice may not be removed or altered from any source distribution. - - René Nyffenegger rene.nyffenegger@adp-gmbh.ch - -*/ - -#ifndef __BASE64_H -#define __BASE64_H - -#include - -namespace misc -{ - -static const std::string base64_chars = - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "abcdefghijklmnopqrstuvwxyz" - "0123456789-_"; - - -static inline bool is_base64(unsigned char c) { - return (isalnum(c) || (c == '-') || (c == '_')); -} - -std::string base64_encode(char const* bytes_to_encode, unsigned int in_len) { - std::string ret; - int i = 0; - int j = 0; - unsigned char char_array_3[3]; - unsigned char char_array_4[4]; - - while (in_len--) { - char_array_3[i++] = *(bytes_to_encode++); - if (i == 3) { - char_array_4[0] = (char_array_3[0] & 0xfc) >> 2; - char_array_4[1] = ((char_array_3[0] & 0x03) << 4) + ((char_array_3[1] & 0xf0) >> 4); - char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6); - char_array_4[3] = char_array_3[2] & 0x3f; - - for(i = 0; (i <4) ; i++) - ret += base64_chars[char_array_4[i]]; - i = 0; - } - } - - if (i) - { - for(j = i; j < 3; j++) - char_array_3[j] = '\0'; - - char_array_4[0] = (char_array_3[0] & 0xfc) >> 2; - char_array_4[1] = ((char_array_3[0] & 0x03) << 4) + ((char_array_3[1] & 0xf0) >> 4); - char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6); - char_array_4[3] = char_array_3[2] & 0x3f; - - for (j = 0; (j < i + 1); j++) - ret += base64_chars[char_array_4[j]]; - - while((i++ < 3)) - ret += '.'; - - } - - return ret; - -} -std::string base64_decode(std::string const& encoded_string) { - int in_len = encoded_string.size(); - int i = 0; - int j = 0; - int in_ = 0; - unsigned char char_array_4[4], char_array_3[3]; - std::string ret; - - while (in_len-- && ( encoded_string[in_] != '.') && is_base64(encoded_string[in_])) { - char_array_4[i++] = encoded_string[in_]; in_++; - if (i ==4) { - for (i = 0; i <4; i++) - char_array_4[i] = base64_chars.find(char_array_4[i]); - - char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4); - char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2); - char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3]; - - for (i = 0; (i < 3); i++) - ret += char_array_3[i]; - i = 0; - } - } - - if (i) { - for (j = i; j <4; j++) - char_array_4[j] = 0; - - for (j = 0; j <4; j++) - char_array_4[j] = base64_chars.find(char_array_4[j]); - - char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4); - char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2); - char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3]; - - for (j = 0; (j < i - 1); j++) ret += char_array_3[j]; - } - - return ret; -} -} //End of namespace misc - -#endif \ No newline at end of file diff --git a/cxx/include/misc/blas.h b/cxx/include/misc/blas.h deleted file mode 100644 index 442442e64..000000000 --- a/cxx/include/misc/blas.h +++ /dev/null @@ -1,30 +0,0 @@ -#ifndef __BLAS_H -#define __BLAS_H - -#include "FC.h" - -extern "C" { -double ddot(const int& n,const double *x,const int& incx, - const double *y,const int& incy); -double dscal(const int& n, const double& a, const double *x, const int& incx); -double daxpy(const int &n, const double& a, const double *x,const int& incx, - const double *y,const int& incy); -double dcopy(const int &n, const double *x,const int& incx, - const double *y,const int& incy); -double dnrm2(const int &n, const double *x,const int& incx); -void dgetrf(int&,int&,double*,int&,int*,int&); -void dgetri(int&,double*,int&,int*,double*,int&,int&); -double dlamch(const char *cmach); -int dstebz(char *range, char *order, int *n, double - *vl, double *vu, int *il, int *iu, double *abstol, - double *d__, double *e, int *m, int *nsplit, - double *w, int *iblock, int *isplit, double *work, - int *iwork, int *info); -int dstein(int *n, double *d__, double *e, - int *m, double *w, int *iblock, int *isplit, - double *z__, int *ldz, double *work, int *iwork, - int *ifail, int *info); - -}; - -#endif diff --git a/scripts/start-mspass-tacc.sh b/scripts/start-mspass-tacc.sh new file mode 100755 index 000000000..244281ee2 --- /dev/null +++ b/scripts/start-mspass-tacc.sh @@ -0,0 +1,348 @@ +#!/bin/bash + +# If running with docker use /home, else use pwd to store all data and logs +if grep "docker/containers" /proc/self/mountinfo -qa; then + MSPASS_WORKDIR=/home +elif [[ -z ${MSPASS_WORK_DIR} ]]; then + MSPASS_WORKDIR=`pwd` +else + MSPASS_WORKDIR=$MSPASS_WORK_DIR +fi + +if [ "$1" != "--batch" ]; then + ## Creating reverse tunnel port to login nodes + NODE_HOSTNAME=`hostname -s` > /dev/null 2>&1 + LOGIN_PORT=`echo $NODE_HOSTNAME | perl -ne 'print (($2+1).$3.$1) if /c\d(\d\d)-(\d)(\d\d)/;'` > /dev/null 2>&1 + STATUS_PORT=$(($LOGIN_PORT + 1)) + echo "got login node port $LOGIN_PORT" + echo "got status node port $STATUS_PORT" + # create reverse tunnel port to login nodes. Make one tunnel for each login so the user can just + # connect to stampede.tacc + for i in `seq 4`; do + ssh -q -f -g -N -R $LOGIN_PORT:$NODE_HOSTNAME:8888 login$i > /dev/null 2>&1 + ssh -q -f -g -N -R $STATUS_PORT:$NODE_HOSTNAME:8787 login$i > /dev/null 2>&1 + done + # creating notebook password, giving link + MSPASS_JUPYTER_PWD=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 20 | head -n 1) + echo "Created reverse ports on Stampede2 logins" + printf '%*s\n' "${COLUMNS:-$(tput cols)}" '' | tr ' ' - + echo "Welcome to Tapis interactive" | sed -e :a -e "s/^.\{1,$(tput cols)\}$/ & /;ta" | tr -d '\n' | head -c $(tput cols); echo -e "\n" + echo "The Link below will open a MsPASS Jupyter Notebook" | sed -e :a -e "s/^.\{1,$(tput cols)\}$/ & /;ta" | tr -d '\n' | head -c $(tput cols);echo -e "\n" + echo "http://frontera.tacc.utexas.edu:$LOGIN_PORT/lab?token=$MSPASS_JUPYTER_PWD"| sed -e :a -e "s/^.\{1,$(tput cols)\}$/ & /;ta" | tr -d '\n' | head -c $(tput cols);echo -e "\n" + printf '%*s\n' "${COLUMNS:-$(tput cols)}" '' | tr ' ' - + + # echo "Notebook Token:" $MSPASS_JUPYTER_PWD +fi + +# define SLEEP_TIME +if [[ -z $MSPASS_SLEEP_TIME ]]; then + MSPASS_SLEEP_TIME=15 +fi + +# This sets defaults for this set of env variables +if [[ -z ${MSPASS_DB_DIR} ]]; then + MSPASS_DB_DIR=${MSPASS_WORKDIR}/db +fi + +if [[ -z ${MSPASS_LOG_DIR} ]]; then + MSPASS_LOG_DIR=${MSPASS_WORKDIR}/logs +fi +if [[ -z ${MSPASS_WORKER_DIR} ]]; then + MSPASS_WORKER_DIR=${MSPASS_WORKDIR}/work +fi +# Note that only log is required for all roles. Other dirs will be created later when needed. +[[ -d $MSPASS_LOG_DIR ]] || mkdir -p $MSPASS_LOG_DIR + +MONGO_DATA=${MSPASS_DB_DIR}/data +MONGO_LOG=${MSPASS_LOG_DIR}/mongo_log +export SPARK_WORKER_DIR=${MSPASS_WORKER_DIR} +export SPARK_LOG_DIR=${MSPASS_LOG_DIR} + +if [ $# -eq 0 ] || [ $1 = "--batch" ]; then + + function start_mspass_frontend { + BATCH_MODE_ARGS="--to notebook --inplace --execute $1" + NOTEBOOK_ARGS="--notebook-dir=${MSPASS_WORKDIR} --port=${JUPYTER_PORT} --no-browser --ip=0.0.0.0 --allow-root" + # if MSPASS_JUPYTER_PWD is not set, notebook will generate a default token + # we rely on jupyter's python function to hash the password + MSPASS_JUPYTER_PWD_HASHED=$(python3 -c "from notebook.auth import passwd; print(passwd('${MSPASS_JUPYTER_PWD}'))") > nb_stdout.txt 2>&1 + NOTEBOOK_ARGS="${NOTEBOOK_ARGS} --NotebookApp.token=${MSPASS_JUPYTER_PWD}" + + if [ "$MSPASS_SCHEDULER" = "spark" ]; then + export PYSPARK_DRIVER_PYTHON=jupyter + if [ -z $1 ]; then + export PYSPARK_DRIVER_PYTHON_OPTS="lab ${NOTEBOOK_ARGS}" > nb_stdout.txt 2>&1 + else + export PYSPARK_DRIVER_PYTHON_OPTS="nbconvert ${BATCH_MODE_ARGS}" > nb_stdout.txt 2>&1 + fi + pyspark \ + --conf "spark.mongodb.input.uri=mongodb://${MSPASS_DB_ADDRESS}:${MONGODB_PORT}/test.misc" \ + --conf "spark.mongodb.output.uri=mongodb://${MSPASS_DB_ADDRESS}:${MONGODB_PORT}/test.misc" \ + --conf "spark.master=spark://${MSPASS_SCHEDULER_ADDRESS}:${SPARK_MASTER_PORT}" \ + --packages org.mongodb.spark:mongo-spark-connector_2.12:3.0.0 > nb_stdout.txt 2>&1 + else # if [ "$MSPASS_SCHEDULER" = "dask" ] + export DASK_SCHEDULER_ADDRESS=${MSPASS_SCHEDULER_ADDRESS}:${DASK_SCHEDULER_PORT} + if [ -z $1 ]; then + jupyter lab ${NOTEBOOK_ARGS} > nb_stdout.txt 2>&1 + else + jupyter nbconvert ${BATCH_MODE_ARGS}> nb_stdout.txt 2>&1 + fi + fi + } + + function clean_up_single_node { + # ---------------------- clean up workflow ------------------------- + # stop mongodb + mongosh --port $MONGODB_PORT admin --eval "db.shutdownServer({force:true})" + sleep 5 + # copy shard data to scratch + if [ "$MSPASS_DB_PATH" = "tmp" ]; then + echo "standalone: copy shard data to scratch" + # copy data + scp -r /tmp/db/data ${MSPASS_DB_DIR} + # copy log + scp -r /tmp/logs/mongo_log ${MSPASS_LOG_DIR} + fi + sleep ${MSPASS_SLEEP_TIME} + } + + function clean_up_multiple_nodes { + # ---------------------- clean up workflow ------------------------- + # stop mongos routers + mongosh --port $MONGODB_PORT admin --eval "db.shutdownServer({force:true})" + sleep 5 + # stop each shard replica set + for i in ${MSPASS_SHARD_ADDRESS[@]}; do + ssh -o "StrictHostKeyChecking no" ${i} "kill -2 \$(pgrep mongo)" + sleep 5 + done + # stop config servers + mongosh --port $(($MONGODB_PORT+1)) admin --eval "db.shutdownServer({force:true})" + + # copy the shard data to scratch if the shards are deployed in /tmp + if [ "$MSPASS_DB_PATH" = "tmp" ]; then + echo "distributed: copy shard data to scratch" + # copy data + for i in ${MSPASS_SHARD_DB_PATH[@]}; do + scp -r -o StrictHostKeyChecking=no ${i} ${MSPASS_DB_DIR} + done + # copy log + for i in ${MSPASS_SHARD_LOGS_PATH[@]}; do + scp -r -o StrictHostKeyChecking=no ${i} ${MSPASS_LOG_DIR} + done + fi + sleep ${MSPASS_SLEEP_TIME} + } + + function start_db_scratch { + [[ -d $MONGO_DATA ]] || mkdir -p $MONGO_DATA + mongod --port $MONGODB_PORT --dbpath $MONGO_DATA --logpath $MONGO_LOG --bind_ip_all & + } + + function start_db_tmp { + # create db and log dirs if not exists + [[ -d /tmp/db ]] || mkdir -p /tmp/db + [[ -d /tmp/logs ]] || mkdir -p /tmp/logs && touch /tmp/logs/mongo_log + # copy all data on scratch to the local tmp folder + if [[ -d ${MSPASS_DB_DIR}/data ]]; then + cp -r ${MSPASS_DB_DIR}/data /tmp/db + else + mkdir -p /tmp/db/data + fi + # copy dfiles to /tmp + if [[ -d $MSPASS_SCRATCH_DATA_DIR ]]; then + cp -r $MSPASS_SCRATCH_DATA_DIR /tmp + fi + # start mongodb on /tmp + mongod --port $MONGODB_PORT --dbpath /tmp/db/data --logpath /tmp/logs/mongo_log --bind_ip_all & + } + + mongod --port $MONGODB_PORT --dbpath $MONGO_DATA --logpath $MONGO_LOG --bind_ip_all & + + MY_ID=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 12 | head -n 1) + if [ "$MSPASS_SCHEDULER" = "spark" ]; then + MSPASS_SCHEDULER_CMD='$SPARK_HOME/sbin/start-master.sh' + MSPASS_WORKER_CMD='$SPARK_HOME/sbin/start-slave.sh spark://$MSPASS_SCHEDULER_ADDRESS:$SPARK_MASTER_PORT' + else # if [ "$MSPASS_SCHEDULER" = "dask" ] + MSPASS_SCHEDULER_CMD='dask-scheduler --port $DASK_SCHEDULER_PORT > ${MSPASS_LOG_DIR}/dask-scheduler_log_${MY_ID} 2>&1 & sleep 5' + MSPASS_WORKER_CMD='dask-worker ${MSPASS_WORKER_ARG} --local-directory $MSPASS_WORKER_DIR tcp://$MSPASS_SCHEDULER_ADDRESS:$DASK_SCHEDULER_PORT > ${MSPASS_LOG_DIR}/dask-worker_log_${MY_ID} 2>&1 &' + fi + + if [ "$MSPASS_ROLE" = "db" ]; then + if [ "$MSPASS_DB_PATH" = "tmp" ]; then + start_db_tmp + else + start_db_scratch + fi + elif [ "$MSPASS_ROLE" = "dbmanager" ]; then + # config server configuration + MONGODB_CONFIG_PORT=$(($MONGODB_PORT+1)) + if [ -d ${MONGO_DATA}_config ]; then + echo "restore config server $HOSTNAME cluster" + # start a mongod instance + mongod --port $MONGODB_CONFIG_PORT --dbpath ${MONGO_DATA}_config --logpath ${MONGO_LOG}_config --bind_ip_all & + sleep ${MSPASS_SLEEP_TIME} + # drop the local database + echo "drop local database for config server $HOSTNAME" + mongosh --port $MONGODB_CONFIG_PORT local --eval "db.dropDatabase()" + sleep ${MSPASS_SLEEP_TIME} + # update config.shards collections + echo "update shard host names for config server $HOSTNAME" + # if using ${!MSPASS_SHARD_LIST[@]} style for loop, it doesn't work. Not sure why it doesn't work. + ITER=0 + for i in ${MSPASS_SHARD_LIST[@]}; do + echo "update rs${ITER} with host ${i}" + mongosh --port $MONGODB_CONFIG_PORT config --eval "db.shards.updateOne({\"_id\": \"rs${ITER}\"}, {\$set: {\"host\": \"${i}\"}})" + ((ITER++)) + sleep ${MSPASS_SLEEP_TIME} + done + echo "restart the config server $HOSTNAME as a replica set" + # restart the mongod as a new single-node replica set + mongosh --port $MONGODB_CONFIG_PORT admin --eval "db.shutdownServer()" + sleep ${MSPASS_SLEEP_TIME} + mongod --port $MONGODB_CONFIG_PORT --configsvr --replSet configserver --dbpath ${MONGO_DATA}_config --logpath ${MONGO_LOG}_config --bind_ip_all & + sleep ${MSPASS_SLEEP_TIME} + # initiate the new replica set + mongosh --port $MONGODB_CONFIG_PORT --eval \ + "rs.initiate({_id: \"configserver\", configsvr: true, version: 1, members: [{ _id: 0, host : \"$HOSTNAME:$MONGODB_CONFIG_PORT\" }]})" + sleep ${MSPASS_SLEEP_TIME} + + # start a mongos router server + mongos --port $MONGODB_PORT --configdb configserver/$HOSTNAME:$MONGODB_CONFIG_PORT --logpath ${MONGO_LOG}_router --bind_ip_all & + sleep ${MSPASS_SLEEP_TIME} + else + # create a config dir + mkdir -p ${MONGO_DATA}_config + echo "dbmanager config server $HOSTNAME replicaSet is initialized" + # start a config server + mongod --port $MONGODB_CONFIG_PORT --configsvr --replSet configserver --dbpath ${MONGO_DATA}_config --logpath ${MONGO_LOG}_config --bind_ip_all & + sleep ${MSPASS_SLEEP_TIME} + mongosh --port $MONGODB_CONFIG_PORT --eval \ + "rs.initiate({_id: \"configserver\", configsvr: true, version: 1, members: [{ _id: 0, host : \"$HOSTNAME:$MONGODB_CONFIG_PORT\" }]})" + sleep ${MSPASS_SLEEP_TIME} + + # start a mongos router server + mongos --port $MONGODB_PORT --configdb configserver/$HOSTNAME:$MONGODB_CONFIG_PORT --logpath ${MONGO_LOG}_router --bind_ip_all & + # add shard clusters + for i in ${MSPASS_SHARD_LIST[@]}; do + echo "add shard with host ${i}" + sleep ${MSPASS_SLEEP_TIME} + mongosh --host $HOSTNAME --port $MONGODB_PORT --eval "sh.addShard(\"${i}\")" + done + fi + + # enable database sharding + echo "enable database $MSPASS_SHARD_DATABASE sharding" + mongosh --host $HOSTNAME --port $MONGODB_PORT --eval "sh.enableSharding(\"${MSPASS_SHARD_DATABASE}\")" + sleep ${MSPASS_SLEEP_TIME} + # shard collection(using hashed) + for i in ${MSPASS_SHARD_COLLECTIONS[@]}; do + echo "shard collection $MSPASS_SHARD_DATABASE.${i%%:*} and shard key is ${i##*:}" + mongosh --host $HOSTNAME --port $MONGODB_PORT --eval "sh.shardCollection(\"$MSPASS_SHARD_DATABASE.${i%%:*}\", {${i##*:}: \"hashed\"})" + sleep ${MSPASS_SLEEP_TIME} + done + tail -f /dev/null + elif [ "$MSPASS_ROLE" = "shard" ]; then + [[ -n $MSPASS_SHARD_ID ]] || MSPASS_SHARD_ID=$MY_ID + # Note that we have to create a one-member replica set here + # because certain pymongo API will use "retryWrites=true" + # and thus trigger an error. + if [ "$MSPASS_DB_PATH" = "tmp" ]; then + echo "store shard data in tmp for shard server $HOSTNAME" + # create db and log dirs if not exists + [[ -d /tmp/db ]] || mkdir -p /tmp/db + [[ -d /tmp/logs ]] || mkdir -p /tmp/logs && touch /tmp/logs/mongo_log_shard_${MSPASS_SHARD_ID} + # copy all the shard data to the local tmp folder + if [[ -d ${MSPASS_DB_DIR}/data_shard_${MSPASS_SHARD_ID} ]]; then + scp -r -o StrictHostKeyChecking=no ${MSPASS_DB_DIR}/data_shard_${MSPASS_SHARD_ID} /tmp/db + else + mkdir -p /tmp/db/data_shard_${MSPASS_SHARD_ID} + fi + # reconfig the shard replica set + if [ -d ${MONGO_DATA}_shard_${MSPASS_SHARD_ID} ]; then + # restore the shard replica set + mongod --port $MONGODB_PORT --dbpath /tmp/db/data_shard_${MSPASS_SHARD_ID} --logpath /tmp/logs/mongo_log_shard_${MSPASS_SHARD_ID} --bind_ip_all & + sleep ${MSPASS_SLEEP_TIME} + # drop local database + echo "drop local database for shard server $HOSTNAME" + mongosh --port $MONGODB_PORT local --eval "db.dropDatabase()" + sleep ${MSPASS_SLEEP_TIME} + # update shard metadata in each shard's identity document + echo "update config server host names for shard server $HOSTNAME" + mongosh --port $MONGODB_PORT admin --eval "db.system.version.updateOne({\"_id\": \"shardIdentity\"}, {\$set: {\"configsvrConnectionString\": \"${MSPASS_CONFIG_SERVER_ADDR}\"}})" + sleep ${MSPASS_SLEEP_TIME} + # restart the mongod as a new single-node replica set + echo "restart the shard server $HOSTNAME as a replica set" + mongosh --port $MONGODB_PORT admin --eval "db.shutdownServer()" + sleep ${MSPASS_SLEEP_TIME} + mongod --port $MONGODB_PORT --shardsvr --replSet "rs${MSPASS_SHARD_ID}" --dbpath /tmp/db/data_shard_${MSPASS_SHARD_ID} --logpath /tmp/logs/mongo_log_shard_${MSPASS_SHARD_ID} --bind_ip_all & + else + # store the shard data in the /tmp folder in local machine + mongod --port $MONGODB_PORT --shardsvr --replSet "rs${MSPASS_SHARD_ID}" --dbpath /tmp/db/data_shard_${MSPASS_SHARD_ID} --logpath /tmp/logs/mongo_log_shard_${MSPASS_SHARD_ID} --bind_ip_all & + fi + else + echo "store shard data in scratch for shard server $HOSTNAME" + if [ -d ${MONGO_DATA}_shard_${MSPASS_SHARD_ID} ]; then + # restore the shard replica set + mongod --port $MONGODB_PORT --dbpath ${MONGO_DATA}_shard_${MSPASS_SHARD_ID} --logpath ${MONGO_LOG}_shard_${MSPASS_SHARD_ID} --bind_ip_all & + sleep ${MSPASS_SLEEP_TIME} + # drop local database + echo "drop local database for shard server $HOSTNAME" + mongosh --port $MONGODB_PORT local --eval "db.dropDatabase()" + sleep ${MSPASS_SLEEP_TIME} + # update shard metadata in each shard's identity document + echo "update config server host names for shard server $HOSTNAME" + mongosh --port $MONGODB_PORT admin --eval "db.system.version.updateOne({\"_id\": \"shardIdentity\"}, {\$set: {\"configsvrConnectionString\": \"${MSPASS_CONFIG_SERVER_ADDR}\"}})" + sleep ${MSPASS_SLEEP_TIME} + # restart the mongod as a new single-node replica set + echo "restart the shard server $HOSTNAME as a replica set" + mongosh --port $MONGODB_PORT admin --eval "db.shutdownServer()" + sleep ${MSPASS_SLEEP_TIME} + mongod --port $MONGODB_PORT --shardsvr --replSet "rs${MSPASS_SHARD_ID}" --dbpath ${MONGO_DATA}_shard_${MSPASS_SHARD_ID} --logpath ${MONGO_LOG}_shard_${MSPASS_SHARD_ID} --bind_ip_all & + else + # initialize the shard replica set + mkdir -p ${MONGO_DATA}_shard_${MSPASS_SHARD_ID} + mongod --port $MONGODB_PORT --shardsvr --replSet "rs${MSPASS_SHARD_ID}" --dbpath ${MONGO_DATA}_shard_${MSPASS_SHARD_ID} --logpath ${MONGO_LOG}_shard_${MSPASS_SHARD_ID} --bind_ip_all & + fi + fi + sleep ${MSPASS_SLEEP_TIME} + + # shard server configuration + echo "shard server $HOSTNAME replicaSet is initialized" + mongosh --port $MONGODB_PORT --eval \ + "rs.initiate({_id: \"rs${MSPASS_SHARD_ID}\", version: 1, members: [{ _id: 0, host : \"$HOSTNAME:$MONGODB_PORT\" }]})" + tail -f /dev/null + elif [ "$MSPASS_ROLE" = "scheduler" ]; then + eval $MSPASS_SCHEDULER_CMD + tail -f /dev/null + elif [ "$MSPASS_ROLE" = "worker" ]; then + [[ -d $MSPASS_WORKER_DIR ]] || mkdir -p $MSPASS_WORKER_DIR + eval $MSPASS_WORKER_CMD + # copy dfiles to /tmp + if [[ -d $MSPASS_SCRATCH_DATA_DIR ]]; then + cp -r $MSPASS_SCRATCH_DATA_DIR /tmp + fi + tail -f /dev/null + elif [ "$MSPASS_ROLE" = "frontend" ]; then + start_mspass_frontend $2 + if [ "$MSPASS_DB_MODE" = "shard" ]; then + clean_up_multiple_nodes + else + clean_up_single_node + fi + else # if [ "$MSPASS_ROLE" = "all" ] + MSPASS_DB_ADDRESS=$HOSTNAME + MSPASS_SCHEDULER_ADDRESS=$HOSTNAME + eval $MSPASS_SCHEDULER_CMD + eval $MSPASS_WORKER_CMD + if [ "$MSPASS_DB_PATH" = "tmp" ]; then + start_db_tmp + else + start_db_scratch + fi + start_mspass_frontend $2 + clean_up_single_node + fi +else + docker-entrypoint.sh $@ +fi