From 62f3a3cfacf86ca5f0bb06ec2e2d29fd5a677d48 Mon Sep 17 00:00:00 2001 From: Aaron Brethorst Date: Mon, 16 Sep 2024 10:12:39 -0700 Subject: [PATCH] Add support for PostgreSQL Fixes #50 * Add golang to the builder build stage * Add a simple handlebars template renderer app; compile it in the build stage and transfer over to the main image * Rebuild the bootstrap.sh script to use a simple HBS renderer built in Golang * Replace all use of xmlstarlet with handlebars * Add an optional PG database to the docker-compose.yml file --- README.md | 9 +- docker-compose.prod.yml | 1 + docker-compose.standalone.yml | 1 + docker-compose.yml | 20 ++ oba/Dockerfile | 63 ++++-- oba/bootstrap.sh | 196 ++++++------------ oba/config/{context.xml => context.xml.hbs} | 8 +- oba/config/googlemaps.config.json.hbs | 9 + .../onebusaway-api-webapp-data-sources.xml | 95 --------- ...onebusaway-api-webapp-data-sources.xml.hbs | 91 ++++++++ ...terprise-acta-webapp-data-sources.xml.hbs} | 13 +- ...ta-federation-webapp-data-sources.xml.hbs} | 31 ++- oba/config/pom.xml | 5 + oba/config/template_renderer/go.mod | 10 + oba/config/template_renderer/go.sum | 20 ++ oba/config/template_renderer/main.go | 71 +++++++ oba/config/template_renderer/main_test.go | 81 ++++++++ oba/retrieve_maven_artifacts.sh | 4 + render.yaml | 4 +- 19 files changed, 473 insertions(+), 259 deletions(-) rename oba/config/{context.xml => context.xml.hbs} (89%) create mode 100644 oba/config/googlemaps.config.json.hbs delete mode 100644 oba/config/onebusaway-api-webapp-data-sources.xml create mode 100644 oba/config/onebusaway-api-webapp-data-sources.xml.hbs rename oba/config/{onebusaway-enterprise-acta-webapp-data-sources.xml => onebusaway-enterprise-acta-webapp-data-sources.xml.hbs} (90%) rename oba/config/{onebusaway-transit-data-federation-webapp-data-sources.xml => onebusaway-transit-data-federation-webapp-data-sources.xml.hbs} (65%) create mode 100644 oba/config/template_renderer/go.mod create mode 100644 oba/config/template_renderer/go.sum create mode 100644 oba/config/template_renderer/main.go create mode 100644 oba/config/template_renderer/main_test.go diff --git a/README.md b/README.md index e7eedd0..8d1868c 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ When done using this web server, you can use the shell-standard `^C` to exit out ### Inspecting the database -The MySQL database Docker Compose service should remain up after a call of `docker compose up oba_app`. Otherwise, you can always invoke it using `docker compose up oba_database`. +The Docker Compose database service should remain up after a call of `docker compose up oba_app`. Otherwise, you can always invoke it using `docker compose up oba_database`. A database port is open to your host machine, so you can connect to it programmatically using `mysql`: @@ -92,9 +92,10 @@ You can find the latest published Docker images on Docker Hub: ### Deployment Parameters * Database - * `JDBC_URL` - The JDBC connection URL for your MySQL database. - * `JDBC_USER` - The username for your MySQL database. - * `JDBC_PASSWORD` - The password for your MySQL database. + * `JDBC_URL` - The JDBC connection URL for your MySQL or PostgreSQL database. + * `JDBC_DRIVER` - The JDBC driver class name: `com.mysql.cj.jdbc.Driver` or `org.postgresql.Driver` + * `JDBC_USER` - The username for your database. + * `JDBC_PASSWORD` - The password for your database. * GTFS (Optional, required only when using `oba_app` independently) * `GTFS_URL` - The URL to the GTFS feed you want to use. * GTFS-RT Support (Optional) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 4fb04fe..e3d5846 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -40,6 +40,7 @@ services: - PAT_TOKEN_FOR_GH=${PAT_TOKEN_FOR_GH} environment: - JDBC_URL=jdbc:mysql://oba_database:3306/oba_database + - JDBC_DRIVER=com.mysql.cj.jdbc.Driver - JDBC_USER=oba_user - JDBC_PASSWORD=oba_password - TEST_API_KEY diff --git a/docker-compose.standalone.yml b/docker-compose.standalone.yml index 2707716..ceee018 100644 --- a/docker-compose.standalone.yml +++ b/docker-compose.standalone.yml @@ -26,6 +26,7 @@ services: - PAT_TOKEN_FOR_GH=${PAT_TOKEN_FOR_GH} environment: - JDBC_URL=jdbc:mysql://oba_database:3306/oba_database + - JDBC_DRIVER=com.mysql.cj.jdbc.Driver - JDBC_USER=oba_user - JDBC_PASSWORD=oba_password - TEST_API_KEY=test # For test only, remove in production diff --git a/docker-compose.yml b/docker-compose.yml index c10937b..43c7091 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,10 +26,26 @@ services: target: /var/lib/mysql restart: always + oba_database_pg: + image: postgres:15 + container_name: oba_database_pg + environment: + POSTGRES_USER: oba_user + POSTGRES_PASSWORD: oba_password + POSTGRES_DB: oba_database + ports: + - "5432:5432" + volumes: + - type: volume + source: pg-data + target: /var/lib/postgresql/data + restart: always + oba_app: container_name: oba_app depends_on: - oba_database + # - oba_database_pg build: context: ./oba args: @@ -37,6 +53,9 @@ services: - PAT_TOKEN_FOR_GH=${PAT_TOKEN_FOR_GH} environment: - JDBC_URL=jdbc:mysql://oba_database:3306/oba_database + - JDBC_DRIVER=com.mysql.cj.jdbc.Driver + # - JDBC_URL=jdbc:postgresql://oba_database_pg:5432/oba_database + # - JDBC_DRIVER=org.postgresql.Driver - JDBC_USER=oba_user - JDBC_PASSWORD=oba_password - TEST_API_KEY=test # For test only, remove in production @@ -54,3 +73,4 @@ services: volumes: mysql-data: + pg-data: \ No newline at end of file diff --git a/oba/Dockerfile b/oba/Dockerfile index fc3e317..d5e96fc 100644 --- a/oba/Dockerfile +++ b/oba/Dockerfile @@ -9,10 +9,20 @@ ENV OBA_VERSION=${OBA_VERSION} ARG MYSQL_CONNECTOR_VERSION=8.4.0 ENV MYSQL_CONNECTOR_VERSION=${MYSQL_CONNECTOR_VERSION} + +ARG POSTGRESQL_CONNECTOR_VERSION=42.7.4 +ENV POSTGRESQL_CONNECTOR_VERSION=${POSTGRESQL_CONNECTOR_VERSION} + ARG PAT_USERNAME_FOR_GH ARG PAT_TOKEN_FOR_GH -RUN apt-get update && apt-get install -y maven +RUN apt-get update && \ + apt-get install -y maven golang + +# Build the template renderer, which is called hbs_renderer +WORKDIR /oba/template_renderer +COPY ./config/template_renderer . +RUN go build -o hbs_renderer # Start configuring OBA WORKDIR /oba/libs @@ -50,13 +60,14 @@ RUN apt-get update && apt-get install -y \ supervisor \ tzdata \ unzip \ - xmlstarlet \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* RUN pip install supervisord-dependent-startup RUN apt remove -y python3-pip +COPY --from=builder /oba/template_renderer/hbs_renderer /usr/local/bin/hbs_renderer + # Set the configured time zone RUN ln -fs /usr/share/zoneinfo/$TZ /etc/localtime && dpkg-reconfigure -f noninteractive tzdata @@ -76,6 +87,18 @@ COPY --from=builder \ --chown=oba_user:oba_group \ /oba/libs/onebusaway-transit-data-federation-builder-withAllDependencies.jar . +########## +# Copy over config files +########## + +WORKDIR /oba/config + +COPY ./config/context.xml.hbs . +COPY ./config/googlemaps.config.json.hbs . +COPY ./config/onebusaway-api-webapp-data-sources.xml.hbs . +COPY ./config/onebusaway-enterprise-acta-webapp-data-sources.xml.hbs . +COPY ./config/onebusaway-transit-data-federation-webapp-data-sources.xml.hbs . + ########## # Configure OBA Server ########## @@ -89,15 +112,22 @@ WORKDIR $CATALINA_HOME/webapps COPY --from=builder \ --chown=oba_user:oba_group \ /oba/libs/onebusaway-api-webapp.war . + RUN mkdir onebusaway-api-webapp && \ cd onebusaway-api-webapp && \ jar xvf ../onebusaway-api-webapp.war && \ rm ../onebusaway-api-webapp.war + COPY --from=builder \ --chown=oba_user:oba_group \ /oba/libs/mysql-connector-j.jar onebusaway-api-webapp/WEB-INF/lib/ -COPY ./config/onebusaway-api-webapp-data-sources.xml \ - $CATALINA_HOME/webapps/onebusaway-api-webapp/WEB-INF/classes/data-sources.xml.bak + +COPY --from=builder \ + --chown=oba_user:oba_group \ + /oba/libs/postgresql.jar onebusaway-api-webapp/WEB-INF/lib/ + +COPY ./config/onebusaway-api-webapp-data-sources.xml.hbs \ + $CATALINA_HOME/webapps/onebusaway-api-webapp/WEB-INF/classes/data-sources.xml.hbs ########## # Configure onebusaway-enterprise-acta-webapp as ROOT (i.e. the path `/`) @@ -117,8 +147,13 @@ COPY --from=builder \ /oba/libs/mysql-connector-j.jar \ $CATALINA_HOME/webapps/ROOT/WEB-INF/lib/ -COPY ./config/onebusaway-enterprise-acta-webapp-data-sources.xml \ - $CATALINA_HOME/webapps/ROOT/WEB-INF/classes/data-sources.xml.bak +COPY --from=builder \ + --chown=oba_user:oba_group \ + /oba/libs/postgresql.jar \ + $CATALINA_HOME/webapps/ROOT/WEB-INF/lib/ + +COPY ./config/onebusaway-enterprise-acta-webapp-data-sources.xml.hbs \ + $CATALINA_HOME/webapps/ROOT/WEB-INF/classes/data-sources.xml.hbs # TODO: when/where/why is this needed? @@ -133,23 +168,24 @@ RUN chmod 755 /opt/oba/logs COPY --from=builder \ --chown=oba_user:oba_group \ /oba/libs/onebusaway-transit-data-federation-webapp.war . + RUN mkdir onebusaway-transit-data-federation-webapp && \ cd onebusaway-transit-data-federation-webapp && \ jar xvf ../onebusaway-transit-data-federation-webapp.war && \ rm ../onebusaway-transit-data-federation-webapp.war + COPY --from=builder \ --chown=oba_user:oba_group \ /oba/libs/mysql-connector-j.jar \ onebusaway-transit-data-federation-webapp/WEB-INF/lib/ -COPY ./config/onebusaway-transit-data-federation-webapp-data-sources.xml \ - $CATALINA_HOME/webapps/onebusaway-transit-data-federation-webapp/WEB-INF/classes/data-sources.xml.bak -########## -# Tomcat Configuration -########## +COPY --from=builder \ + --chown=oba_user:oba_group \ + /oba/libs/postgresql.jar \ + onebusaway-transit-data-federation-webapp/WEB-INF/lib/ -WORKDIR $CATALINA_HOME -COPY --chown=oba_user:oba_group ./config/context.xml ./conf/ +COPY ./config/onebusaway-transit-data-federation-webapp-data-sources.xml.hbs \ + $CATALINA_HOME/webapps/onebusaway-transit-data-federation-webapp/WEB-INF/classes/data-sources.xml.hbs ########## # Clean up @@ -166,4 +202,5 @@ COPY bootstrap.sh /oba/bootstrap.sh COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf RUN chmod +x /oba/bootstrap.sh +# CMD ["tail", "-f", "/dev/null"] CMD ["supervisord", "-n"] diff --git a/oba/bootstrap.sh b/oba/bootstrap.sh index 1bca1d9..16ee1ba 100644 --- a/oba/bootstrap.sh +++ b/oba/bootstrap.sh @@ -21,149 +21,79 @@ if [ -n "$USER_CONFIGURED" ]; then exit 0 fi -WEBAPP_API_XML_FILE="$CATALINA_HOME/webapps/onebusaway-api-webapp/WEB-INF/classes/data-sources.xml" -NAMESPACE_PREFIX="x" -NAMESPACE_URI="http://www.springframework.org/schema/beans" -BEAN_ID="testAPIKey" -# Idempotence -cp "$WEBAPP_API_XML_FILE.bak" "$WEBAPP_API_XML_FILE" -# Check if the TEST_API_KEY environment variable is set -if [ -n "$TEST_API_KEY" ]; then - # If it is set, then add the API key to the data-sources.xml file - echo "TEST_API_KEY set to $TEST_API_KEY, setting API key in data-sources.xml" - xmlstarlet ed -L -N ${NAMESPACE_PREFIX}=${NAMESPACE_URI} \ - -s "//${NAMESPACE_PREFIX}:bean[@id='${BEAN_ID}']" -t elem -n "property" -v "" \ - -i "//${NAMESPACE_PREFIX}:bean[@id='${BEAN_ID}']/property[not(@name)]" -t attr -n "name" -v "key" \ - -i "//${NAMESPACE_PREFIX}:bean[@id='${BEAN_ID}']/property[@name='key']" -t attr -n "value" -v "${TEST_API_KEY}" \ - ${WEBAPP_API_XML_FILE} -else - # If it is not set, then remove the element from the data-sources.xml file - echo "TEST_API_KEY environment variable is not set. Removing element from data-sources.xml" - xmlstarlet ed -L -N ${NAMESPACE_PREFIX}=${NAMESPACE_URI} \ - -d "//${NAMESPACE_PREFIX}:bean[@id='${BEAN_ID}']" \ - ${WEBAPP_API_XML_FILE} -fi +##### +# onebusaway-api-webapp +##### -DATA_FEDERATION_XML_FILE="$CATALINA_HOME/webapps/onebusaway-transit-data-federation-webapp/WEB-INF/classes/data-sources.xml" -DATA_SOURCE_CLASS="org.onebusaway.transit_data_federation.impl.realtime.gtfs_realtime.GtfsRealtimeSource" -BEAN_ID="gtfsRT" -# Idempotence -cp "$DATA_FEDERATION_XML_FILE.bak" "$DATA_FEDERATION_XML_FILE" -# Check if GTFS-Rt related environment variables are set -if [ -z "$TRIP_UPDATES_URL" ] && [ -z "$VEHICLE_POSITIONS_URL" ] && [ -z "$ALERTS_URL" ]; then - echo "No GTFS-RT related environment variables are set. Removing element from data-sources.xml" - xmlstarlet ed -L -N ${NAMESPACE_PREFIX}=${NAMESPACE_URI} \ - -d "//${NAMESPACE_PREFIX}:bean[@class=${DATA_SOURCE_CLASS}]" \ - ${DATA_FEDERATION_XML_FILE} +API_XML_SOURCE="/oba/config/onebusaway-api-webapp-data-sources.xml.hbs" +API_XML_DESTINATION="$CATALINA_HOME/webapps/onebusaway-api-webapp/WEB-INF/classes/data-sources.xml" + +hbs_renderer -input "$API_XML_SOURCE" \ + -json '{"TEST_API_KEY": "'$TEST_API_KEY'", "JDBC_DRIVER": "'$JDBC_DRIVER'"}' \ + -output "$API_XML_DESTINATION" + +##### +# onebusaway-transit-data-federation-webapp +##### + +FEDERATION_XML_SOURCE="/oba/config/onebusaway-transit-data-federation-webapp-data-sources.xml.hbs" +FEDERATION_XML_DESTINATION="$CATALINA_HOME/webapps/onebusaway-transit-data-federation-webapp/WEB-INF/classes/data-sources.xml" +if [ -z "$TRIP_UPDATES_URL" ] && [ -z "$VEHICLE_POSITIONS_URL" ]; then + GTFS_RT_AVAILABLE="" + echo "No GTFS-RT related environment variables are set. Removing element from data-sources.xml" else + GTFS_RT_AVAILABLE="1" echo "GTFS-RT related environment variables are set. Setting them in data-sources.xml" - if [ -n "$TRIP_UPDATES_URL" ]; then - echo "TRIP_UPDATES_URL set to $TRIP_UPDATES_URL, setting trip updates URL in data-sources.xml" - xmlstarlet ed -L -N ${NAMESPACE_PREFIX}=${NAMESPACE_URI} \ - -s "//${NAMESPACE_PREFIX}:bean[@id='${BEAN_ID}']" -t elem -n "property" -v "" \ - -i "//${NAMESPACE_PREFIX}:bean[@id='${BEAN_ID}']/property[not(@name)]" -t attr -n "name" -v "tripUpdatesUrl" \ - -i "//${NAMESPACE_PREFIX}:bean[@id='${BEAN_ID}']/property[@name='tripUpdatesUrl']" -t attr -n "value" -v "${TRIP_UPDATES_URL}" \ - ${DATA_FEDERATION_XML_FILE} - fi - if [ -n "$VEHICLE_POSITIONS_URL" ]; then - echo "VEHICLE_POSITIONS_URL set to $VEHICLE_POSITIONS_URL, setting vehicle positions URL in data-sources.xml" - xmlstarlet ed -L -N ${NAMESPACE_PREFIX}=${NAMESPACE_URI} \ - -s "//${NAMESPACE_PREFIX}:bean[@id='${BEAN_ID}']" -t elem -n "property" -v "" \ - -i "//${NAMESPACE_PREFIX}:bean[@id='${BEAN_ID}']/property[not(@name)]" -t attr -n "name" -v "vehiclePositionsUrl" \ - -i "//${NAMESPACE_PREFIX}:bean[@id='${BEAN_ID}']/property[@name='vehiclePositionsUrl']" -t attr -n "value" -v "${VEHICLE_POSITIONS_URL}" \ - ${DATA_FEDERATION_XML_FILE} - fi - if [ -n "$ALERTS_URL" ]; then - echo "ALERTS_URL set to $ALERTS_URL, setting alerts URL in data-sources.xml" - xmlstarlet ed -L -N ${NAMESPACE_PREFIX}=${NAMESPACE_URI} \ - -s "//${NAMESPACE_PREFIX}:bean[@id='${BEAN_ID}']" -t elem -n "property" -v "" \ - -i "//${NAMESPACE_PREFIX}:bean[@id='${BEAN_ID}']/property[not(@name)]" -t attr -n "name" -v "alertsUrl" \ - -i "//${NAMESPACE_PREFIX}:bean[@id='${BEAN_ID}']/property[@name='alertsUrl']" -t attr -n "value" -v "${ALERTS_URL}" \ - ${DATA_FEDERATION_XML_FILE} - fi - if [ -n "$REFRESH_INTERVAL" ]; then - echo "REFRESH_INTERVAL set to $REFRESH_INTERVAL, setting refresh interval in data-sources.xml" - xmlstarlet ed -L -N ${NAMESPACE_PREFIX}=${NAMESPACE_URI} \ - -s "//${NAMESPACE_PREFIX}:bean[@id='${BEAN_ID}']" -t elem -n "property" -v "" \ - -i "//${NAMESPACE_PREFIX}:bean[@id='${BEAN_ID}']/property[not(@name)]" -t attr -n "name" -v "refreshInterval" \ - -i "//${NAMESPACE_PREFIX}:bean[@id='${BEAN_ID}']/property[@name='refreshInterval']" -t attr -n "value" -v "${REFRESH_INTERVAL}" \ - ${DATA_FEDERATION_XML_FILE} - fi - if [ -n "$AGENCY_ID" ]; then - echo "AGENCY_ID set to $AGENCY_ID, setting agency ID in data-sources.xml" - xmlstarlet ed -L -N ${NAMESPACE_PREFIX}=${NAMESPACE_URI} \ - -s "//${NAMESPACE_PREFIX}:bean[@id='${BEAN_ID}']" -t elem -n "property" -v "" \ - -i "//${NAMESPACE_PREFIX}:bean[@id='${BEAN_ID}']/property[not(@name)]" -t attr -n "name" -v "agencyId" \ - -i "//${NAMESPACE_PREFIX}:bean[@id='${BEAN_ID}']/property[@name='agencyId']" -t attr -n "value" -v "${AGENCY_ID}" \ - ${DATA_FEDERATION_XML_FILE} - fi - # Check if the GTFS_RT authentication header is set - if [ -n "$FEED_API_KEY" ] && [ -n "$FEED_API_VALUE" ]; then - echo "FEED_API_KEY and FEED_API_VALUE set to $FEED_API_KEY and $FEED_API_VALUE, setting auth header in data-sources.xml" - xmlstarlet ed -L -N ${NAMESPACE_PREFIX}=${NAMESPACE_URI} \ - -s "//${NAMESPACE_PREFIX}:bean[@id='${BEAN_ID}']" -t elem -n "property" -v "" \ - -i "//${NAMESPACE_PREFIX}:bean[@id='${BEAN_ID}']/property[not(@name)]" -t attr -n "name" -v "headersMap" \ - -s "//${NAMESPACE_PREFIX}:bean[@id='${BEAN_ID}']/property[@name='headersMap']" -t elem -n "map" -v "" \ - -s "//${NAMESPACE_PREFIX}:bean[@id='${BEAN_ID}']/property[@name='headersMap']/map" -t elem -n "entry" -v "" \ - -i "//${NAMESPACE_PREFIX}:bean[@id='${BEAN_ID}']/property[@name='headersMap']/map/entry" -t attr -n "key" -v "${FEED_API_KEY}" \ - -i "//${NAMESPACE_PREFIX}:bean[@id='${BEAN_ID}']/property[@name='headersMap']/map/entry" -t attr -n "value" -v "${FEED_API_VALUE}" \ - ${DATA_FEDERATION_XML_FILE} - fi fi +# Check if the GTFS_RT authentication header is set +if [ -n "$FEED_API_KEY" ] && [ -n "$FEED_API_VALUE" ]; then + HAS_API_KEY="1" +else + HAS_API_KEY="" +fi + +hbs_renderer -input "$FEDERATION_XML_SOURCE" \ + -json '{"GTFS_RT_AVAILABLE": "'$GTFS_RT_AVAILABLE'", "TRIP_UPDATES_URL": "'$TRIP_UPDATES_URL'", "VEHICLE_POSITIONS_URL": "'$VEHICLE_POSITIONS_URL'", "ALERTS_URL": "'$ALERTS_URL'", "REFRESH_INTERVAL": "'$REFRESH_INTERVAL'", "AGENCY_ID": "'$AGENCY_ID'", "HAS_API_KEY": "'$HAS_API_KEY'", "FEED_API_KEY": "'$FEED_API_KEY'", "FEED_API_VALUE": "'$FEED_API_VALUE'"}' \ + -output "$FEDERATION_XML_DESTINATION" + +##### +# onebusaway-enterprise-acta-webapp +##### + # Google map related environment variables -ENTERPRISE_ACTA_WEBAPP_XML_FILE="$CATALINA_HOME/webapps/ROOT/WEB-INF/classes/data-sources.xml" -BEAN_ID="configurationServiceClient" -LOCAL_JSON_FILE="/var/lib/oba/config.json" -# Idempotence -cp "$ENTERPRISE_ACTA_WEBAPP_XML_FILE.bak" "$ENTERPRISE_ACTA_WEBAPP_XML_FILE" -mkdir -p $(dirname "$LOCAL_JSON_FILE") +WEBAPP_XML_SOURCE="/oba/config/onebusaway-enterprise-acta-webapp-data-sources.xml.hbs" +WEBAPP_XML_DESTINATION="$CATALINA_HOME/webapps/ROOT/WEB-INF/classes/data-sources.xml" + +mkdir -p $(dirname "$MAP_CONFIG_FILE") + if [ -z "$GOOGLE_MAPS_API_KEY" ] && [ -z "$GOOGLE_MAPS_CLIENT_ID" ] && [ -z "$GOOGLE_MAPS_CHANNEL_ID" ]; then echo "No Google Maps related environment variables are set. Removing element from data-sources.xml" - xmlstarlet ed -L -N ${NAMESPACE_PREFIX}=${NAMESPACE_URI} \ - -d "//${NAMESPACE_PREFIX}:bean[@id=${BEAN_ID}]" \ - ${ENTERPRISE_ACTA_WEBAPP_XML_FILE} + GOOGLE_MAPS_CONFIGURED="" else echo "Google Maps related environment variables are set. Setting them in data-sources.xml" - # Indempotence - rm -f "$LOCAL_JSON_FILE" - touch "$LOCAL_JSON_FILE" - echo '{"config":[]}' > "$LOCAL_JSON_FILE" - xmlstarlet ed -L -N ${NAMESPACE_PREFIX}=${NAMESPACE_URI} \ - -s "//${NAMESPACE_PREFIX}:bean[@id='${BEAN_ID}']" -t elem -n "constructor-arg" -v "" \ - -i "//${NAMESPACE_PREFIX}:bean[@id='${BEAN_ID}']/constructor-arg[not(@type)]" -t attr -n "type" -v "java.lang.String" \ - -i "//${NAMESPACE_PREFIX}:bean[@id='${BEAN_ID}']/constructor-arg[@type='java.lang.String']" -t attr -n "value" -v "/var/lib/oba/config.json" \ - ${ENTERPRISE_ACTA_WEBAPP_XML_FILE} - - # Avoid read and write same file in one pipe to avoid race condition - TMP_JSON_FILE="./tmp.json" - touch "$TMP_JSON_FILE" - if [ -n "$GOOGLE_MAPS_API_KEY" ]; then - cat "$LOCAL_JSON_FILE" | jq '.config += [{"component": "display", "key": "display.googleMapsApiKey", "value": "'"$GOOGLE_MAPS_API_KEY"'"}]' > "$TMP_JSON_FILE" - mv "$TMP_JSON_FILE" "$LOCAL_JSON_FILE" - fi - if [ -n "$GOOGLE_MAPS_CLIENT_ID" ]; then - cat "$LOCAL_JSON_FILE" | jq '.config += [{"component": "display", "key": "display.googleMapsClientId", "value": "'"$GOOGLE_MAPS_CLIENT_ID"'"}]' > "$TMP_JSON_FILE" - mv "$TMP_JSON_FILE" "$LOCAL_JSON_FILE" - fi - if [ -n "$GOOGLE_MAPS_CHANNEL_ID" ]; then - cat "$LOCAL_JSON_FILE" | jq '.config += [{"component": "display", "key": "display.googleMapsChannelId", "value": "'"$GOOGLE_MAPS_CHANNEL_ID"'"}]' > "$TMP_JSON_FILE" - mv "$TMP_JSON_FILE" "$LOCAL_JSON_FILE" - fi -fi + GOOGLE_MAPS_CONFIGURED="1" -CONTEXT_XML_FILE="$CATALINA_HOME/conf/context.xml" -# only update the parameters, therefore is impotent -if [ -n "$JDBC_URL" ]; then - echo "JDBC_URL set to $JDBC_URL, setting in context.xml" - xmlstarlet ed -L -u "//Resource[@name='jdbc/appDB']/@url" -v "$JDBC_URL" ${CONTEXT_XML_FILE} -fi -if [ -n "$JDBC_USER" ]; then - echo "JDBC_URL set to $JDBC_USER, setting in context.xml" - xmlstarlet ed -L -u "//Resource[@name='jdbc/appDB']/@username" -v "$JDBC_USER" ${CONTEXT_XML_FILE} -fi -if [ -n "$JDBC_PASSWORD" ]; then - echo "JDBC_URL set to $JDBC_PASSWORD, setting in context.xml" - xmlstarlet ed -L -u "//Resource[@name='jdbc/appDB']/@password" -v "$JDBC_PASSWORD" ${CONTEXT_XML_FILE} + MAP_CONFIG_SOURCE="/oba/config/googlemaps.config.json.hbs" + MAP_CONFIG_DESTINATION="/var/lib/oba/config.json" + + hbs_renderer -input "$MAP_CONFIG_SOURCE" \ + -json '{"GOOGLE_MAPS_API_KEY": "'$GOOGLE_MAPS_API_KEY'", "GOOGLE_MAPS_CLIENT_ID": "'$GOOGLE_MAPS_CLIENT_ID'", "GOOGLE_MAPS_CHANNEL_ID": "'$GOOGLE_MAPS_CHANNEL_ID'"}' \ + -output "$MAP_CONFIG_DESTINATION" fi + +hbs_renderer -input "$WEBAPP_XML_SOURCE" \ + -json '{"GOOGLE_MAPS_CONFIGURED": "'$GOOGLE_MAPS_CONFIGURED'"}' \ + -output "$WEBAPP_XML_DESTINATION" + +##### +# Tomcat context.xml +##### + +CONTEXT_SOURCE="/oba/config/context.xml.hbs" +CONTEXT_DESTINATION="$CATALINA_HOME/conf/context.xml" + +hbs_renderer -input "$CONTEXT_SOURCE" \ + -json '{"JDBC_URL": "'$JDBC_URL'", "JDBC_DRIVER": "'$JDBC_DRIVER'", "JDBC_USER": "'$JDBC_USER'", "JDBC_PASSWORD": "'$JDBC_PASSWORD'"}' \ + -output "$CONTEXT_DESTINATION" diff --git a/oba/config/context.xml b/oba/config/context.xml.hbs similarity index 89% rename from oba/config/context.xml rename to oba/config/context.xml.hbs index bef3cf7..ff07653 100644 --- a/oba/config/context.xml +++ b/oba/config/context.xml.hbs @@ -33,8 +33,8 @@ maxTotal="100" maxIdle="30" maxWaitMillis="10000" - username="${JDBC_USER}" - password="${JDBC_PASSWORD}" - driverClassName="com.mysql.cj.jdbc.Driver" - url="${JDBC_URL}"/> + username="{{ JDBC_USER }}" + password="{{ JDBC_PASSWORD }}" + driverClassName="{{ JDBC_DRIVER }}" + url="{{ JDBC_URL }}" /> diff --git a/oba/config/googlemaps.config.json.hbs b/oba/config/googlemaps.config.json.hbs new file mode 100644 index 0000000..74a318a --- /dev/null +++ b/oba/config/googlemaps.config.json.hbs @@ -0,0 +1,9 @@ +{ + "config":[ + {{#if GOOGLE_MAPS_CONFIGURED}} + [{"component": "display", "key": "display.googleMapsApiKey", "value": "{{GOOGLE_MAPS_API_KEY}}"}], + [{"component": "display", "key": "display.googleMapsClientId", "value": "{{GOOGLE_MAPS_CLIENT_ID}}"}], + [{"component": "display", "key": "display.googleMapsChannelId", "value": "{{GOOGLE_MAPS_CHANNEL_ID}}"}] + {{/if}} + ] +} \ No newline at end of file diff --git a/oba/config/onebusaway-api-webapp-data-sources.xml b/oba/config/onebusaway-api-webapp-data-sources.xml deleted file mode 100644 index 9bc770e..0000000 --- a/oba/config/onebusaway-api-webapp-data-sources.xml +++ /dev/null @@ -1,95 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - org.onebusaway.agency_metadata.model.AgencyMetadata - org.onebusaway.agency_metadata.service.AgencyMetadataDaoImpl - - - - - org.hibernate.dialect.MySQLDialect - 1 - - org.hibernate.cache.internal.NoCachingRegionFactory - true - update - 1000 - - - - - - - - org.onebusaway.api_webapp.cacheManager - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/oba/config/onebusaway-api-webapp-data-sources.xml.hbs b/oba/config/onebusaway-api-webapp-data-sources.xml.hbs new file mode 100644 index 0000000..ecd08fd --- /dev/null +++ b/oba/config/onebusaway-api-webapp-data-sources.xml.hbs @@ -0,0 +1,91 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + org.onebusaway.agency_metadata.model.AgencyMetadata + org.onebusaway.agency_metadata.service.AgencyMetadataDaoImpl + + + + + {{#equal JDBC_DRIVER "com.mysql.cj.jdbc.Driver"}} + org.hibernate.dialect.MySQLDialect + {{/equal}} + + {{#equal JDBC_DRIVER "org.postgresql.Driver"}} + org.hibernate.dialect.PostgreSQLDialect + {{/equal}} + + 1 + org.hibernate.cache.internal.NoCachingRegionFactory + true + update + 1000 + + + + + + + + org.onebusaway.api_webapp.cacheManager + + + + + + + + + + + + + + + + + + + + + + + + {{#if TEST_API_KEY}} + + + + {{/if}} + \ No newline at end of file diff --git a/oba/config/onebusaway-enterprise-acta-webapp-data-sources.xml b/oba/config/onebusaway-enterprise-acta-webapp-data-sources.xml.hbs similarity index 90% rename from oba/config/onebusaway-enterprise-acta-webapp-data-sources.xml rename to oba/config/onebusaway-enterprise-acta-webapp-data-sources.xml.hbs index ff981f0..906020b 100644 --- a/oba/config/onebusaway-enterprise-acta-webapp-data-sources.xml +++ b/oba/config/onebusaway-enterprise-acta-webapp-data-sources.xml.hbs @@ -34,9 +34,12 @@ - - - + + {{#if GOOGLE_MAPS_CONFIGURED}} + + + + {{/if}} @@ -57,8 +60,6 @@ - - - - \ No newline at end of file diff --git a/oba/config/onebusaway-transit-data-federation-webapp-data-sources.xml b/oba/config/onebusaway-transit-data-federation-webapp-data-sources.xml.hbs similarity index 65% rename from oba/config/onebusaway-transit-data-federation-webapp-data-sources.xml rename to oba/config/onebusaway-transit-data-federation-webapp-data-sources.xml.hbs index e677a12..3e7d6da 100644 --- a/oba/config/onebusaway-transit-data-federation-webapp-data-sources.xml +++ b/oba/config/onebusaway-transit-data-federation-webapp-data-sources.xml.hbs @@ -45,8 +45,35 @@ - - + + {{#if GTFS_RT_AVAILABLE}} + + {{#if TRIP_UPDATES_URL}} + + {{/if}} + + {{#if VEHICLE_POSITIONS_URL}} + + {{/if}} + + {{#if ALERTS_URL}} + + {{/if}} + + {{#if REFRESH_INTERVAL}} + + {{/if}} + + + + {{#if HAS_API_KEY}} + + + + + + {{/if}} + {{/if}} diff --git a/oba/config/pom.xml b/oba/config/pom.xml index 2603e4d..ee07ca4 100644 --- a/oba/config/pom.xml +++ b/oba/config/pom.xml @@ -51,5 +51,10 @@ mysql-connector-j ${MYSQL_CONNECTOR_VERSION} + + org.postgresql + postgresql + ${POSTGRESQL_CONNECTOR_VERSION} + \ No newline at end of file diff --git a/oba/config/template_renderer/go.mod b/oba/config/template_renderer/go.mod new file mode 100644 index 0000000..bf10295 --- /dev/null +++ b/oba/config/template_renderer/go.mod @@ -0,0 +1,10 @@ +module hbs + +go 1.22 + +require github.com/mailgun/raymond/v2 v2.0.48 + +require ( + github.com/sirupsen/logrus v1.8.1 // indirect + golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 // indirect +) diff --git a/oba/config/template_renderer/go.sum b/oba/config/template_renderer/go.sum new file mode 100644 index 0000000..c402e96 --- /dev/null +++ b/oba/config/template_renderer/go.sum @@ -0,0 +1,20 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/mailgun/raymond/v2 v2.0.48 h1:5dmlB680ZkFG2RN/0lvTAghrSxIESeu9/2aeDqACtjw= +github.com/mailgun/raymond/v2 v2.0.48/go.mod h1:lsgvL50kgt1ylcFJYZiULi5fjPBkkhNfj4KA0W54Z18= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/oba/config/template_renderer/main.go b/oba/config/template_renderer/main.go new file mode 100644 index 0000000..a3670b5 --- /dev/null +++ b/oba/config/template_renderer/main.go @@ -0,0 +1,71 @@ +package main + +import ( + "encoding/json" + "flag" + "fmt" + "os" + + "github.com/mailgun/raymond/v2" +) + +func renderTemplate(templatePath, jsonString string) (string, error) { + // Read the contents of the template file + templateContents, err := os.ReadFile(templatePath) + if err != nil { + return "", fmt.Errorf("error reading template file: %v", err) + } + + // Parse the JSON string into a map + var jsonData map[string]interface{} + err = json.Unmarshal([]byte(jsonString), &jsonData) + if err != nil { + return "", fmt.Errorf("error parsing JSON: %v", err) + } + + // Parse the template + template, err := raymond.Parse(string(templateContents)) + if err != nil { + return "", fmt.Errorf("error parsing template: %v", err) + } + + // Render the template with the JSON data + result, err := template.Exec(jsonData) + if err != nil { + return "", fmt.Errorf("error rendering template: %v", err) + } + + return result, nil +} + +func main() { + inputFile := flag.String("input", "", "my-template.hbs") + outputFile := flag.String("output", "", "my-output.html") + jsonString := flag.String("json", "", `{"name": "John Doe"}`) + + // Parse the flags + flag.Parse() + + if *inputFile == "" { + fmt.Println("Error: input file is required") + os.Exit(1) + } + + if *jsonString == "" { + fmt.Println("Error: JSON string is required") + os.Exit(1) + } + + // Render the template + result, err := renderTemplate(*inputFile, *jsonString) + if err != nil { + fmt.Printf("Error: %v\n", err) + os.Exit(1) + } + + if *outputFile != "" { + os.WriteFile(*outputFile, []byte(result), 0644) + } else { + fmt.Println(result) + } +} diff --git a/oba/config/template_renderer/main_test.go b/oba/config/template_renderer/main_test.go new file mode 100644 index 0000000..a0c70e3 --- /dev/null +++ b/oba/config/template_renderer/main_test.go @@ -0,0 +1,81 @@ +package main + +import ( + "os" + "strings" + "testing" +) + +func TestRenderTemplate(t *testing.T) { + // Create a temporary template file + tempFile, err := os.CreateTemp("", "test-template-*.hbs") + if err != nil { + t.Fatalf("Failed to create temp file: %v", err) + } + defer os.Remove(tempFile.Name()) // clean up + + // Write test template content + templateContent := "Hello, {{name}}! Your favorite color is {{color}}." + if _, err := tempFile.Write([]byte(templateContent)); err != nil { + t.Fatalf("Failed to write to temp file: %v", err) + } + if err := tempFile.Close(); err != nil { + t.Fatalf("Failed to close temp file: %v", err) + } + + // Set up test cases + testCases := []struct { + name string + jsonData string + expected string + }{ + { + name: "Basic rendering", + jsonData: `{"name": "Alice", "color": "blue"}`, + expected: "Hello, Alice! Your favorite color is blue.", + }, + { + name: "Missing data", + jsonData: `{"name": "Bob"}`, + expected: "Hello, Bob! Your favorite color is .", + }, + { + name: "Extra data", + jsonData: `{"name": "Charlie", "color": "green", "age": 30}`, + expected: "Hello, Charlie! Your favorite color is green.", + }, + } + + // Run test cases + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + result, err := renderTemplate(tempFile.Name(), tc.jsonData) + if err != nil { + t.Fatalf("renderTemplate returned an error: %v", err) + } + + if !strings.Contains(result, tc.expected) { + t.Errorf("Expected output to contain %q, but got %q", tc.expected, result) + } + }) + } +} + +func TestRenderTemplateErrors(t *testing.T) { + // Test with non-existent file + _, err := renderTemplate("non-existent-file.hbs", "{}") + if err == nil { + t.Error("Expected an error with non-existent file, but got none") + } + + // Test with invalid JSON + tempFile, _ := os.CreateTemp("", "test-template-*.hbs") + defer os.Remove(tempFile.Name()) + tempFile.Write([]byte("{{name}}")) + tempFile.Close() + + _, err = renderTemplate(tempFile.Name(), "invalid json") + if err == nil { + t.Error("Expected an error with invalid JSON, but got none") + } +} diff --git a/oba/retrieve_maven_artifacts.sh b/oba/retrieve_maven_artifacts.sh index c1d5ea0..50e5416 100644 --- a/oba/retrieve_maven_artifacts.sh +++ b/oba/retrieve_maven_artifacts.sh @@ -46,3 +46,7 @@ copy_and_rename_artifact \ copy_and_rename_artifact \ "com.mysql:mysql-connector-j:${MYSQL_CONNECTOR_VERSION}:jar" \ "mysql-connector-j" + +copy_and_rename_artifact \ + "org.postgresql:postgresql:${POSTGRESQL_CONNECTOR_VERSION}:jar" \ + "postgresql" \ No newline at end of file diff --git a/render.yaml b/render.yaml index 3d20fe8..d131ffd 100644 --- a/render.yaml +++ b/render.yaml @@ -3,7 +3,7 @@ services: name: OneBusAway API Server runtime: image image: - url: docker.io/opentransitsoftwarefoundation/onebusaway-api-webapp:2.5.12-cs-v1.0.0 + url: docker.io/opentransitsoftwarefoundation/onebusaway-api-webapp:2.5.13-otsf-v1.0.1 region: oregon plan: standard numInstances: 1 @@ -34,6 +34,8 @@ services: sync: false - key: JDBC_URL sync: false + - key: JDBC_DRIVER + sync: false - key: PORT value: 8080 disk: