diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..485dee6 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.idea diff --git a/0.7.51/bin/apply_osc_to_db.sh b/0.7.51/bin/apply_osc_to_db.sh deleted file mode 100755 index 94d1879..0000000 --- a/0.7.51/bin/apply_osc_to_db.sh +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht -# -# This file is part of Overpass_API. -# -# Overpass_API is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Overpass_API is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Overpass_API. If not, see . - -if [[ -z $3 ]]; then -{ - echo "Usage: $0 replicate_dir start_id --meta=(attic|yes|no)" - exit 0 -}; fi - -EXEC_DIR="`dirname $0`/" -if [[ ! ${EXEC_DIR:0:1} == "/" ]]; then -{ - EXEC_DIR="`pwd`/$EXEC_DIR" -}; fi - -DB_DIR=`$EXEC_DIR/dispatcher --show-dir` - -if [[ ! -d $DB_DIR ]] ; then - echo "Can't find DB_DIR. Returned value: $DB_DIR" - exit 1; -fi - -REPLICATE_DIR="$1" -if [[ ! ${REPLICATE_DIR:0:1} == "/" ]]; then -{ - REPLICATE_DIR="`pwd`/$REPLICATE_DIR" -}; fi - -START=$2 -META= - -if [[ $3 == "--meta=attic" ]]; then - META="--keep-attic" -elif [[ $3 == "--meta=yes" || $3 == "--meta" ]]; then - META="--meta" -elif [[ $3 == "--meta=no" ]]; then - META= -else -{ - echo "You must specify --meta=yes or --meta=no" - exit 0 -}; fi - -PRODUCE_DIFF= - -get_replicate_filename() -{ - printf -v TDIGIT3 %03u $(($1 % 1000)) - ARG=$(($1 / 1000)) - printf -v TDIGIT2 %03u $(($ARG % 1000)) - ARG=$(($ARG / 1000)) - printf -v TDIGIT1 %03u $ARG - REPLICATE_TRUNK_DIR=$TDIGIT1/$TDIGIT2/ - REPLICATE_FILENAME=$TDIGIT1/$TDIGIT2/$TDIGIT3 -}; - - -collect_minute_diffs() -{ - TEMP_DIR=$1 - TARGET=$(($START + 1)) - - get_replicate_filename $TARGET - - while [[ ( -s $REPLICATE_DIR/$REPLICATE_FILENAME.state.txt ) && ( $(($START + 1440)) -ge $(($TARGET)) ) && ( `du -m $TEMP_DIR | awk '{ print $1; }'` -le 512 ) ]]; - do - { - printf -v TARGET_FILE %09u $TARGET - gunzip <$REPLICATE_DIR/$REPLICATE_FILENAME.osc.gz >$TEMP_DIR/$TARGET_FILE.osc - TARGET=$(($TARGET + 1)) - get_replicate_filename $TARGET - }; - done - TARGET=$(($TARGET - 1)) -}; - - -apply_minute_diffs() -{ - ./update_from_dir --osc-dir=$1 --version=$DATA_VERSION $META --flush-size=0 - EXITCODE=$? - while [[ $EXITCODE -ne 0 ]]; - do - { - sleep 60 - ./update_from_dir --osc-dir=$1 --version=$DATA_VERSION $META --flush-size=0 - EXITCODE=$? - }; - done - DIFF_COUNT=$(($DIFF_COUNT + 1)) -}; - - -update_state() -{ - get_replicate_filename $TARGET - TIMESTAMP_LINE=`grep "^timestamp" <$REPLICATE_DIR/$REPLICATE_FILENAME.state.txt` - while [[ -z $TIMESTAMP_LINE ]]; do - { - sleep 5 - TIMESTAMP_LINE=`grep "^timestamp" <$REPLICATE_DIR/$REPLICATE_FILENAME.state.txt` - }; done - DATA_VERSION=${TIMESTAMP_LINE:10} -}; - - -echo >>$DB_DIR/apply_osc_to_db.log - -mkdir -p $DB_DIR/augmented_diffs/ -DIFF_COUNT=0 - -# update_state - -pushd "$EXEC_DIR" - -while [[ true ]]; do -{ - if [[ $START == "auto" ]]; then - { - START=`cat $DB_DIR/replicate_id` - }; fi - - echo "`date -u '+%F %T'`: updating from $START" >>$DB_DIR/apply_osc_to_db.log - - TEMP_DIR=`mktemp -d /tmp/osm-3s_update_XXXXXX` - collect_minute_diffs $TEMP_DIR - - if [[ $TARGET -gt $START ]]; then - { - echo "`date -u '+%F %T'`: updating to $TARGET" >>$DB_DIR/apply_osc_to_db.log - - update_state - apply_minute_diffs $TEMP_DIR - echo "$TARGET" >$DB_DIR/replicate_id - - echo "`date -u '+%F %T'`: update complete" $TARGET >>$DB_DIR/apply_osc_to_db.log - }; - else - { - sleep 5 - }; fi - - rm -f $TEMP_DIR/* - rmdir $TEMP_DIR - - START=$TARGET -}; done diff --git a/0.7.51/bin/fetch_osc.sh b/0.7.51/bin/fetch_osc.sh deleted file mode 100755 index a9d163f..0000000 --- a/0.7.51/bin/fetch_osc.sh +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht -# -# This file is part of Overpass_API. -# -# Overpass_API is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Overpass_API is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Overpass_API. If not, see . - -if [[ -z $1 ]]; then -{ - echo Usage: $0 Replicate_id Source_dir Local_dir [Sleep] - exit 0 -}; -fi - -REPLICATE_ID=$1 -SOURCE_DIR=$2 -LOCAL_DIR=$3 -SLEEP_BETWEEN_DLS=15 # How long to sleep between download attempts (sec). Default: 15. See also FAIL_COUNTER_ALERT -FAIL_COUNTER_ALERT=20 # After how many sleep cycles do we get nervous -FAIL_COUNTER=0 -FILE_PANIC= - -if [[ ! -d $LOCAL_DIR ]]; - then { - mkdir $LOCAL_DIR -}; -fi - -if [[ $REPLICATE_ID == "auto" ]] ; then - REPLICATE_ID=`find $LOCAL_DIR -type f -name '*state.txt' -not -size 0 | sort | tail -n 1 | sed -e "s#^$LOCAL_DIR##" -e 's#[./]##g' -e 's#statetxt##' -e 's/^0\+//g'` - - EXEC_DIR="`dirname $0`/" - DB_DIR=`$EXEC_DIR/dispatcher --show-dir` - - if [[ ! -d $DB_DIR ]] ; then - echo "Can't find DB_DIR. Returned value: $DB_DIR" - exit 1; - fi - - if [[ "x$REPLICATE_ID" == "x" ]] ; then - REPLICATE_ID=`cat $DB_DIR/replicate_id` - fi - if [[ "x$REPLICATE_ID" == "x" ]] ; then - echo "Could not determine REPLICATE_ID. Exiting" - exit 1 - fi -fi - -# $1 - remote source -# $2 - local destination -fetch_file() -{ - wget -nv -O "$2" "$1" -}; - -retry_fetch_file() -{ - FILE_PANIC= - FAIL_COUNTER=0 - if [[ ! -s "$2" ]]; then { - fetch_file "$1" "$2" - if [[ "$3" == "gzip" ]]; then { - gunzip -t <"$2" - if [[ $? -ne 0 ]]; then { - rm "$2" - }; fi - }; fi - }; fi - until [[ -s "$2" || $FAIL_COUNTER -ge $FAIL_COUNTER_ALERT ]]; do { - FAIL_COUNTER=$(($FAIL_COUNTER+1)) - sleep $SLEEP_BETWEEN_DLS - fetch_file "$1" "$2" - if [[ "$3" == "gzip" ]]; then { - gunzip -t <"$2" - if [[ $? -ne 0 ]]; then { - rm "$2" - }; fi - }; fi - }; done - if [[ $FAIL_COUNTER -ge $FAIL_COUNTER_ALERT ]]; then - FILE_PANIC="true" - fi -}; - -file_panic() -{ - echo "fetch_osc()@"`date -u "+%F %T"`": upstream_delay $REPLICATE_ID" >>$LOCAL_DIR/fetch_osc.log - REPLICATE_ID=$(($REPLICATE_ID - 1)) - - printf -v TDIGIT3 %03u $(($REPLICATE_ID % 1000)) - ARG=$(($REPLICATE_ID / 1000)) - printf -v TDIGIT2 %03u $(($ARG % 1000)) - ARG=$(($ARG / 1000)) - printf -v TDIGIT1 %03u $ARG - - FILE_PANIC="true" - until [[ ! -n $FILE_PANIC ]]; do { - retry_fetch_file "$REMOTE_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.new.osc.gz" "gzip" - }; done - FILE_PANIC="true" - until [[ ! -n $FILE_PANIC ]]; do { - retry_fetch_file "$REMOTE_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.new.state.txt" "text" - }; done - - RES_GZIP=`diff -q "$LOCAL_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.new.osc.gz"` - RES_TEXT=`diff -q "$LOCAL_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.new.state.txt"` - if [[ -n $RES_GZIP || -n $RES_TEXT ]]; then - echo "fetch_osc()@"`date -u "+%F %T"`": file_panic $REPLICATE_ID" >>$LOCAL_DIR/fetch_osc.log - echo "fetch_osc()@"`date -u "+%F %T"`": $RES_GZIP" >>$LOCAL_DIR/fetch_osc.log - echo "fetch_osc()@"`date -u "+%F %T"`": $RES_TEXT" >>$LOCAL_DIR/fetch_osc.log - exit 1 - fi - - rm "$LOCAL_PATH/$TDIGIT3.new.osc.gz" - rm "$LOCAL_PATH/$TDIGIT3.new.state.txt" -}; - -fetch_minute_diff() -{ - printf -v TDIGIT3 %03u $(($REPLICATE_ID % 1000)) - ARG=$(($REPLICATE_ID / 1000)) - printf -v TDIGIT2 %03u $(($ARG % 1000)) - ARG=$(($ARG / 1000)) - printf -v TDIGIT1 %03u $ARG - - LOCAL_PATH="$LOCAL_DIR/$TDIGIT1/$TDIGIT2" - REMOTE_PATH="$SOURCE_DIR/$TDIGIT1/$TDIGIT2" - mkdir -p "$LOCAL_DIR/$TDIGIT1/$TDIGIT2" - - retry_fetch_file "$REMOTE_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.osc.gz" "gzip" - if [[ -n $FILE_PANIC ]]; then - file_panic - fi - retry_fetch_file "$REMOTE_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.state.txt" "text" - if [[ -n $FILE_PANIC ]]; then - file_panic - fi - - TIMESTAMP_LINE=`grep timestamp $LOCAL_DIR/$TDIGIT1/$TDIGIT2/$TDIGIT3.state.txt` - TIMESTAMP=${TIMESTAMP_LINE:10} -}; - -while [[ true ]]; -do -{ - REPLICATE_ID=$(($REPLICATE_ID + 1)) - fetch_minute_diff - echo "fetch_osc()@"`date -u "+%F %T"`": new_replicate_diff $REPLICATE_ID $TIMESTAMP" >>$LOCAL_DIR/fetch_osc.log - sleep 1 -}; -done diff --git a/0.7.51/docker-entrypoint.sh b/0.7.51/docker-entrypoint.sh deleted file mode 100644 index 5a43603..0000000 --- a/0.7.51/docker-entrypoint.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -set -eo pipefail -shopt -s nullglob -OVERPASS_META=${OVERPASS_META:-no} -OVERPASS_MODE=${OVERPASS_MODE:-clone} -OVERPASS_COMPRESSION=${OVERPASS_COMPRESSION:-gz} - -if [ ! -d /db/db ] ; then - if [ "$OVERPASS_MODE" = "clone" ]; then - if [[ "$OVERPASS_META" == "attic" ]] ; then - META="--keep-attic" - else - META="--meta=$OVERPASS_META" - fi - mkdir -p /db/db \ - && /app/bin/download_clone.sh --db-dir=/db/db --source=http://dev.overpass-api.de/api_drolbr/ $META \ - && cp -r /app/etc/rules /db/db \ - && chown -R overpass:overpass /db \ - && echo "Overpass ready, you can start your container with docker start" - exit - fi - - if [ "$OVERPASS_MODE" = "init" ]; then - lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet; exit" \ - && /app/bin/init_osm3s.sh /db/planet /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION" \ - && echo $OVERPASS_PLANET_SEQUENCE_ID > /db/db/replicate_id \ - && rm /db/planet \ - && cp -r /app/etc/rules /db/db \ - && chown -R overpass:overpass /db \ - && echo "Overpass ready, you can start your container with docker start" - exit - fi -fi - -exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf diff --git a/0.7.53/Dockerfile b/0.7.53/Dockerfile index f14622a..adc83bb 100644 --- a/0.7.53/Dockerfile +++ b/0.7.53/Dockerfile @@ -1,27 +1,34 @@ -FROM nginx:stable +FROM nginx:1.14 RUN addgroup overpass && adduser --home /db --disabled-password --gecos overpass --ingroup overpass overpass +ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.53.tar.gz /app/src.tar.gz + RUN apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ autoconf \ automake \ + bash \ + bzip2 \ ca-certificates \ expat \ - libexpat1-dev \ + fcgiwrap \ g++ \ + lftp \ + libexpat1-dev \ liblz4-1 \ liblz4-dev \ libtool \ m4 \ make \ + osmium-tool \ + python3 \ + python3-venv \ + supervisor \ + wget \ zlib1g \ - zlib1g-dev - - -ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.53.tar.gz /app/src.tar.gz - -RUN mkdir -p /app/src \ + zlib1g-dev \ + && mkdir -p /app/src \ && cd /app/src \ && tar -x -z --strip-components 1 -f ../src.tar.gz \ && autoscan \ @@ -45,26 +52,28 @@ RUN mkdir -p /app/src \ && apt-get autoremove -y \ && mkdir -p /db/diffs /app/etc \ && cp -r /app/src/rules /app/etc/rules \ - && rm -rf /app/src /app/src.tar.gz + && rm -rf /app/src /app/src.tar.gz \ + && rm -rf /var/lib/apt/lists/* -RUN apt-get install --no-install-recommends --no-install-suggests -y \ - supervisor \ - bash \ - lftp \ - wget \ - fcgiwrap \ - bzip2 +COPY requirements.txt /app/ +RUN python3 -m venv /app/venv \ + && /app/venv/bin/pip install -r /app/requirements.txt RUN mkdir /nginx && chown nginx:nginx /nginx && chown -R overpass:overpass /db + COPY etc/supervisord.conf /etc/supervisor/conf.d/supervisord.conf + COPY etc/nginx-overpass.conf /etc/nginx/nginx.conf -COPY bin/apply_osc_to_db.sh bin/fetch_osc.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/ + +COPY bin/update_overpass.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/ + COPY docker-entrypoint.sh /app/ -RUN chmod a+x /app/docker-entrypoint.sh /app/bin/apply_osc_to_db.sh /app/bin/fetch_osc.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh -VOLUME /db + +RUN chmod a+x /app/docker-entrypoint.sh /app/bin/update_overpass.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh ENV OVERPASS_RULES_LOAD 1 + EXPOSE 80 # CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"] CMD ["/app/docker-entrypoint.sh"] diff --git a/0.7.53/bin/apply_osc_to_db.sh b/0.7.53/bin/apply_osc_to_db.sh deleted file mode 100755 index 94d1879..0000000 --- a/0.7.53/bin/apply_osc_to_db.sh +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht -# -# This file is part of Overpass_API. -# -# Overpass_API is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Overpass_API is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Overpass_API. If not, see . - -if [[ -z $3 ]]; then -{ - echo "Usage: $0 replicate_dir start_id --meta=(attic|yes|no)" - exit 0 -}; fi - -EXEC_DIR="`dirname $0`/" -if [[ ! ${EXEC_DIR:0:1} == "/" ]]; then -{ - EXEC_DIR="`pwd`/$EXEC_DIR" -}; fi - -DB_DIR=`$EXEC_DIR/dispatcher --show-dir` - -if [[ ! -d $DB_DIR ]] ; then - echo "Can't find DB_DIR. Returned value: $DB_DIR" - exit 1; -fi - -REPLICATE_DIR="$1" -if [[ ! ${REPLICATE_DIR:0:1} == "/" ]]; then -{ - REPLICATE_DIR="`pwd`/$REPLICATE_DIR" -}; fi - -START=$2 -META= - -if [[ $3 == "--meta=attic" ]]; then - META="--keep-attic" -elif [[ $3 == "--meta=yes" || $3 == "--meta" ]]; then - META="--meta" -elif [[ $3 == "--meta=no" ]]; then - META= -else -{ - echo "You must specify --meta=yes or --meta=no" - exit 0 -}; fi - -PRODUCE_DIFF= - -get_replicate_filename() -{ - printf -v TDIGIT3 %03u $(($1 % 1000)) - ARG=$(($1 / 1000)) - printf -v TDIGIT2 %03u $(($ARG % 1000)) - ARG=$(($ARG / 1000)) - printf -v TDIGIT1 %03u $ARG - REPLICATE_TRUNK_DIR=$TDIGIT1/$TDIGIT2/ - REPLICATE_FILENAME=$TDIGIT1/$TDIGIT2/$TDIGIT3 -}; - - -collect_minute_diffs() -{ - TEMP_DIR=$1 - TARGET=$(($START + 1)) - - get_replicate_filename $TARGET - - while [[ ( -s $REPLICATE_DIR/$REPLICATE_FILENAME.state.txt ) && ( $(($START + 1440)) -ge $(($TARGET)) ) && ( `du -m $TEMP_DIR | awk '{ print $1; }'` -le 512 ) ]]; - do - { - printf -v TARGET_FILE %09u $TARGET - gunzip <$REPLICATE_DIR/$REPLICATE_FILENAME.osc.gz >$TEMP_DIR/$TARGET_FILE.osc - TARGET=$(($TARGET + 1)) - get_replicate_filename $TARGET - }; - done - TARGET=$(($TARGET - 1)) -}; - - -apply_minute_diffs() -{ - ./update_from_dir --osc-dir=$1 --version=$DATA_VERSION $META --flush-size=0 - EXITCODE=$? - while [[ $EXITCODE -ne 0 ]]; - do - { - sleep 60 - ./update_from_dir --osc-dir=$1 --version=$DATA_VERSION $META --flush-size=0 - EXITCODE=$? - }; - done - DIFF_COUNT=$(($DIFF_COUNT + 1)) -}; - - -update_state() -{ - get_replicate_filename $TARGET - TIMESTAMP_LINE=`grep "^timestamp" <$REPLICATE_DIR/$REPLICATE_FILENAME.state.txt` - while [[ -z $TIMESTAMP_LINE ]]; do - { - sleep 5 - TIMESTAMP_LINE=`grep "^timestamp" <$REPLICATE_DIR/$REPLICATE_FILENAME.state.txt` - }; done - DATA_VERSION=${TIMESTAMP_LINE:10} -}; - - -echo >>$DB_DIR/apply_osc_to_db.log - -mkdir -p $DB_DIR/augmented_diffs/ -DIFF_COUNT=0 - -# update_state - -pushd "$EXEC_DIR" - -while [[ true ]]; do -{ - if [[ $START == "auto" ]]; then - { - START=`cat $DB_DIR/replicate_id` - }; fi - - echo "`date -u '+%F %T'`: updating from $START" >>$DB_DIR/apply_osc_to_db.log - - TEMP_DIR=`mktemp -d /tmp/osm-3s_update_XXXXXX` - collect_minute_diffs $TEMP_DIR - - if [[ $TARGET -gt $START ]]; then - { - echo "`date -u '+%F %T'`: updating to $TARGET" >>$DB_DIR/apply_osc_to_db.log - - update_state - apply_minute_diffs $TEMP_DIR - echo "$TARGET" >$DB_DIR/replicate_id - - echo "`date -u '+%F %T'`: update complete" $TARGET >>$DB_DIR/apply_osc_to_db.log - }; - else - { - sleep 5 - }; fi - - rm -f $TEMP_DIR/* - rmdir $TEMP_DIR - - START=$TARGET -}; done diff --git a/0.7.53/bin/fetch_osc.sh b/0.7.53/bin/fetch_osc.sh deleted file mode 100755 index a9d163f..0000000 --- a/0.7.53/bin/fetch_osc.sh +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht -# -# This file is part of Overpass_API. -# -# Overpass_API is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Overpass_API is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Overpass_API. If not, see . - -if [[ -z $1 ]]; then -{ - echo Usage: $0 Replicate_id Source_dir Local_dir [Sleep] - exit 0 -}; -fi - -REPLICATE_ID=$1 -SOURCE_DIR=$2 -LOCAL_DIR=$3 -SLEEP_BETWEEN_DLS=15 # How long to sleep between download attempts (sec). Default: 15. See also FAIL_COUNTER_ALERT -FAIL_COUNTER_ALERT=20 # After how many sleep cycles do we get nervous -FAIL_COUNTER=0 -FILE_PANIC= - -if [[ ! -d $LOCAL_DIR ]]; - then { - mkdir $LOCAL_DIR -}; -fi - -if [[ $REPLICATE_ID == "auto" ]] ; then - REPLICATE_ID=`find $LOCAL_DIR -type f -name '*state.txt' -not -size 0 | sort | tail -n 1 | sed -e "s#^$LOCAL_DIR##" -e 's#[./]##g' -e 's#statetxt##' -e 's/^0\+//g'` - - EXEC_DIR="`dirname $0`/" - DB_DIR=`$EXEC_DIR/dispatcher --show-dir` - - if [[ ! -d $DB_DIR ]] ; then - echo "Can't find DB_DIR. Returned value: $DB_DIR" - exit 1; - fi - - if [[ "x$REPLICATE_ID" == "x" ]] ; then - REPLICATE_ID=`cat $DB_DIR/replicate_id` - fi - if [[ "x$REPLICATE_ID" == "x" ]] ; then - echo "Could not determine REPLICATE_ID. Exiting" - exit 1 - fi -fi - -# $1 - remote source -# $2 - local destination -fetch_file() -{ - wget -nv -O "$2" "$1" -}; - -retry_fetch_file() -{ - FILE_PANIC= - FAIL_COUNTER=0 - if [[ ! -s "$2" ]]; then { - fetch_file "$1" "$2" - if [[ "$3" == "gzip" ]]; then { - gunzip -t <"$2" - if [[ $? -ne 0 ]]; then { - rm "$2" - }; fi - }; fi - }; fi - until [[ -s "$2" || $FAIL_COUNTER -ge $FAIL_COUNTER_ALERT ]]; do { - FAIL_COUNTER=$(($FAIL_COUNTER+1)) - sleep $SLEEP_BETWEEN_DLS - fetch_file "$1" "$2" - if [[ "$3" == "gzip" ]]; then { - gunzip -t <"$2" - if [[ $? -ne 0 ]]; then { - rm "$2" - }; fi - }; fi - }; done - if [[ $FAIL_COUNTER -ge $FAIL_COUNTER_ALERT ]]; then - FILE_PANIC="true" - fi -}; - -file_panic() -{ - echo "fetch_osc()@"`date -u "+%F %T"`": upstream_delay $REPLICATE_ID" >>$LOCAL_DIR/fetch_osc.log - REPLICATE_ID=$(($REPLICATE_ID - 1)) - - printf -v TDIGIT3 %03u $(($REPLICATE_ID % 1000)) - ARG=$(($REPLICATE_ID / 1000)) - printf -v TDIGIT2 %03u $(($ARG % 1000)) - ARG=$(($ARG / 1000)) - printf -v TDIGIT1 %03u $ARG - - FILE_PANIC="true" - until [[ ! -n $FILE_PANIC ]]; do { - retry_fetch_file "$REMOTE_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.new.osc.gz" "gzip" - }; done - FILE_PANIC="true" - until [[ ! -n $FILE_PANIC ]]; do { - retry_fetch_file "$REMOTE_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.new.state.txt" "text" - }; done - - RES_GZIP=`diff -q "$LOCAL_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.new.osc.gz"` - RES_TEXT=`diff -q "$LOCAL_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.new.state.txt"` - if [[ -n $RES_GZIP || -n $RES_TEXT ]]; then - echo "fetch_osc()@"`date -u "+%F %T"`": file_panic $REPLICATE_ID" >>$LOCAL_DIR/fetch_osc.log - echo "fetch_osc()@"`date -u "+%F %T"`": $RES_GZIP" >>$LOCAL_DIR/fetch_osc.log - echo "fetch_osc()@"`date -u "+%F %T"`": $RES_TEXT" >>$LOCAL_DIR/fetch_osc.log - exit 1 - fi - - rm "$LOCAL_PATH/$TDIGIT3.new.osc.gz" - rm "$LOCAL_PATH/$TDIGIT3.new.state.txt" -}; - -fetch_minute_diff() -{ - printf -v TDIGIT3 %03u $(($REPLICATE_ID % 1000)) - ARG=$(($REPLICATE_ID / 1000)) - printf -v TDIGIT2 %03u $(($ARG % 1000)) - ARG=$(($ARG / 1000)) - printf -v TDIGIT1 %03u $ARG - - LOCAL_PATH="$LOCAL_DIR/$TDIGIT1/$TDIGIT2" - REMOTE_PATH="$SOURCE_DIR/$TDIGIT1/$TDIGIT2" - mkdir -p "$LOCAL_DIR/$TDIGIT1/$TDIGIT2" - - retry_fetch_file "$REMOTE_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.osc.gz" "gzip" - if [[ -n $FILE_PANIC ]]; then - file_panic - fi - retry_fetch_file "$REMOTE_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.state.txt" "text" - if [[ -n $FILE_PANIC ]]; then - file_panic - fi - - TIMESTAMP_LINE=`grep timestamp $LOCAL_DIR/$TDIGIT1/$TDIGIT2/$TDIGIT3.state.txt` - TIMESTAMP=${TIMESTAMP_LINE:10} -}; - -while [[ true ]]; -do -{ - REPLICATE_ID=$(($REPLICATE_ID + 1)) - fetch_minute_diff - echo "fetch_osc()@"`date -u "+%F %T"`": new_replicate_diff $REPLICATE_ID $TIMESTAMP" >>$LOCAL_DIR/fetch_osc.log - sleep 1 -}; -done diff --git a/0.7.53/bin/update_overpass.sh b/0.7.53/bin/update_overpass.sh new file mode 100644 index 0000000..328e375 --- /dev/null +++ b/0.7.53/bin/update_overpass.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env sh + +( + set -e + + if [ "x$OVERPASS_META" = "xattic" ] ; then + META="--keep-attic" + elif [ "x$OVERPASS_META" = "xyes" ] ; then + META="--meta" + else + META="" + fi + + if [ ! -d /db/diffs ] ; then + mkdir /db/diffs + fi + + while `true` ; do + ( + if [ ! -e /db/diffs/changes.osm ] ; then + ! /app/venv/bin/pyosmium-get-changes --server $OVERPASS_DIFF_URL -o /db/diffs/changes.osm -f /db/replicate_id + OSMIUM_STATUS=$? + if [ $OSMIUM_STATUS -eq 1 ]; then + echo "There are still some updates remainging" + fi + if [ $OSMIUM_STATUS -eq 2 ]; then + echo "Failure downloading updates" + exit 0 + fi + else + echo "/db/diffs/changes.osm exists. Trying to apply again." + fi + /app/bin/update_from_dir --osc-dir=/db/diffs/ $META + rm /db/diffs/changes.osm + ) + sleep 60 + done +) 2>&1 | tee -a /db/changes.log \ No newline at end of file diff --git a/0.7.53/docker-entrypoint.sh b/0.7.53/docker-entrypoint.sh index 5a43603..cb69627 100644 --- a/0.7.53/docker-entrypoint.sh +++ b/0.7.53/docker-entrypoint.sh @@ -6,13 +6,15 @@ OVERPASS_META=${OVERPASS_META:-no} OVERPASS_MODE=${OVERPASS_MODE:-clone} OVERPASS_COMPRESSION=${OVERPASS_COMPRESSION:-gz} +if [[ "$OVERPASS_META" == "attic" ]] ; then + META="--keep-attic" +else + META="--meta" +fi + + if [ ! -d /db/db ] ; then if [ "$OVERPASS_MODE" = "clone" ]; then - if [[ "$OVERPASS_META" == "attic" ]] ; then - META="--keep-attic" - else - META="--meta=$OVERPASS_META" - fi mkdir -p /db/db \ && /app/bin/download_clone.sh --db-dir=/db/db --source=http://dev.overpass-api.de/api_drolbr/ $META \ && cp -r /app/etc/rules /db/db \ @@ -22,10 +24,21 @@ if [ ! -d /db/db ] ; then fi if [ "$OVERPASS_MODE" = "init" ]; then - lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet; exit" \ - && /app/bin/init_osm3s.sh /db/planet /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION" \ - && echo $OVERPASS_PLANET_SEQUENCE_ID > /db/db/replicate_id \ - && rm /db/planet \ + lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet.osm.bz2; exit" \ + && /app/bin/init_osm3s.sh /db/planet.osm.bz2 /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION --map-compression-method=$OVERPASS_COMPRESSION"\ + && echo "Database created. Now updating it." && ( + ! /app/venv/bin/pyosmium-get-changes -O /db/planet.osm.bz2 --server $OVERPASS_DIFF_URL -o /db/changes.osm -f /db/replicate_id + OSMIUM_STATUS=$? + if [ $OSMIUM_STATUS -eq 1 ]; then + echo "There are still some updates remainging" + fi + if [ $OSMIUM_STATUS -eq 2 ]; then + echo "Failure downloading updates" + exit 0 + fi + (cat /db/changes.osm | /app/bin/update_database --db-dir=/db/db $META --compression-method=$OVERPASS_COMPRESSION) 2>&1 | tee -a /db/changes.log + ) \ + && rm /db/planet.osm.bz2 /db/changes.osm \ && cp -r /app/etc/rules /db/db \ && chown -R overpass:overpass /db \ && echo "Overpass ready, you can start your container with docker start" diff --git a/0.7.53/etc/nginx-overpass.conf b/0.7.53/etc/nginx-overpass.conf index c019615..a19dde4 100644 --- a/0.7.53/etc/nginx-overpass.conf +++ b/0.7.53/etc/nginx-overpass.conf @@ -25,7 +25,7 @@ http { sendfile on; #tcp_nopush on; - keepalive_timeout 65; + keepalive_timeout 1000s; #gzip on; @@ -55,6 +55,7 @@ http { fastcgi_param SCRIPT_FILENAME /app/$fastcgi_script_name; fastcgi_param PATH_INFO $fastcgi_path_info; fastcgi_pass unix:/nginx/fcgiwrap.socket; + fastcgi_read_timeout 1000s; } location /api/ { diff --git a/0.7.53/etc/supervisord.conf b/0.7.53/etc/supervisord.conf index 6c4623b..fc67b74 100644 --- a/0.7.53/etc/supervisord.conf +++ b/0.7.53/etc/supervisord.conf @@ -17,18 +17,12 @@ redirect_stderr=true user=nginx priority=3 -[program:fetch_diff] -command=/app/bin/fetch_osc.sh auto %(ENV_OVERPASS_DIFF_URL)s /db/diffs +[program:update_overpass] +command=/app/bin/update_overpass.sh user=overpass redirect_stderr=true priority=5 -[program:apply_diff] -command=/app/bin/apply_osc_to_db.sh /db/diffs/ auto --meta=%(ENV_OVERPASS_META)s -redirect_stderr=true -user=overpass -priority=4 - [program:dispatcher_areas] command=nice /app/bin/dispatcher --areas --db-dir="/db/db" user=overpass diff --git a/0.7.53/requirements.txt b/0.7.53/requirements.txt new file mode 100644 index 0000000..8314012 --- /dev/null +++ b/0.7.53/requirements.txt @@ -0,0 +1 @@ +osmium diff --git a/0.7.54.10/Dockerfile b/0.7.54.10/Dockerfile index 5cd623f..ae1e6aa 100644 --- a/0.7.54.10/Dockerfile +++ b/0.7.54.10/Dockerfile @@ -1,27 +1,34 @@ -FROM nginx:stable +FROM nginx:1.14 RUN addgroup overpass && adduser --home /db --disabled-password --gecos overpass --ingroup overpass overpass +ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.54.10.tar.gz /app/src.tar.gz + RUN apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ autoconf \ automake \ + bash \ + bzip2 \ ca-certificates \ expat \ - libexpat1-dev \ + fcgiwrap \ g++ \ + lftp \ + libexpat1-dev \ liblz4-1 \ liblz4-dev \ libtool \ m4 \ make \ + osmium-tool \ + python3 \ + python3-venv \ + supervisor \ + wget \ zlib1g \ - zlib1g-dev - - -ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.54.10.tar.gz /app/src.tar.gz - -RUN mkdir -p /app/src \ + zlib1g-dev \ + && mkdir -p /app/src \ && cd /app/src \ && tar -x -z --strip-components 1 -f ../src.tar.gz \ && autoscan \ @@ -45,26 +52,28 @@ RUN mkdir -p /app/src \ && apt-get autoremove -y \ && mkdir -p /db/diffs /app/etc \ && cp -r /app/src/rules /app/etc/rules \ - && rm -rf /app/src /app/src.tar.gz + && rm -rf /app/src /app/src.tar.gz \ + && rm -rf /var/lib/apt/lists/* -RUN apt-get install --no-install-recommends --no-install-suggests -y \ - supervisor \ - bash \ - lftp \ - wget \ - fcgiwrap \ - bzip2 +COPY requirements.txt /app/ +RUN python3 -m venv /app/venv \ + && /app/venv/bin/pip install -r /app/requirements.txt RUN mkdir /nginx && chown nginx:nginx /nginx && chown -R overpass:overpass /db + COPY etc/supervisord.conf /etc/supervisor/conf.d/supervisord.conf + COPY etc/nginx-overpass.conf /etc/nginx/nginx.conf -COPY bin/apply_osc_to_db.sh bin/fetch_osc.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/ + +COPY bin/update_overpass.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/ + COPY docker-entrypoint.sh /app/ -RUN chmod a+x /app/docker-entrypoint.sh /app/bin/apply_osc_to_db.sh /app/bin/fetch_osc.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh -VOLUME /db + +RUN chmod a+x /app/docker-entrypoint.sh /app/bin/update_overpass.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh ENV OVERPASS_RULES_LOAD 1 + EXPOSE 80 # CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"] CMD ["/app/docker-entrypoint.sh"] diff --git a/0.7.54.10/bin/apply_osc_to_db.sh b/0.7.54.10/bin/apply_osc_to_db.sh deleted file mode 100755 index 94d1879..0000000 --- a/0.7.54.10/bin/apply_osc_to_db.sh +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht -# -# This file is part of Overpass_API. -# -# Overpass_API is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Overpass_API is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Overpass_API. If not, see . - -if [[ -z $3 ]]; then -{ - echo "Usage: $0 replicate_dir start_id --meta=(attic|yes|no)" - exit 0 -}; fi - -EXEC_DIR="`dirname $0`/" -if [[ ! ${EXEC_DIR:0:1} == "/" ]]; then -{ - EXEC_DIR="`pwd`/$EXEC_DIR" -}; fi - -DB_DIR=`$EXEC_DIR/dispatcher --show-dir` - -if [[ ! -d $DB_DIR ]] ; then - echo "Can't find DB_DIR. Returned value: $DB_DIR" - exit 1; -fi - -REPLICATE_DIR="$1" -if [[ ! ${REPLICATE_DIR:0:1} == "/" ]]; then -{ - REPLICATE_DIR="`pwd`/$REPLICATE_DIR" -}; fi - -START=$2 -META= - -if [[ $3 == "--meta=attic" ]]; then - META="--keep-attic" -elif [[ $3 == "--meta=yes" || $3 == "--meta" ]]; then - META="--meta" -elif [[ $3 == "--meta=no" ]]; then - META= -else -{ - echo "You must specify --meta=yes or --meta=no" - exit 0 -}; fi - -PRODUCE_DIFF= - -get_replicate_filename() -{ - printf -v TDIGIT3 %03u $(($1 % 1000)) - ARG=$(($1 / 1000)) - printf -v TDIGIT2 %03u $(($ARG % 1000)) - ARG=$(($ARG / 1000)) - printf -v TDIGIT1 %03u $ARG - REPLICATE_TRUNK_DIR=$TDIGIT1/$TDIGIT2/ - REPLICATE_FILENAME=$TDIGIT1/$TDIGIT2/$TDIGIT3 -}; - - -collect_minute_diffs() -{ - TEMP_DIR=$1 - TARGET=$(($START + 1)) - - get_replicate_filename $TARGET - - while [[ ( -s $REPLICATE_DIR/$REPLICATE_FILENAME.state.txt ) && ( $(($START + 1440)) -ge $(($TARGET)) ) && ( `du -m $TEMP_DIR | awk '{ print $1; }'` -le 512 ) ]]; - do - { - printf -v TARGET_FILE %09u $TARGET - gunzip <$REPLICATE_DIR/$REPLICATE_FILENAME.osc.gz >$TEMP_DIR/$TARGET_FILE.osc - TARGET=$(($TARGET + 1)) - get_replicate_filename $TARGET - }; - done - TARGET=$(($TARGET - 1)) -}; - - -apply_minute_diffs() -{ - ./update_from_dir --osc-dir=$1 --version=$DATA_VERSION $META --flush-size=0 - EXITCODE=$? - while [[ $EXITCODE -ne 0 ]]; - do - { - sleep 60 - ./update_from_dir --osc-dir=$1 --version=$DATA_VERSION $META --flush-size=0 - EXITCODE=$? - }; - done - DIFF_COUNT=$(($DIFF_COUNT + 1)) -}; - - -update_state() -{ - get_replicate_filename $TARGET - TIMESTAMP_LINE=`grep "^timestamp" <$REPLICATE_DIR/$REPLICATE_FILENAME.state.txt` - while [[ -z $TIMESTAMP_LINE ]]; do - { - sleep 5 - TIMESTAMP_LINE=`grep "^timestamp" <$REPLICATE_DIR/$REPLICATE_FILENAME.state.txt` - }; done - DATA_VERSION=${TIMESTAMP_LINE:10} -}; - - -echo >>$DB_DIR/apply_osc_to_db.log - -mkdir -p $DB_DIR/augmented_diffs/ -DIFF_COUNT=0 - -# update_state - -pushd "$EXEC_DIR" - -while [[ true ]]; do -{ - if [[ $START == "auto" ]]; then - { - START=`cat $DB_DIR/replicate_id` - }; fi - - echo "`date -u '+%F %T'`: updating from $START" >>$DB_DIR/apply_osc_to_db.log - - TEMP_DIR=`mktemp -d /tmp/osm-3s_update_XXXXXX` - collect_minute_diffs $TEMP_DIR - - if [[ $TARGET -gt $START ]]; then - { - echo "`date -u '+%F %T'`: updating to $TARGET" >>$DB_DIR/apply_osc_to_db.log - - update_state - apply_minute_diffs $TEMP_DIR - echo "$TARGET" >$DB_DIR/replicate_id - - echo "`date -u '+%F %T'`: update complete" $TARGET >>$DB_DIR/apply_osc_to_db.log - }; - else - { - sleep 5 - }; fi - - rm -f $TEMP_DIR/* - rmdir $TEMP_DIR - - START=$TARGET -}; done diff --git a/0.7.54.10/bin/fetch_osc.sh b/0.7.54.10/bin/fetch_osc.sh deleted file mode 100755 index a9d163f..0000000 --- a/0.7.54.10/bin/fetch_osc.sh +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht -# -# This file is part of Overpass_API. -# -# Overpass_API is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Overpass_API is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Overpass_API. If not, see . - -if [[ -z $1 ]]; then -{ - echo Usage: $0 Replicate_id Source_dir Local_dir [Sleep] - exit 0 -}; -fi - -REPLICATE_ID=$1 -SOURCE_DIR=$2 -LOCAL_DIR=$3 -SLEEP_BETWEEN_DLS=15 # How long to sleep between download attempts (sec). Default: 15. See also FAIL_COUNTER_ALERT -FAIL_COUNTER_ALERT=20 # After how many sleep cycles do we get nervous -FAIL_COUNTER=0 -FILE_PANIC= - -if [[ ! -d $LOCAL_DIR ]]; - then { - mkdir $LOCAL_DIR -}; -fi - -if [[ $REPLICATE_ID == "auto" ]] ; then - REPLICATE_ID=`find $LOCAL_DIR -type f -name '*state.txt' -not -size 0 | sort | tail -n 1 | sed -e "s#^$LOCAL_DIR##" -e 's#[./]##g' -e 's#statetxt##' -e 's/^0\+//g'` - - EXEC_DIR="`dirname $0`/" - DB_DIR=`$EXEC_DIR/dispatcher --show-dir` - - if [[ ! -d $DB_DIR ]] ; then - echo "Can't find DB_DIR. Returned value: $DB_DIR" - exit 1; - fi - - if [[ "x$REPLICATE_ID" == "x" ]] ; then - REPLICATE_ID=`cat $DB_DIR/replicate_id` - fi - if [[ "x$REPLICATE_ID" == "x" ]] ; then - echo "Could not determine REPLICATE_ID. Exiting" - exit 1 - fi -fi - -# $1 - remote source -# $2 - local destination -fetch_file() -{ - wget -nv -O "$2" "$1" -}; - -retry_fetch_file() -{ - FILE_PANIC= - FAIL_COUNTER=0 - if [[ ! -s "$2" ]]; then { - fetch_file "$1" "$2" - if [[ "$3" == "gzip" ]]; then { - gunzip -t <"$2" - if [[ $? -ne 0 ]]; then { - rm "$2" - }; fi - }; fi - }; fi - until [[ -s "$2" || $FAIL_COUNTER -ge $FAIL_COUNTER_ALERT ]]; do { - FAIL_COUNTER=$(($FAIL_COUNTER+1)) - sleep $SLEEP_BETWEEN_DLS - fetch_file "$1" "$2" - if [[ "$3" == "gzip" ]]; then { - gunzip -t <"$2" - if [[ $? -ne 0 ]]; then { - rm "$2" - }; fi - }; fi - }; done - if [[ $FAIL_COUNTER -ge $FAIL_COUNTER_ALERT ]]; then - FILE_PANIC="true" - fi -}; - -file_panic() -{ - echo "fetch_osc()@"`date -u "+%F %T"`": upstream_delay $REPLICATE_ID" >>$LOCAL_DIR/fetch_osc.log - REPLICATE_ID=$(($REPLICATE_ID - 1)) - - printf -v TDIGIT3 %03u $(($REPLICATE_ID % 1000)) - ARG=$(($REPLICATE_ID / 1000)) - printf -v TDIGIT2 %03u $(($ARG % 1000)) - ARG=$(($ARG / 1000)) - printf -v TDIGIT1 %03u $ARG - - FILE_PANIC="true" - until [[ ! -n $FILE_PANIC ]]; do { - retry_fetch_file "$REMOTE_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.new.osc.gz" "gzip" - }; done - FILE_PANIC="true" - until [[ ! -n $FILE_PANIC ]]; do { - retry_fetch_file "$REMOTE_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.new.state.txt" "text" - }; done - - RES_GZIP=`diff -q "$LOCAL_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.new.osc.gz"` - RES_TEXT=`diff -q "$LOCAL_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.new.state.txt"` - if [[ -n $RES_GZIP || -n $RES_TEXT ]]; then - echo "fetch_osc()@"`date -u "+%F %T"`": file_panic $REPLICATE_ID" >>$LOCAL_DIR/fetch_osc.log - echo "fetch_osc()@"`date -u "+%F %T"`": $RES_GZIP" >>$LOCAL_DIR/fetch_osc.log - echo "fetch_osc()@"`date -u "+%F %T"`": $RES_TEXT" >>$LOCAL_DIR/fetch_osc.log - exit 1 - fi - - rm "$LOCAL_PATH/$TDIGIT3.new.osc.gz" - rm "$LOCAL_PATH/$TDIGIT3.new.state.txt" -}; - -fetch_minute_diff() -{ - printf -v TDIGIT3 %03u $(($REPLICATE_ID % 1000)) - ARG=$(($REPLICATE_ID / 1000)) - printf -v TDIGIT2 %03u $(($ARG % 1000)) - ARG=$(($ARG / 1000)) - printf -v TDIGIT1 %03u $ARG - - LOCAL_PATH="$LOCAL_DIR/$TDIGIT1/$TDIGIT2" - REMOTE_PATH="$SOURCE_DIR/$TDIGIT1/$TDIGIT2" - mkdir -p "$LOCAL_DIR/$TDIGIT1/$TDIGIT2" - - retry_fetch_file "$REMOTE_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.osc.gz" "gzip" - if [[ -n $FILE_PANIC ]]; then - file_panic - fi - retry_fetch_file "$REMOTE_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.state.txt" "text" - if [[ -n $FILE_PANIC ]]; then - file_panic - fi - - TIMESTAMP_LINE=`grep timestamp $LOCAL_DIR/$TDIGIT1/$TDIGIT2/$TDIGIT3.state.txt` - TIMESTAMP=${TIMESTAMP_LINE:10} -}; - -while [[ true ]]; -do -{ - REPLICATE_ID=$(($REPLICATE_ID + 1)) - fetch_minute_diff - echo "fetch_osc()@"`date -u "+%F %T"`": new_replicate_diff $REPLICATE_ID $TIMESTAMP" >>$LOCAL_DIR/fetch_osc.log - sleep 1 -}; -done diff --git a/0.7.54.10/bin/update_overpass.sh b/0.7.54.10/bin/update_overpass.sh new file mode 100644 index 0000000..328e375 --- /dev/null +++ b/0.7.54.10/bin/update_overpass.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env sh + +( + set -e + + if [ "x$OVERPASS_META" = "xattic" ] ; then + META="--keep-attic" + elif [ "x$OVERPASS_META" = "xyes" ] ; then + META="--meta" + else + META="" + fi + + if [ ! -d /db/diffs ] ; then + mkdir /db/diffs + fi + + while `true` ; do + ( + if [ ! -e /db/diffs/changes.osm ] ; then + ! /app/venv/bin/pyosmium-get-changes --server $OVERPASS_DIFF_URL -o /db/diffs/changes.osm -f /db/replicate_id + OSMIUM_STATUS=$? + if [ $OSMIUM_STATUS -eq 1 ]; then + echo "There are still some updates remainging" + fi + if [ $OSMIUM_STATUS -eq 2 ]; then + echo "Failure downloading updates" + exit 0 + fi + else + echo "/db/diffs/changes.osm exists. Trying to apply again." + fi + /app/bin/update_from_dir --osc-dir=/db/diffs/ $META + rm /db/diffs/changes.osm + ) + sleep 60 + done +) 2>&1 | tee -a /db/changes.log \ No newline at end of file diff --git a/0.7.54.10/docker-entrypoint.sh b/0.7.54.10/docker-entrypoint.sh index 5a43603..cb69627 100644 --- a/0.7.54.10/docker-entrypoint.sh +++ b/0.7.54.10/docker-entrypoint.sh @@ -6,13 +6,15 @@ OVERPASS_META=${OVERPASS_META:-no} OVERPASS_MODE=${OVERPASS_MODE:-clone} OVERPASS_COMPRESSION=${OVERPASS_COMPRESSION:-gz} +if [[ "$OVERPASS_META" == "attic" ]] ; then + META="--keep-attic" +else + META="--meta" +fi + + if [ ! -d /db/db ] ; then if [ "$OVERPASS_MODE" = "clone" ]; then - if [[ "$OVERPASS_META" == "attic" ]] ; then - META="--keep-attic" - else - META="--meta=$OVERPASS_META" - fi mkdir -p /db/db \ && /app/bin/download_clone.sh --db-dir=/db/db --source=http://dev.overpass-api.de/api_drolbr/ $META \ && cp -r /app/etc/rules /db/db \ @@ -22,10 +24,21 @@ if [ ! -d /db/db ] ; then fi if [ "$OVERPASS_MODE" = "init" ]; then - lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet; exit" \ - && /app/bin/init_osm3s.sh /db/planet /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION" \ - && echo $OVERPASS_PLANET_SEQUENCE_ID > /db/db/replicate_id \ - && rm /db/planet \ + lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet.osm.bz2; exit" \ + && /app/bin/init_osm3s.sh /db/planet.osm.bz2 /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION --map-compression-method=$OVERPASS_COMPRESSION"\ + && echo "Database created. Now updating it." && ( + ! /app/venv/bin/pyosmium-get-changes -O /db/planet.osm.bz2 --server $OVERPASS_DIFF_URL -o /db/changes.osm -f /db/replicate_id + OSMIUM_STATUS=$? + if [ $OSMIUM_STATUS -eq 1 ]; then + echo "There are still some updates remainging" + fi + if [ $OSMIUM_STATUS -eq 2 ]; then + echo "Failure downloading updates" + exit 0 + fi + (cat /db/changes.osm | /app/bin/update_database --db-dir=/db/db $META --compression-method=$OVERPASS_COMPRESSION) 2>&1 | tee -a /db/changes.log + ) \ + && rm /db/planet.osm.bz2 /db/changes.osm \ && cp -r /app/etc/rules /db/db \ && chown -R overpass:overpass /db \ && echo "Overpass ready, you can start your container with docker start" diff --git a/0.7.54.10/etc/nginx-overpass.conf b/0.7.54.10/etc/nginx-overpass.conf index c019615..a19dde4 100644 --- a/0.7.54.10/etc/nginx-overpass.conf +++ b/0.7.54.10/etc/nginx-overpass.conf @@ -25,7 +25,7 @@ http { sendfile on; #tcp_nopush on; - keepalive_timeout 65; + keepalive_timeout 1000s; #gzip on; @@ -55,6 +55,7 @@ http { fastcgi_param SCRIPT_FILENAME /app/$fastcgi_script_name; fastcgi_param PATH_INFO $fastcgi_path_info; fastcgi_pass unix:/nginx/fcgiwrap.socket; + fastcgi_read_timeout 1000s; } location /api/ { diff --git a/0.7.54.10/etc/supervisord.conf b/0.7.54.10/etc/supervisord.conf index 6c4623b..fc67b74 100644 --- a/0.7.54.10/etc/supervisord.conf +++ b/0.7.54.10/etc/supervisord.conf @@ -17,18 +17,12 @@ redirect_stderr=true user=nginx priority=3 -[program:fetch_diff] -command=/app/bin/fetch_osc.sh auto %(ENV_OVERPASS_DIFF_URL)s /db/diffs +[program:update_overpass] +command=/app/bin/update_overpass.sh user=overpass redirect_stderr=true priority=5 -[program:apply_diff] -command=/app/bin/apply_osc_to_db.sh /db/diffs/ auto --meta=%(ENV_OVERPASS_META)s -redirect_stderr=true -user=overpass -priority=4 - [program:dispatcher_areas] command=nice /app/bin/dispatcher --areas --db-dir="/db/db" user=overpass diff --git a/0.7.54.10/requirements.txt b/0.7.54.10/requirements.txt new file mode 100644 index 0000000..8314012 --- /dev/null +++ b/0.7.54.10/requirements.txt @@ -0,0 +1 @@ +osmium diff --git a/0.7.54.11/bin/apply_osc_to_db.sh b/0.7.54.11/bin/apply_osc_to_db.sh deleted file mode 100755 index 94d1879..0000000 --- a/0.7.54.11/bin/apply_osc_to_db.sh +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht -# -# This file is part of Overpass_API. -# -# Overpass_API is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Overpass_API is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Overpass_API. If not, see . - -if [[ -z $3 ]]; then -{ - echo "Usage: $0 replicate_dir start_id --meta=(attic|yes|no)" - exit 0 -}; fi - -EXEC_DIR="`dirname $0`/" -if [[ ! ${EXEC_DIR:0:1} == "/" ]]; then -{ - EXEC_DIR="`pwd`/$EXEC_DIR" -}; fi - -DB_DIR=`$EXEC_DIR/dispatcher --show-dir` - -if [[ ! -d $DB_DIR ]] ; then - echo "Can't find DB_DIR. Returned value: $DB_DIR" - exit 1; -fi - -REPLICATE_DIR="$1" -if [[ ! ${REPLICATE_DIR:0:1} == "/" ]]; then -{ - REPLICATE_DIR="`pwd`/$REPLICATE_DIR" -}; fi - -START=$2 -META= - -if [[ $3 == "--meta=attic" ]]; then - META="--keep-attic" -elif [[ $3 == "--meta=yes" || $3 == "--meta" ]]; then - META="--meta" -elif [[ $3 == "--meta=no" ]]; then - META= -else -{ - echo "You must specify --meta=yes or --meta=no" - exit 0 -}; fi - -PRODUCE_DIFF= - -get_replicate_filename() -{ - printf -v TDIGIT3 %03u $(($1 % 1000)) - ARG=$(($1 / 1000)) - printf -v TDIGIT2 %03u $(($ARG % 1000)) - ARG=$(($ARG / 1000)) - printf -v TDIGIT1 %03u $ARG - REPLICATE_TRUNK_DIR=$TDIGIT1/$TDIGIT2/ - REPLICATE_FILENAME=$TDIGIT1/$TDIGIT2/$TDIGIT3 -}; - - -collect_minute_diffs() -{ - TEMP_DIR=$1 - TARGET=$(($START + 1)) - - get_replicate_filename $TARGET - - while [[ ( -s $REPLICATE_DIR/$REPLICATE_FILENAME.state.txt ) && ( $(($START + 1440)) -ge $(($TARGET)) ) && ( `du -m $TEMP_DIR | awk '{ print $1; }'` -le 512 ) ]]; - do - { - printf -v TARGET_FILE %09u $TARGET - gunzip <$REPLICATE_DIR/$REPLICATE_FILENAME.osc.gz >$TEMP_DIR/$TARGET_FILE.osc - TARGET=$(($TARGET + 1)) - get_replicate_filename $TARGET - }; - done - TARGET=$(($TARGET - 1)) -}; - - -apply_minute_diffs() -{ - ./update_from_dir --osc-dir=$1 --version=$DATA_VERSION $META --flush-size=0 - EXITCODE=$? - while [[ $EXITCODE -ne 0 ]]; - do - { - sleep 60 - ./update_from_dir --osc-dir=$1 --version=$DATA_VERSION $META --flush-size=0 - EXITCODE=$? - }; - done - DIFF_COUNT=$(($DIFF_COUNT + 1)) -}; - - -update_state() -{ - get_replicate_filename $TARGET - TIMESTAMP_LINE=`grep "^timestamp" <$REPLICATE_DIR/$REPLICATE_FILENAME.state.txt` - while [[ -z $TIMESTAMP_LINE ]]; do - { - sleep 5 - TIMESTAMP_LINE=`grep "^timestamp" <$REPLICATE_DIR/$REPLICATE_FILENAME.state.txt` - }; done - DATA_VERSION=${TIMESTAMP_LINE:10} -}; - - -echo >>$DB_DIR/apply_osc_to_db.log - -mkdir -p $DB_DIR/augmented_diffs/ -DIFF_COUNT=0 - -# update_state - -pushd "$EXEC_DIR" - -while [[ true ]]; do -{ - if [[ $START == "auto" ]]; then - { - START=`cat $DB_DIR/replicate_id` - }; fi - - echo "`date -u '+%F %T'`: updating from $START" >>$DB_DIR/apply_osc_to_db.log - - TEMP_DIR=`mktemp -d /tmp/osm-3s_update_XXXXXX` - collect_minute_diffs $TEMP_DIR - - if [[ $TARGET -gt $START ]]; then - { - echo "`date -u '+%F %T'`: updating to $TARGET" >>$DB_DIR/apply_osc_to_db.log - - update_state - apply_minute_diffs $TEMP_DIR - echo "$TARGET" >$DB_DIR/replicate_id - - echo "`date -u '+%F %T'`: update complete" $TARGET >>$DB_DIR/apply_osc_to_db.log - }; - else - { - sleep 5 - }; fi - - rm -f $TEMP_DIR/* - rmdir $TEMP_DIR - - START=$TARGET -}; done diff --git a/0.7.54.11/bin/fetch_osc.sh b/0.7.54.11/bin/fetch_osc.sh deleted file mode 100755 index a9d163f..0000000 --- a/0.7.54.11/bin/fetch_osc.sh +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht -# -# This file is part of Overpass_API. -# -# Overpass_API is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Overpass_API is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Overpass_API. If not, see . - -if [[ -z $1 ]]; then -{ - echo Usage: $0 Replicate_id Source_dir Local_dir [Sleep] - exit 0 -}; -fi - -REPLICATE_ID=$1 -SOURCE_DIR=$2 -LOCAL_DIR=$3 -SLEEP_BETWEEN_DLS=15 # How long to sleep between download attempts (sec). Default: 15. See also FAIL_COUNTER_ALERT -FAIL_COUNTER_ALERT=20 # After how many sleep cycles do we get nervous -FAIL_COUNTER=0 -FILE_PANIC= - -if [[ ! -d $LOCAL_DIR ]]; - then { - mkdir $LOCAL_DIR -}; -fi - -if [[ $REPLICATE_ID == "auto" ]] ; then - REPLICATE_ID=`find $LOCAL_DIR -type f -name '*state.txt' -not -size 0 | sort | tail -n 1 | sed -e "s#^$LOCAL_DIR##" -e 's#[./]##g' -e 's#statetxt##' -e 's/^0\+//g'` - - EXEC_DIR="`dirname $0`/" - DB_DIR=`$EXEC_DIR/dispatcher --show-dir` - - if [[ ! -d $DB_DIR ]] ; then - echo "Can't find DB_DIR. Returned value: $DB_DIR" - exit 1; - fi - - if [[ "x$REPLICATE_ID" == "x" ]] ; then - REPLICATE_ID=`cat $DB_DIR/replicate_id` - fi - if [[ "x$REPLICATE_ID" == "x" ]] ; then - echo "Could not determine REPLICATE_ID. Exiting" - exit 1 - fi -fi - -# $1 - remote source -# $2 - local destination -fetch_file() -{ - wget -nv -O "$2" "$1" -}; - -retry_fetch_file() -{ - FILE_PANIC= - FAIL_COUNTER=0 - if [[ ! -s "$2" ]]; then { - fetch_file "$1" "$2" - if [[ "$3" == "gzip" ]]; then { - gunzip -t <"$2" - if [[ $? -ne 0 ]]; then { - rm "$2" - }; fi - }; fi - }; fi - until [[ -s "$2" || $FAIL_COUNTER -ge $FAIL_COUNTER_ALERT ]]; do { - FAIL_COUNTER=$(($FAIL_COUNTER+1)) - sleep $SLEEP_BETWEEN_DLS - fetch_file "$1" "$2" - if [[ "$3" == "gzip" ]]; then { - gunzip -t <"$2" - if [[ $? -ne 0 ]]; then { - rm "$2" - }; fi - }; fi - }; done - if [[ $FAIL_COUNTER -ge $FAIL_COUNTER_ALERT ]]; then - FILE_PANIC="true" - fi -}; - -file_panic() -{ - echo "fetch_osc()@"`date -u "+%F %T"`": upstream_delay $REPLICATE_ID" >>$LOCAL_DIR/fetch_osc.log - REPLICATE_ID=$(($REPLICATE_ID - 1)) - - printf -v TDIGIT3 %03u $(($REPLICATE_ID % 1000)) - ARG=$(($REPLICATE_ID / 1000)) - printf -v TDIGIT2 %03u $(($ARG % 1000)) - ARG=$(($ARG / 1000)) - printf -v TDIGIT1 %03u $ARG - - FILE_PANIC="true" - until [[ ! -n $FILE_PANIC ]]; do { - retry_fetch_file "$REMOTE_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.new.osc.gz" "gzip" - }; done - FILE_PANIC="true" - until [[ ! -n $FILE_PANIC ]]; do { - retry_fetch_file "$REMOTE_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.new.state.txt" "text" - }; done - - RES_GZIP=`diff -q "$LOCAL_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.new.osc.gz"` - RES_TEXT=`diff -q "$LOCAL_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.new.state.txt"` - if [[ -n $RES_GZIP || -n $RES_TEXT ]]; then - echo "fetch_osc()@"`date -u "+%F %T"`": file_panic $REPLICATE_ID" >>$LOCAL_DIR/fetch_osc.log - echo "fetch_osc()@"`date -u "+%F %T"`": $RES_GZIP" >>$LOCAL_DIR/fetch_osc.log - echo "fetch_osc()@"`date -u "+%F %T"`": $RES_TEXT" >>$LOCAL_DIR/fetch_osc.log - exit 1 - fi - - rm "$LOCAL_PATH/$TDIGIT3.new.osc.gz" - rm "$LOCAL_PATH/$TDIGIT3.new.state.txt" -}; - -fetch_minute_diff() -{ - printf -v TDIGIT3 %03u $(($REPLICATE_ID % 1000)) - ARG=$(($REPLICATE_ID / 1000)) - printf -v TDIGIT2 %03u $(($ARG % 1000)) - ARG=$(($ARG / 1000)) - printf -v TDIGIT1 %03u $ARG - - LOCAL_PATH="$LOCAL_DIR/$TDIGIT1/$TDIGIT2" - REMOTE_PATH="$SOURCE_DIR/$TDIGIT1/$TDIGIT2" - mkdir -p "$LOCAL_DIR/$TDIGIT1/$TDIGIT2" - - retry_fetch_file "$REMOTE_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.osc.gz" "gzip" - if [[ -n $FILE_PANIC ]]; then - file_panic - fi - retry_fetch_file "$REMOTE_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.state.txt" "text" - if [[ -n $FILE_PANIC ]]; then - file_panic - fi - - TIMESTAMP_LINE=`grep timestamp $LOCAL_DIR/$TDIGIT1/$TDIGIT2/$TDIGIT3.state.txt` - TIMESTAMP=${TIMESTAMP_LINE:10} -}; - -while [[ true ]]; -do -{ - REPLICATE_ID=$(($REPLICATE_ID + 1)) - fetch_minute_diff - echo "fetch_osc()@"`date -u "+%F %T"`": new_replicate_diff $REPLICATE_ID $TIMESTAMP" >>$LOCAL_DIR/fetch_osc.log - sleep 1 -}; -done diff --git a/0.7.54.11/docker-entrypoint.sh b/0.7.54.11/docker-entrypoint.sh deleted file mode 100644 index 5a43603..0000000 --- a/0.7.54.11/docker-entrypoint.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -set -eo pipefail -shopt -s nullglob -OVERPASS_META=${OVERPASS_META:-no} -OVERPASS_MODE=${OVERPASS_MODE:-clone} -OVERPASS_COMPRESSION=${OVERPASS_COMPRESSION:-gz} - -if [ ! -d /db/db ] ; then - if [ "$OVERPASS_MODE" = "clone" ]; then - if [[ "$OVERPASS_META" == "attic" ]] ; then - META="--keep-attic" - else - META="--meta=$OVERPASS_META" - fi - mkdir -p /db/db \ - && /app/bin/download_clone.sh --db-dir=/db/db --source=http://dev.overpass-api.de/api_drolbr/ $META \ - && cp -r /app/etc/rules /db/db \ - && chown -R overpass:overpass /db \ - && echo "Overpass ready, you can start your container with docker start" - exit - fi - - if [ "$OVERPASS_MODE" = "init" ]; then - lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet; exit" \ - && /app/bin/init_osm3s.sh /db/planet /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION" \ - && echo $OVERPASS_PLANET_SEQUENCE_ID > /db/db/replicate_id \ - && rm /db/planet \ - && cp -r /app/etc/rules /db/db \ - && chown -R overpass:overpass /db \ - && echo "Overpass ready, you can start your container with docker start" - exit - fi -fi - -exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf diff --git a/0.7.54.12/Dockerfile b/0.7.54.12/Dockerfile index c5eccd1..f1543f2 100644 --- a/0.7.54.12/Dockerfile +++ b/0.7.54.12/Dockerfile @@ -1,27 +1,34 @@ -FROM nginx:stable +FROM nginx:1.14 RUN addgroup overpass && adduser --home /db --disabled-password --gecos overpass --ingroup overpass overpass +ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.54.12.tar.gz /app/src.tar.gz + RUN apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ autoconf \ automake \ + bash \ + bzip2 \ ca-certificates \ expat \ - libexpat1-dev \ + fcgiwrap \ g++ \ + lftp \ + libexpat1-dev \ liblz4-1 \ liblz4-dev \ libtool \ m4 \ make \ + osmium-tool \ + python3 \ + python3-venv \ + supervisor \ + wget \ zlib1g \ - zlib1g-dev - - -ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.54.12.tar.gz /app/src.tar.gz - -RUN mkdir -p /app/src \ + zlib1g-dev \ + && mkdir -p /app/src \ && cd /app/src \ && tar -x -z --strip-components 1 -f ../src.tar.gz \ && autoscan \ @@ -45,26 +52,28 @@ RUN mkdir -p /app/src \ && apt-get autoremove -y \ && mkdir -p /db/diffs /app/etc \ && cp -r /app/src/rules /app/etc/rules \ - && rm -rf /app/src /app/src.tar.gz + && rm -rf /app/src /app/src.tar.gz \ + && rm -rf /var/lib/apt/lists/* -RUN apt-get install --no-install-recommends --no-install-suggests -y \ - supervisor \ - bash \ - lftp \ - wget \ - fcgiwrap \ - bzip2 +COPY requirements.txt /app/ +RUN python3 -m venv /app/venv \ + && /app/venv/bin/pip install -r /app/requirements.txt RUN mkdir /nginx && chown nginx:nginx /nginx && chown -R overpass:overpass /db + COPY etc/supervisord.conf /etc/supervisor/conf.d/supervisord.conf + COPY etc/nginx-overpass.conf /etc/nginx/nginx.conf -COPY bin/apply_osc_to_db.sh bin/fetch_osc.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/ + +COPY bin/update_overpass.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/ + COPY docker-entrypoint.sh /app/ -RUN chmod a+x /app/docker-entrypoint.sh /app/bin/apply_osc_to_db.sh /app/bin/fetch_osc.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh -VOLUME /db + +RUN chmod a+x /app/docker-entrypoint.sh /app/bin/update_overpass.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh ENV OVERPASS_RULES_LOAD 1 + EXPOSE 80 # CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"] CMD ["/app/docker-entrypoint.sh"] diff --git a/0.7.54.12/bin/apply_osc_to_db.sh b/0.7.54.12/bin/apply_osc_to_db.sh deleted file mode 100755 index 94d1879..0000000 --- a/0.7.54.12/bin/apply_osc_to_db.sh +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht -# -# This file is part of Overpass_API. -# -# Overpass_API is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Overpass_API is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Overpass_API. If not, see . - -if [[ -z $3 ]]; then -{ - echo "Usage: $0 replicate_dir start_id --meta=(attic|yes|no)" - exit 0 -}; fi - -EXEC_DIR="`dirname $0`/" -if [[ ! ${EXEC_DIR:0:1} == "/" ]]; then -{ - EXEC_DIR="`pwd`/$EXEC_DIR" -}; fi - -DB_DIR=`$EXEC_DIR/dispatcher --show-dir` - -if [[ ! -d $DB_DIR ]] ; then - echo "Can't find DB_DIR. Returned value: $DB_DIR" - exit 1; -fi - -REPLICATE_DIR="$1" -if [[ ! ${REPLICATE_DIR:0:1} == "/" ]]; then -{ - REPLICATE_DIR="`pwd`/$REPLICATE_DIR" -}; fi - -START=$2 -META= - -if [[ $3 == "--meta=attic" ]]; then - META="--keep-attic" -elif [[ $3 == "--meta=yes" || $3 == "--meta" ]]; then - META="--meta" -elif [[ $3 == "--meta=no" ]]; then - META= -else -{ - echo "You must specify --meta=yes or --meta=no" - exit 0 -}; fi - -PRODUCE_DIFF= - -get_replicate_filename() -{ - printf -v TDIGIT3 %03u $(($1 % 1000)) - ARG=$(($1 / 1000)) - printf -v TDIGIT2 %03u $(($ARG % 1000)) - ARG=$(($ARG / 1000)) - printf -v TDIGIT1 %03u $ARG - REPLICATE_TRUNK_DIR=$TDIGIT1/$TDIGIT2/ - REPLICATE_FILENAME=$TDIGIT1/$TDIGIT2/$TDIGIT3 -}; - - -collect_minute_diffs() -{ - TEMP_DIR=$1 - TARGET=$(($START + 1)) - - get_replicate_filename $TARGET - - while [[ ( -s $REPLICATE_DIR/$REPLICATE_FILENAME.state.txt ) && ( $(($START + 1440)) -ge $(($TARGET)) ) && ( `du -m $TEMP_DIR | awk '{ print $1; }'` -le 512 ) ]]; - do - { - printf -v TARGET_FILE %09u $TARGET - gunzip <$REPLICATE_DIR/$REPLICATE_FILENAME.osc.gz >$TEMP_DIR/$TARGET_FILE.osc - TARGET=$(($TARGET + 1)) - get_replicate_filename $TARGET - }; - done - TARGET=$(($TARGET - 1)) -}; - - -apply_minute_diffs() -{ - ./update_from_dir --osc-dir=$1 --version=$DATA_VERSION $META --flush-size=0 - EXITCODE=$? - while [[ $EXITCODE -ne 0 ]]; - do - { - sleep 60 - ./update_from_dir --osc-dir=$1 --version=$DATA_VERSION $META --flush-size=0 - EXITCODE=$? - }; - done - DIFF_COUNT=$(($DIFF_COUNT + 1)) -}; - - -update_state() -{ - get_replicate_filename $TARGET - TIMESTAMP_LINE=`grep "^timestamp" <$REPLICATE_DIR/$REPLICATE_FILENAME.state.txt` - while [[ -z $TIMESTAMP_LINE ]]; do - { - sleep 5 - TIMESTAMP_LINE=`grep "^timestamp" <$REPLICATE_DIR/$REPLICATE_FILENAME.state.txt` - }; done - DATA_VERSION=${TIMESTAMP_LINE:10} -}; - - -echo >>$DB_DIR/apply_osc_to_db.log - -mkdir -p $DB_DIR/augmented_diffs/ -DIFF_COUNT=0 - -# update_state - -pushd "$EXEC_DIR" - -while [[ true ]]; do -{ - if [[ $START == "auto" ]]; then - { - START=`cat $DB_DIR/replicate_id` - }; fi - - echo "`date -u '+%F %T'`: updating from $START" >>$DB_DIR/apply_osc_to_db.log - - TEMP_DIR=`mktemp -d /tmp/osm-3s_update_XXXXXX` - collect_minute_diffs $TEMP_DIR - - if [[ $TARGET -gt $START ]]; then - { - echo "`date -u '+%F %T'`: updating to $TARGET" >>$DB_DIR/apply_osc_to_db.log - - update_state - apply_minute_diffs $TEMP_DIR - echo "$TARGET" >$DB_DIR/replicate_id - - echo "`date -u '+%F %T'`: update complete" $TARGET >>$DB_DIR/apply_osc_to_db.log - }; - else - { - sleep 5 - }; fi - - rm -f $TEMP_DIR/* - rmdir $TEMP_DIR - - START=$TARGET -}; done diff --git a/0.7.54.12/bin/fetch_osc.sh b/0.7.54.12/bin/fetch_osc.sh deleted file mode 100755 index a9d163f..0000000 --- a/0.7.54.12/bin/fetch_osc.sh +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht -# -# This file is part of Overpass_API. -# -# Overpass_API is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Overpass_API is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Overpass_API. If not, see . - -if [[ -z $1 ]]; then -{ - echo Usage: $0 Replicate_id Source_dir Local_dir [Sleep] - exit 0 -}; -fi - -REPLICATE_ID=$1 -SOURCE_DIR=$2 -LOCAL_DIR=$3 -SLEEP_BETWEEN_DLS=15 # How long to sleep between download attempts (sec). Default: 15. See also FAIL_COUNTER_ALERT -FAIL_COUNTER_ALERT=20 # After how many sleep cycles do we get nervous -FAIL_COUNTER=0 -FILE_PANIC= - -if [[ ! -d $LOCAL_DIR ]]; - then { - mkdir $LOCAL_DIR -}; -fi - -if [[ $REPLICATE_ID == "auto" ]] ; then - REPLICATE_ID=`find $LOCAL_DIR -type f -name '*state.txt' -not -size 0 | sort | tail -n 1 | sed -e "s#^$LOCAL_DIR##" -e 's#[./]##g' -e 's#statetxt##' -e 's/^0\+//g'` - - EXEC_DIR="`dirname $0`/" - DB_DIR=`$EXEC_DIR/dispatcher --show-dir` - - if [[ ! -d $DB_DIR ]] ; then - echo "Can't find DB_DIR. Returned value: $DB_DIR" - exit 1; - fi - - if [[ "x$REPLICATE_ID" == "x" ]] ; then - REPLICATE_ID=`cat $DB_DIR/replicate_id` - fi - if [[ "x$REPLICATE_ID" == "x" ]] ; then - echo "Could not determine REPLICATE_ID. Exiting" - exit 1 - fi -fi - -# $1 - remote source -# $2 - local destination -fetch_file() -{ - wget -nv -O "$2" "$1" -}; - -retry_fetch_file() -{ - FILE_PANIC= - FAIL_COUNTER=0 - if [[ ! -s "$2" ]]; then { - fetch_file "$1" "$2" - if [[ "$3" == "gzip" ]]; then { - gunzip -t <"$2" - if [[ $? -ne 0 ]]; then { - rm "$2" - }; fi - }; fi - }; fi - until [[ -s "$2" || $FAIL_COUNTER -ge $FAIL_COUNTER_ALERT ]]; do { - FAIL_COUNTER=$(($FAIL_COUNTER+1)) - sleep $SLEEP_BETWEEN_DLS - fetch_file "$1" "$2" - if [[ "$3" == "gzip" ]]; then { - gunzip -t <"$2" - if [[ $? -ne 0 ]]; then { - rm "$2" - }; fi - }; fi - }; done - if [[ $FAIL_COUNTER -ge $FAIL_COUNTER_ALERT ]]; then - FILE_PANIC="true" - fi -}; - -file_panic() -{ - echo "fetch_osc()@"`date -u "+%F %T"`": upstream_delay $REPLICATE_ID" >>$LOCAL_DIR/fetch_osc.log - REPLICATE_ID=$(($REPLICATE_ID - 1)) - - printf -v TDIGIT3 %03u $(($REPLICATE_ID % 1000)) - ARG=$(($REPLICATE_ID / 1000)) - printf -v TDIGIT2 %03u $(($ARG % 1000)) - ARG=$(($ARG / 1000)) - printf -v TDIGIT1 %03u $ARG - - FILE_PANIC="true" - until [[ ! -n $FILE_PANIC ]]; do { - retry_fetch_file "$REMOTE_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.new.osc.gz" "gzip" - }; done - FILE_PANIC="true" - until [[ ! -n $FILE_PANIC ]]; do { - retry_fetch_file "$REMOTE_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.new.state.txt" "text" - }; done - - RES_GZIP=`diff -q "$LOCAL_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.new.osc.gz"` - RES_TEXT=`diff -q "$LOCAL_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.new.state.txt"` - if [[ -n $RES_GZIP || -n $RES_TEXT ]]; then - echo "fetch_osc()@"`date -u "+%F %T"`": file_panic $REPLICATE_ID" >>$LOCAL_DIR/fetch_osc.log - echo "fetch_osc()@"`date -u "+%F %T"`": $RES_GZIP" >>$LOCAL_DIR/fetch_osc.log - echo "fetch_osc()@"`date -u "+%F %T"`": $RES_TEXT" >>$LOCAL_DIR/fetch_osc.log - exit 1 - fi - - rm "$LOCAL_PATH/$TDIGIT3.new.osc.gz" - rm "$LOCAL_PATH/$TDIGIT3.new.state.txt" -}; - -fetch_minute_diff() -{ - printf -v TDIGIT3 %03u $(($REPLICATE_ID % 1000)) - ARG=$(($REPLICATE_ID / 1000)) - printf -v TDIGIT2 %03u $(($ARG % 1000)) - ARG=$(($ARG / 1000)) - printf -v TDIGIT1 %03u $ARG - - LOCAL_PATH="$LOCAL_DIR/$TDIGIT1/$TDIGIT2" - REMOTE_PATH="$SOURCE_DIR/$TDIGIT1/$TDIGIT2" - mkdir -p "$LOCAL_DIR/$TDIGIT1/$TDIGIT2" - - retry_fetch_file "$REMOTE_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.osc.gz" "gzip" - if [[ -n $FILE_PANIC ]]; then - file_panic - fi - retry_fetch_file "$REMOTE_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.state.txt" "text" - if [[ -n $FILE_PANIC ]]; then - file_panic - fi - - TIMESTAMP_LINE=`grep timestamp $LOCAL_DIR/$TDIGIT1/$TDIGIT2/$TDIGIT3.state.txt` - TIMESTAMP=${TIMESTAMP_LINE:10} -}; - -while [[ true ]]; -do -{ - REPLICATE_ID=$(($REPLICATE_ID + 1)) - fetch_minute_diff - echo "fetch_osc()@"`date -u "+%F %T"`": new_replicate_diff $REPLICATE_ID $TIMESTAMP" >>$LOCAL_DIR/fetch_osc.log - sleep 1 -}; -done diff --git a/0.7.54.12/bin/update_overpass.sh b/0.7.54.12/bin/update_overpass.sh new file mode 100644 index 0000000..328e375 --- /dev/null +++ b/0.7.54.12/bin/update_overpass.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env sh + +( + set -e + + if [ "x$OVERPASS_META" = "xattic" ] ; then + META="--keep-attic" + elif [ "x$OVERPASS_META" = "xyes" ] ; then + META="--meta" + else + META="" + fi + + if [ ! -d /db/diffs ] ; then + mkdir /db/diffs + fi + + while `true` ; do + ( + if [ ! -e /db/diffs/changes.osm ] ; then + ! /app/venv/bin/pyosmium-get-changes --server $OVERPASS_DIFF_URL -o /db/diffs/changes.osm -f /db/replicate_id + OSMIUM_STATUS=$? + if [ $OSMIUM_STATUS -eq 1 ]; then + echo "There are still some updates remainging" + fi + if [ $OSMIUM_STATUS -eq 2 ]; then + echo "Failure downloading updates" + exit 0 + fi + else + echo "/db/diffs/changes.osm exists. Trying to apply again." + fi + /app/bin/update_from_dir --osc-dir=/db/diffs/ $META + rm /db/diffs/changes.osm + ) + sleep 60 + done +) 2>&1 | tee -a /db/changes.log \ No newline at end of file diff --git a/0.7.54.12/docker-entrypoint.sh b/0.7.54.12/docker-entrypoint.sh index 5a43603..cb69627 100644 --- a/0.7.54.12/docker-entrypoint.sh +++ b/0.7.54.12/docker-entrypoint.sh @@ -6,13 +6,15 @@ OVERPASS_META=${OVERPASS_META:-no} OVERPASS_MODE=${OVERPASS_MODE:-clone} OVERPASS_COMPRESSION=${OVERPASS_COMPRESSION:-gz} +if [[ "$OVERPASS_META" == "attic" ]] ; then + META="--keep-attic" +else + META="--meta" +fi + + if [ ! -d /db/db ] ; then if [ "$OVERPASS_MODE" = "clone" ]; then - if [[ "$OVERPASS_META" == "attic" ]] ; then - META="--keep-attic" - else - META="--meta=$OVERPASS_META" - fi mkdir -p /db/db \ && /app/bin/download_clone.sh --db-dir=/db/db --source=http://dev.overpass-api.de/api_drolbr/ $META \ && cp -r /app/etc/rules /db/db \ @@ -22,10 +24,21 @@ if [ ! -d /db/db ] ; then fi if [ "$OVERPASS_MODE" = "init" ]; then - lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet; exit" \ - && /app/bin/init_osm3s.sh /db/planet /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION" \ - && echo $OVERPASS_PLANET_SEQUENCE_ID > /db/db/replicate_id \ - && rm /db/planet \ + lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet.osm.bz2; exit" \ + && /app/bin/init_osm3s.sh /db/planet.osm.bz2 /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION --map-compression-method=$OVERPASS_COMPRESSION"\ + && echo "Database created. Now updating it." && ( + ! /app/venv/bin/pyosmium-get-changes -O /db/planet.osm.bz2 --server $OVERPASS_DIFF_URL -o /db/changes.osm -f /db/replicate_id + OSMIUM_STATUS=$? + if [ $OSMIUM_STATUS -eq 1 ]; then + echo "There are still some updates remainging" + fi + if [ $OSMIUM_STATUS -eq 2 ]; then + echo "Failure downloading updates" + exit 0 + fi + (cat /db/changes.osm | /app/bin/update_database --db-dir=/db/db $META --compression-method=$OVERPASS_COMPRESSION) 2>&1 | tee -a /db/changes.log + ) \ + && rm /db/planet.osm.bz2 /db/changes.osm \ && cp -r /app/etc/rules /db/db \ && chown -R overpass:overpass /db \ && echo "Overpass ready, you can start your container with docker start" diff --git a/0.7.54.12/etc/nginx-overpass.conf b/0.7.54.12/etc/nginx-overpass.conf index c019615..a19dde4 100644 --- a/0.7.54.12/etc/nginx-overpass.conf +++ b/0.7.54.12/etc/nginx-overpass.conf @@ -25,7 +25,7 @@ http { sendfile on; #tcp_nopush on; - keepalive_timeout 65; + keepalive_timeout 1000s; #gzip on; @@ -55,6 +55,7 @@ http { fastcgi_param SCRIPT_FILENAME /app/$fastcgi_script_name; fastcgi_param PATH_INFO $fastcgi_path_info; fastcgi_pass unix:/nginx/fcgiwrap.socket; + fastcgi_read_timeout 1000s; } location /api/ { diff --git a/0.7.54.12/etc/supervisord.conf b/0.7.54.12/etc/supervisord.conf index 6c4623b..fc67b74 100644 --- a/0.7.54.12/etc/supervisord.conf +++ b/0.7.54.12/etc/supervisord.conf @@ -17,18 +17,12 @@ redirect_stderr=true user=nginx priority=3 -[program:fetch_diff] -command=/app/bin/fetch_osc.sh auto %(ENV_OVERPASS_DIFF_URL)s /db/diffs +[program:update_overpass] +command=/app/bin/update_overpass.sh user=overpass redirect_stderr=true priority=5 -[program:apply_diff] -command=/app/bin/apply_osc_to_db.sh /db/diffs/ auto --meta=%(ENV_OVERPASS_META)s -redirect_stderr=true -user=overpass -priority=4 - [program:dispatcher_areas] command=nice /app/bin/dispatcher --areas --db-dir="/db/db" user=overpass diff --git a/0.7.54.12/requirements.txt b/0.7.54.12/requirements.txt new file mode 100644 index 0000000..8314012 --- /dev/null +++ b/0.7.54.12/requirements.txt @@ -0,0 +1 @@ +osmium diff --git a/0.7.54.13/Dockerfile b/0.7.54.13/Dockerfile index 1e1796c..221a5e8 100644 --- a/0.7.54.13/Dockerfile +++ b/0.7.54.13/Dockerfile @@ -1,27 +1,34 @@ -FROM nginx:stable +FROM nginx:1.14 RUN addgroup overpass && adduser --home /db --disabled-password --gecos overpass --ingroup overpass overpass +ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.54.13.tar.gz /app/src.tar.gz + RUN apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ autoconf \ automake \ + bash \ + bzip2 \ ca-certificates \ expat \ - libexpat1-dev \ + fcgiwrap \ g++ \ + lftp \ + libexpat1-dev \ liblz4-1 \ liblz4-dev \ libtool \ m4 \ make \ + osmium-tool \ + python3 \ + python3-venv \ + supervisor \ + wget \ zlib1g \ - zlib1g-dev - - -ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.54.13.tar.gz /app/src.tar.gz - -RUN mkdir -p /app/src \ + zlib1g-dev \ + && mkdir -p /app/src \ && cd /app/src \ && tar -x -z --strip-components 1 -f ../src.tar.gz \ && autoscan \ @@ -45,26 +52,28 @@ RUN mkdir -p /app/src \ && apt-get autoremove -y \ && mkdir -p /db/diffs /app/etc \ && cp -r /app/src/rules /app/etc/rules \ - && rm -rf /app/src /app/src.tar.gz + && rm -rf /app/src /app/src.tar.gz \ + && rm -rf /var/lib/apt/lists/* -RUN apt-get install --no-install-recommends --no-install-suggests -y \ - supervisor \ - bash \ - lftp \ - wget \ - fcgiwrap \ - bzip2 +COPY requirements.txt /app/ +RUN python3 -m venv /app/venv \ + && /app/venv/bin/pip install -r /app/requirements.txt RUN mkdir /nginx && chown nginx:nginx /nginx && chown -R overpass:overpass /db + COPY etc/supervisord.conf /etc/supervisor/conf.d/supervisord.conf + COPY etc/nginx-overpass.conf /etc/nginx/nginx.conf -COPY bin/apply_osc_to_db.sh bin/fetch_osc.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/ + +COPY bin/update_overpass.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/ + COPY docker-entrypoint.sh /app/ -RUN chmod a+x /app/docker-entrypoint.sh /app/bin/apply_osc_to_db.sh /app/bin/fetch_osc.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh -VOLUME /db + +RUN chmod a+x /app/docker-entrypoint.sh /app/bin/update_overpass.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh ENV OVERPASS_RULES_LOAD 1 + EXPOSE 80 # CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"] CMD ["/app/docker-entrypoint.sh"] diff --git a/0.7.54.13/bin/apply_osc_to_db.sh b/0.7.54.13/bin/apply_osc_to_db.sh deleted file mode 100755 index 94d1879..0000000 --- a/0.7.54.13/bin/apply_osc_to_db.sh +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht -# -# This file is part of Overpass_API. -# -# Overpass_API is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Overpass_API is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Overpass_API. If not, see . - -if [[ -z $3 ]]; then -{ - echo "Usage: $0 replicate_dir start_id --meta=(attic|yes|no)" - exit 0 -}; fi - -EXEC_DIR="`dirname $0`/" -if [[ ! ${EXEC_DIR:0:1} == "/" ]]; then -{ - EXEC_DIR="`pwd`/$EXEC_DIR" -}; fi - -DB_DIR=`$EXEC_DIR/dispatcher --show-dir` - -if [[ ! -d $DB_DIR ]] ; then - echo "Can't find DB_DIR. Returned value: $DB_DIR" - exit 1; -fi - -REPLICATE_DIR="$1" -if [[ ! ${REPLICATE_DIR:0:1} == "/" ]]; then -{ - REPLICATE_DIR="`pwd`/$REPLICATE_DIR" -}; fi - -START=$2 -META= - -if [[ $3 == "--meta=attic" ]]; then - META="--keep-attic" -elif [[ $3 == "--meta=yes" || $3 == "--meta" ]]; then - META="--meta" -elif [[ $3 == "--meta=no" ]]; then - META= -else -{ - echo "You must specify --meta=yes or --meta=no" - exit 0 -}; fi - -PRODUCE_DIFF= - -get_replicate_filename() -{ - printf -v TDIGIT3 %03u $(($1 % 1000)) - ARG=$(($1 / 1000)) - printf -v TDIGIT2 %03u $(($ARG % 1000)) - ARG=$(($ARG / 1000)) - printf -v TDIGIT1 %03u $ARG - REPLICATE_TRUNK_DIR=$TDIGIT1/$TDIGIT2/ - REPLICATE_FILENAME=$TDIGIT1/$TDIGIT2/$TDIGIT3 -}; - - -collect_minute_diffs() -{ - TEMP_DIR=$1 - TARGET=$(($START + 1)) - - get_replicate_filename $TARGET - - while [[ ( -s $REPLICATE_DIR/$REPLICATE_FILENAME.state.txt ) && ( $(($START + 1440)) -ge $(($TARGET)) ) && ( `du -m $TEMP_DIR | awk '{ print $1; }'` -le 512 ) ]]; - do - { - printf -v TARGET_FILE %09u $TARGET - gunzip <$REPLICATE_DIR/$REPLICATE_FILENAME.osc.gz >$TEMP_DIR/$TARGET_FILE.osc - TARGET=$(($TARGET + 1)) - get_replicate_filename $TARGET - }; - done - TARGET=$(($TARGET - 1)) -}; - - -apply_minute_diffs() -{ - ./update_from_dir --osc-dir=$1 --version=$DATA_VERSION $META --flush-size=0 - EXITCODE=$? - while [[ $EXITCODE -ne 0 ]]; - do - { - sleep 60 - ./update_from_dir --osc-dir=$1 --version=$DATA_VERSION $META --flush-size=0 - EXITCODE=$? - }; - done - DIFF_COUNT=$(($DIFF_COUNT + 1)) -}; - - -update_state() -{ - get_replicate_filename $TARGET - TIMESTAMP_LINE=`grep "^timestamp" <$REPLICATE_DIR/$REPLICATE_FILENAME.state.txt` - while [[ -z $TIMESTAMP_LINE ]]; do - { - sleep 5 - TIMESTAMP_LINE=`grep "^timestamp" <$REPLICATE_DIR/$REPLICATE_FILENAME.state.txt` - }; done - DATA_VERSION=${TIMESTAMP_LINE:10} -}; - - -echo >>$DB_DIR/apply_osc_to_db.log - -mkdir -p $DB_DIR/augmented_diffs/ -DIFF_COUNT=0 - -# update_state - -pushd "$EXEC_DIR" - -while [[ true ]]; do -{ - if [[ $START == "auto" ]]; then - { - START=`cat $DB_DIR/replicate_id` - }; fi - - echo "`date -u '+%F %T'`: updating from $START" >>$DB_DIR/apply_osc_to_db.log - - TEMP_DIR=`mktemp -d /tmp/osm-3s_update_XXXXXX` - collect_minute_diffs $TEMP_DIR - - if [[ $TARGET -gt $START ]]; then - { - echo "`date -u '+%F %T'`: updating to $TARGET" >>$DB_DIR/apply_osc_to_db.log - - update_state - apply_minute_diffs $TEMP_DIR - echo "$TARGET" >$DB_DIR/replicate_id - - echo "`date -u '+%F %T'`: update complete" $TARGET >>$DB_DIR/apply_osc_to_db.log - }; - else - { - sleep 5 - }; fi - - rm -f $TEMP_DIR/* - rmdir $TEMP_DIR - - START=$TARGET -}; done diff --git a/0.7.54.13/bin/fetch_osc.sh b/0.7.54.13/bin/fetch_osc.sh deleted file mode 100755 index a9d163f..0000000 --- a/0.7.54.13/bin/fetch_osc.sh +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht -# -# This file is part of Overpass_API. -# -# Overpass_API is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Overpass_API is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Overpass_API. If not, see . - -if [[ -z $1 ]]; then -{ - echo Usage: $0 Replicate_id Source_dir Local_dir [Sleep] - exit 0 -}; -fi - -REPLICATE_ID=$1 -SOURCE_DIR=$2 -LOCAL_DIR=$3 -SLEEP_BETWEEN_DLS=15 # How long to sleep between download attempts (sec). Default: 15. See also FAIL_COUNTER_ALERT -FAIL_COUNTER_ALERT=20 # After how many sleep cycles do we get nervous -FAIL_COUNTER=0 -FILE_PANIC= - -if [[ ! -d $LOCAL_DIR ]]; - then { - mkdir $LOCAL_DIR -}; -fi - -if [[ $REPLICATE_ID == "auto" ]] ; then - REPLICATE_ID=`find $LOCAL_DIR -type f -name '*state.txt' -not -size 0 | sort | tail -n 1 | sed -e "s#^$LOCAL_DIR##" -e 's#[./]##g' -e 's#statetxt##' -e 's/^0\+//g'` - - EXEC_DIR="`dirname $0`/" - DB_DIR=`$EXEC_DIR/dispatcher --show-dir` - - if [[ ! -d $DB_DIR ]] ; then - echo "Can't find DB_DIR. Returned value: $DB_DIR" - exit 1; - fi - - if [[ "x$REPLICATE_ID" == "x" ]] ; then - REPLICATE_ID=`cat $DB_DIR/replicate_id` - fi - if [[ "x$REPLICATE_ID" == "x" ]] ; then - echo "Could not determine REPLICATE_ID. Exiting" - exit 1 - fi -fi - -# $1 - remote source -# $2 - local destination -fetch_file() -{ - wget -nv -O "$2" "$1" -}; - -retry_fetch_file() -{ - FILE_PANIC= - FAIL_COUNTER=0 - if [[ ! -s "$2" ]]; then { - fetch_file "$1" "$2" - if [[ "$3" == "gzip" ]]; then { - gunzip -t <"$2" - if [[ $? -ne 0 ]]; then { - rm "$2" - }; fi - }; fi - }; fi - until [[ -s "$2" || $FAIL_COUNTER -ge $FAIL_COUNTER_ALERT ]]; do { - FAIL_COUNTER=$(($FAIL_COUNTER+1)) - sleep $SLEEP_BETWEEN_DLS - fetch_file "$1" "$2" - if [[ "$3" == "gzip" ]]; then { - gunzip -t <"$2" - if [[ $? -ne 0 ]]; then { - rm "$2" - }; fi - }; fi - }; done - if [[ $FAIL_COUNTER -ge $FAIL_COUNTER_ALERT ]]; then - FILE_PANIC="true" - fi -}; - -file_panic() -{ - echo "fetch_osc()@"`date -u "+%F %T"`": upstream_delay $REPLICATE_ID" >>$LOCAL_DIR/fetch_osc.log - REPLICATE_ID=$(($REPLICATE_ID - 1)) - - printf -v TDIGIT3 %03u $(($REPLICATE_ID % 1000)) - ARG=$(($REPLICATE_ID / 1000)) - printf -v TDIGIT2 %03u $(($ARG % 1000)) - ARG=$(($ARG / 1000)) - printf -v TDIGIT1 %03u $ARG - - FILE_PANIC="true" - until [[ ! -n $FILE_PANIC ]]; do { - retry_fetch_file "$REMOTE_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.new.osc.gz" "gzip" - }; done - FILE_PANIC="true" - until [[ ! -n $FILE_PANIC ]]; do { - retry_fetch_file "$REMOTE_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.new.state.txt" "text" - }; done - - RES_GZIP=`diff -q "$LOCAL_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.new.osc.gz"` - RES_TEXT=`diff -q "$LOCAL_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.new.state.txt"` - if [[ -n $RES_GZIP || -n $RES_TEXT ]]; then - echo "fetch_osc()@"`date -u "+%F %T"`": file_panic $REPLICATE_ID" >>$LOCAL_DIR/fetch_osc.log - echo "fetch_osc()@"`date -u "+%F %T"`": $RES_GZIP" >>$LOCAL_DIR/fetch_osc.log - echo "fetch_osc()@"`date -u "+%F %T"`": $RES_TEXT" >>$LOCAL_DIR/fetch_osc.log - exit 1 - fi - - rm "$LOCAL_PATH/$TDIGIT3.new.osc.gz" - rm "$LOCAL_PATH/$TDIGIT3.new.state.txt" -}; - -fetch_minute_diff() -{ - printf -v TDIGIT3 %03u $(($REPLICATE_ID % 1000)) - ARG=$(($REPLICATE_ID / 1000)) - printf -v TDIGIT2 %03u $(($ARG % 1000)) - ARG=$(($ARG / 1000)) - printf -v TDIGIT1 %03u $ARG - - LOCAL_PATH="$LOCAL_DIR/$TDIGIT1/$TDIGIT2" - REMOTE_PATH="$SOURCE_DIR/$TDIGIT1/$TDIGIT2" - mkdir -p "$LOCAL_DIR/$TDIGIT1/$TDIGIT2" - - retry_fetch_file "$REMOTE_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.osc.gz" "gzip" - if [[ -n $FILE_PANIC ]]; then - file_panic - fi - retry_fetch_file "$REMOTE_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.state.txt" "text" - if [[ -n $FILE_PANIC ]]; then - file_panic - fi - - TIMESTAMP_LINE=`grep timestamp $LOCAL_DIR/$TDIGIT1/$TDIGIT2/$TDIGIT3.state.txt` - TIMESTAMP=${TIMESTAMP_LINE:10} -}; - -while [[ true ]]; -do -{ - REPLICATE_ID=$(($REPLICATE_ID + 1)) - fetch_minute_diff - echo "fetch_osc()@"`date -u "+%F %T"`": new_replicate_diff $REPLICATE_ID $TIMESTAMP" >>$LOCAL_DIR/fetch_osc.log - sleep 1 -}; -done diff --git a/0.7.54.13/bin/update_overpass.sh b/0.7.54.13/bin/update_overpass.sh new file mode 100644 index 0000000..328e375 --- /dev/null +++ b/0.7.54.13/bin/update_overpass.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env sh + +( + set -e + + if [ "x$OVERPASS_META" = "xattic" ] ; then + META="--keep-attic" + elif [ "x$OVERPASS_META" = "xyes" ] ; then + META="--meta" + else + META="" + fi + + if [ ! -d /db/diffs ] ; then + mkdir /db/diffs + fi + + while `true` ; do + ( + if [ ! -e /db/diffs/changes.osm ] ; then + ! /app/venv/bin/pyosmium-get-changes --server $OVERPASS_DIFF_URL -o /db/diffs/changes.osm -f /db/replicate_id + OSMIUM_STATUS=$? + if [ $OSMIUM_STATUS -eq 1 ]; then + echo "There are still some updates remainging" + fi + if [ $OSMIUM_STATUS -eq 2 ]; then + echo "Failure downloading updates" + exit 0 + fi + else + echo "/db/diffs/changes.osm exists. Trying to apply again." + fi + /app/bin/update_from_dir --osc-dir=/db/diffs/ $META + rm /db/diffs/changes.osm + ) + sleep 60 + done +) 2>&1 | tee -a /db/changes.log \ No newline at end of file diff --git a/0.7.54.13/docker-entrypoint.sh b/0.7.54.13/docker-entrypoint.sh index 5a43603..cb69627 100644 --- a/0.7.54.13/docker-entrypoint.sh +++ b/0.7.54.13/docker-entrypoint.sh @@ -6,13 +6,15 @@ OVERPASS_META=${OVERPASS_META:-no} OVERPASS_MODE=${OVERPASS_MODE:-clone} OVERPASS_COMPRESSION=${OVERPASS_COMPRESSION:-gz} +if [[ "$OVERPASS_META" == "attic" ]] ; then + META="--keep-attic" +else + META="--meta" +fi + + if [ ! -d /db/db ] ; then if [ "$OVERPASS_MODE" = "clone" ]; then - if [[ "$OVERPASS_META" == "attic" ]] ; then - META="--keep-attic" - else - META="--meta=$OVERPASS_META" - fi mkdir -p /db/db \ && /app/bin/download_clone.sh --db-dir=/db/db --source=http://dev.overpass-api.de/api_drolbr/ $META \ && cp -r /app/etc/rules /db/db \ @@ -22,10 +24,21 @@ if [ ! -d /db/db ] ; then fi if [ "$OVERPASS_MODE" = "init" ]; then - lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet; exit" \ - && /app/bin/init_osm3s.sh /db/planet /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION" \ - && echo $OVERPASS_PLANET_SEQUENCE_ID > /db/db/replicate_id \ - && rm /db/planet \ + lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet.osm.bz2; exit" \ + && /app/bin/init_osm3s.sh /db/planet.osm.bz2 /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION --map-compression-method=$OVERPASS_COMPRESSION"\ + && echo "Database created. Now updating it." && ( + ! /app/venv/bin/pyosmium-get-changes -O /db/planet.osm.bz2 --server $OVERPASS_DIFF_URL -o /db/changes.osm -f /db/replicate_id + OSMIUM_STATUS=$? + if [ $OSMIUM_STATUS -eq 1 ]; then + echo "There are still some updates remainging" + fi + if [ $OSMIUM_STATUS -eq 2 ]; then + echo "Failure downloading updates" + exit 0 + fi + (cat /db/changes.osm | /app/bin/update_database --db-dir=/db/db $META --compression-method=$OVERPASS_COMPRESSION) 2>&1 | tee -a /db/changes.log + ) \ + && rm /db/planet.osm.bz2 /db/changes.osm \ && cp -r /app/etc/rules /db/db \ && chown -R overpass:overpass /db \ && echo "Overpass ready, you can start your container with docker start" diff --git a/0.7.54.13/etc/nginx-overpass.conf b/0.7.54.13/etc/nginx-overpass.conf index c019615..a19dde4 100644 --- a/0.7.54.13/etc/nginx-overpass.conf +++ b/0.7.54.13/etc/nginx-overpass.conf @@ -25,7 +25,7 @@ http { sendfile on; #tcp_nopush on; - keepalive_timeout 65; + keepalive_timeout 1000s; #gzip on; @@ -55,6 +55,7 @@ http { fastcgi_param SCRIPT_FILENAME /app/$fastcgi_script_name; fastcgi_param PATH_INFO $fastcgi_path_info; fastcgi_pass unix:/nginx/fcgiwrap.socket; + fastcgi_read_timeout 1000s; } location /api/ { diff --git a/0.7.54.13/etc/supervisord.conf b/0.7.54.13/etc/supervisord.conf index 6c4623b..fc67b74 100644 --- a/0.7.54.13/etc/supervisord.conf +++ b/0.7.54.13/etc/supervisord.conf @@ -17,18 +17,12 @@ redirect_stderr=true user=nginx priority=3 -[program:fetch_diff] -command=/app/bin/fetch_osc.sh auto %(ENV_OVERPASS_DIFF_URL)s /db/diffs +[program:update_overpass] +command=/app/bin/update_overpass.sh user=overpass redirect_stderr=true priority=5 -[program:apply_diff] -command=/app/bin/apply_osc_to_db.sh /db/diffs/ auto --meta=%(ENV_OVERPASS_META)s -redirect_stderr=true -user=overpass -priority=4 - [program:dispatcher_areas] command=nice /app/bin/dispatcher --areas --db-dir="/db/db" user=overpass diff --git a/0.7.54.13/requirements.txt b/0.7.54.13/requirements.txt new file mode 100644 index 0000000..8314012 --- /dev/null +++ b/0.7.54.13/requirements.txt @@ -0,0 +1 @@ +osmium diff --git a/0.7.54.9/Dockerfile b/0.7.54.9/Dockerfile index b557fc2..f680449 100644 --- a/0.7.54.9/Dockerfile +++ b/0.7.54.9/Dockerfile @@ -1,27 +1,34 @@ -FROM nginx:stable +FROM nginx:1.14 RUN addgroup overpass && adduser --home /db --disabled-password --gecos overpass --ingroup overpass overpass +ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.54.9.tar.gz /app/src.tar.gz + RUN apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ autoconf \ automake \ + bash \ + bzip2 \ ca-certificates \ expat \ - libexpat1-dev \ + fcgiwrap \ g++ \ + lftp \ + libexpat1-dev \ liblz4-1 \ liblz4-dev \ libtool \ m4 \ make \ + osmium-tool \ + python3 \ + python3-venv \ + supervisor \ + wget \ zlib1g \ - zlib1g-dev - - -ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.54.9.tar.gz /app/src.tar.gz - -RUN mkdir -p /app/src \ + zlib1g-dev \ + && mkdir -p /app/src \ && cd /app/src \ && tar -x -z --strip-components 1 -f ../src.tar.gz \ && autoscan \ @@ -45,26 +52,28 @@ RUN mkdir -p /app/src \ && apt-get autoremove -y \ && mkdir -p /db/diffs /app/etc \ && cp -r /app/src/rules /app/etc/rules \ - && rm -rf /app/src /app/src.tar.gz + && rm -rf /app/src /app/src.tar.gz \ + && rm -rf /var/lib/apt/lists/* -RUN apt-get install --no-install-recommends --no-install-suggests -y \ - supervisor \ - bash \ - lftp \ - wget \ - fcgiwrap \ - bzip2 +COPY requirements.txt /app/ +RUN python3 -m venv /app/venv \ + && /app/venv/bin/pip install -r /app/requirements.txt RUN mkdir /nginx && chown nginx:nginx /nginx && chown -R overpass:overpass /db + COPY etc/supervisord.conf /etc/supervisor/conf.d/supervisord.conf + COPY etc/nginx-overpass.conf /etc/nginx/nginx.conf -COPY bin/apply_osc_to_db.sh bin/fetch_osc.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/ + +COPY bin/update_overpass.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/ + COPY docker-entrypoint.sh /app/ -RUN chmod a+x /app/docker-entrypoint.sh /app/bin/apply_osc_to_db.sh /app/bin/fetch_osc.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh -VOLUME /db + +RUN chmod a+x /app/docker-entrypoint.sh /app/bin/update_overpass.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh ENV OVERPASS_RULES_LOAD 1 + EXPOSE 80 # CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"] CMD ["/app/docker-entrypoint.sh"] diff --git a/0.7.54.9/bin/apply_osc_to_db.sh b/0.7.54.9/bin/apply_osc_to_db.sh deleted file mode 100755 index 94d1879..0000000 --- a/0.7.54.9/bin/apply_osc_to_db.sh +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht -# -# This file is part of Overpass_API. -# -# Overpass_API is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Overpass_API is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Overpass_API. If not, see . - -if [[ -z $3 ]]; then -{ - echo "Usage: $0 replicate_dir start_id --meta=(attic|yes|no)" - exit 0 -}; fi - -EXEC_DIR="`dirname $0`/" -if [[ ! ${EXEC_DIR:0:1} == "/" ]]; then -{ - EXEC_DIR="`pwd`/$EXEC_DIR" -}; fi - -DB_DIR=`$EXEC_DIR/dispatcher --show-dir` - -if [[ ! -d $DB_DIR ]] ; then - echo "Can't find DB_DIR. Returned value: $DB_DIR" - exit 1; -fi - -REPLICATE_DIR="$1" -if [[ ! ${REPLICATE_DIR:0:1} == "/" ]]; then -{ - REPLICATE_DIR="`pwd`/$REPLICATE_DIR" -}; fi - -START=$2 -META= - -if [[ $3 == "--meta=attic" ]]; then - META="--keep-attic" -elif [[ $3 == "--meta=yes" || $3 == "--meta" ]]; then - META="--meta" -elif [[ $3 == "--meta=no" ]]; then - META= -else -{ - echo "You must specify --meta=yes or --meta=no" - exit 0 -}; fi - -PRODUCE_DIFF= - -get_replicate_filename() -{ - printf -v TDIGIT3 %03u $(($1 % 1000)) - ARG=$(($1 / 1000)) - printf -v TDIGIT2 %03u $(($ARG % 1000)) - ARG=$(($ARG / 1000)) - printf -v TDIGIT1 %03u $ARG - REPLICATE_TRUNK_DIR=$TDIGIT1/$TDIGIT2/ - REPLICATE_FILENAME=$TDIGIT1/$TDIGIT2/$TDIGIT3 -}; - - -collect_minute_diffs() -{ - TEMP_DIR=$1 - TARGET=$(($START + 1)) - - get_replicate_filename $TARGET - - while [[ ( -s $REPLICATE_DIR/$REPLICATE_FILENAME.state.txt ) && ( $(($START + 1440)) -ge $(($TARGET)) ) && ( `du -m $TEMP_DIR | awk '{ print $1; }'` -le 512 ) ]]; - do - { - printf -v TARGET_FILE %09u $TARGET - gunzip <$REPLICATE_DIR/$REPLICATE_FILENAME.osc.gz >$TEMP_DIR/$TARGET_FILE.osc - TARGET=$(($TARGET + 1)) - get_replicate_filename $TARGET - }; - done - TARGET=$(($TARGET - 1)) -}; - - -apply_minute_diffs() -{ - ./update_from_dir --osc-dir=$1 --version=$DATA_VERSION $META --flush-size=0 - EXITCODE=$? - while [[ $EXITCODE -ne 0 ]]; - do - { - sleep 60 - ./update_from_dir --osc-dir=$1 --version=$DATA_VERSION $META --flush-size=0 - EXITCODE=$? - }; - done - DIFF_COUNT=$(($DIFF_COUNT + 1)) -}; - - -update_state() -{ - get_replicate_filename $TARGET - TIMESTAMP_LINE=`grep "^timestamp" <$REPLICATE_DIR/$REPLICATE_FILENAME.state.txt` - while [[ -z $TIMESTAMP_LINE ]]; do - { - sleep 5 - TIMESTAMP_LINE=`grep "^timestamp" <$REPLICATE_DIR/$REPLICATE_FILENAME.state.txt` - }; done - DATA_VERSION=${TIMESTAMP_LINE:10} -}; - - -echo >>$DB_DIR/apply_osc_to_db.log - -mkdir -p $DB_DIR/augmented_diffs/ -DIFF_COUNT=0 - -# update_state - -pushd "$EXEC_DIR" - -while [[ true ]]; do -{ - if [[ $START == "auto" ]]; then - { - START=`cat $DB_DIR/replicate_id` - }; fi - - echo "`date -u '+%F %T'`: updating from $START" >>$DB_DIR/apply_osc_to_db.log - - TEMP_DIR=`mktemp -d /tmp/osm-3s_update_XXXXXX` - collect_minute_diffs $TEMP_DIR - - if [[ $TARGET -gt $START ]]; then - { - echo "`date -u '+%F %T'`: updating to $TARGET" >>$DB_DIR/apply_osc_to_db.log - - update_state - apply_minute_diffs $TEMP_DIR - echo "$TARGET" >$DB_DIR/replicate_id - - echo "`date -u '+%F %T'`: update complete" $TARGET >>$DB_DIR/apply_osc_to_db.log - }; - else - { - sleep 5 - }; fi - - rm -f $TEMP_DIR/* - rmdir $TEMP_DIR - - START=$TARGET -}; done diff --git a/0.7.54.9/bin/fetch_osc.sh b/0.7.54.9/bin/fetch_osc.sh deleted file mode 100755 index a9d163f..0000000 --- a/0.7.54.9/bin/fetch_osc.sh +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht -# -# This file is part of Overpass_API. -# -# Overpass_API is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Overpass_API is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Overpass_API. If not, see . - -if [[ -z $1 ]]; then -{ - echo Usage: $0 Replicate_id Source_dir Local_dir [Sleep] - exit 0 -}; -fi - -REPLICATE_ID=$1 -SOURCE_DIR=$2 -LOCAL_DIR=$3 -SLEEP_BETWEEN_DLS=15 # How long to sleep between download attempts (sec). Default: 15. See also FAIL_COUNTER_ALERT -FAIL_COUNTER_ALERT=20 # After how many sleep cycles do we get nervous -FAIL_COUNTER=0 -FILE_PANIC= - -if [[ ! -d $LOCAL_DIR ]]; - then { - mkdir $LOCAL_DIR -}; -fi - -if [[ $REPLICATE_ID == "auto" ]] ; then - REPLICATE_ID=`find $LOCAL_DIR -type f -name '*state.txt' -not -size 0 | sort | tail -n 1 | sed -e "s#^$LOCAL_DIR##" -e 's#[./]##g' -e 's#statetxt##' -e 's/^0\+//g'` - - EXEC_DIR="`dirname $0`/" - DB_DIR=`$EXEC_DIR/dispatcher --show-dir` - - if [[ ! -d $DB_DIR ]] ; then - echo "Can't find DB_DIR. Returned value: $DB_DIR" - exit 1; - fi - - if [[ "x$REPLICATE_ID" == "x" ]] ; then - REPLICATE_ID=`cat $DB_DIR/replicate_id` - fi - if [[ "x$REPLICATE_ID" == "x" ]] ; then - echo "Could not determine REPLICATE_ID. Exiting" - exit 1 - fi -fi - -# $1 - remote source -# $2 - local destination -fetch_file() -{ - wget -nv -O "$2" "$1" -}; - -retry_fetch_file() -{ - FILE_PANIC= - FAIL_COUNTER=0 - if [[ ! -s "$2" ]]; then { - fetch_file "$1" "$2" - if [[ "$3" == "gzip" ]]; then { - gunzip -t <"$2" - if [[ $? -ne 0 ]]; then { - rm "$2" - }; fi - }; fi - }; fi - until [[ -s "$2" || $FAIL_COUNTER -ge $FAIL_COUNTER_ALERT ]]; do { - FAIL_COUNTER=$(($FAIL_COUNTER+1)) - sleep $SLEEP_BETWEEN_DLS - fetch_file "$1" "$2" - if [[ "$3" == "gzip" ]]; then { - gunzip -t <"$2" - if [[ $? -ne 0 ]]; then { - rm "$2" - }; fi - }; fi - }; done - if [[ $FAIL_COUNTER -ge $FAIL_COUNTER_ALERT ]]; then - FILE_PANIC="true" - fi -}; - -file_panic() -{ - echo "fetch_osc()@"`date -u "+%F %T"`": upstream_delay $REPLICATE_ID" >>$LOCAL_DIR/fetch_osc.log - REPLICATE_ID=$(($REPLICATE_ID - 1)) - - printf -v TDIGIT3 %03u $(($REPLICATE_ID % 1000)) - ARG=$(($REPLICATE_ID / 1000)) - printf -v TDIGIT2 %03u $(($ARG % 1000)) - ARG=$(($ARG / 1000)) - printf -v TDIGIT1 %03u $ARG - - FILE_PANIC="true" - until [[ ! -n $FILE_PANIC ]]; do { - retry_fetch_file "$REMOTE_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.new.osc.gz" "gzip" - }; done - FILE_PANIC="true" - until [[ ! -n $FILE_PANIC ]]; do { - retry_fetch_file "$REMOTE_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.new.state.txt" "text" - }; done - - RES_GZIP=`diff -q "$LOCAL_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.new.osc.gz"` - RES_TEXT=`diff -q "$LOCAL_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.new.state.txt"` - if [[ -n $RES_GZIP || -n $RES_TEXT ]]; then - echo "fetch_osc()@"`date -u "+%F %T"`": file_panic $REPLICATE_ID" >>$LOCAL_DIR/fetch_osc.log - echo "fetch_osc()@"`date -u "+%F %T"`": $RES_GZIP" >>$LOCAL_DIR/fetch_osc.log - echo "fetch_osc()@"`date -u "+%F %T"`": $RES_TEXT" >>$LOCAL_DIR/fetch_osc.log - exit 1 - fi - - rm "$LOCAL_PATH/$TDIGIT3.new.osc.gz" - rm "$LOCAL_PATH/$TDIGIT3.new.state.txt" -}; - -fetch_minute_diff() -{ - printf -v TDIGIT3 %03u $(($REPLICATE_ID % 1000)) - ARG=$(($REPLICATE_ID / 1000)) - printf -v TDIGIT2 %03u $(($ARG % 1000)) - ARG=$(($ARG / 1000)) - printf -v TDIGIT1 %03u $ARG - - LOCAL_PATH="$LOCAL_DIR/$TDIGIT1/$TDIGIT2" - REMOTE_PATH="$SOURCE_DIR/$TDIGIT1/$TDIGIT2" - mkdir -p "$LOCAL_DIR/$TDIGIT1/$TDIGIT2" - - retry_fetch_file "$REMOTE_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.osc.gz" "gzip" - if [[ -n $FILE_PANIC ]]; then - file_panic - fi - retry_fetch_file "$REMOTE_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.state.txt" "text" - if [[ -n $FILE_PANIC ]]; then - file_panic - fi - - TIMESTAMP_LINE=`grep timestamp $LOCAL_DIR/$TDIGIT1/$TDIGIT2/$TDIGIT3.state.txt` - TIMESTAMP=${TIMESTAMP_LINE:10} -}; - -while [[ true ]]; -do -{ - REPLICATE_ID=$(($REPLICATE_ID + 1)) - fetch_minute_diff - echo "fetch_osc()@"`date -u "+%F %T"`": new_replicate_diff $REPLICATE_ID $TIMESTAMP" >>$LOCAL_DIR/fetch_osc.log - sleep 1 -}; -done diff --git a/0.7.54.9/bin/update_overpass.sh b/0.7.54.9/bin/update_overpass.sh new file mode 100644 index 0000000..328e375 --- /dev/null +++ b/0.7.54.9/bin/update_overpass.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env sh + +( + set -e + + if [ "x$OVERPASS_META" = "xattic" ] ; then + META="--keep-attic" + elif [ "x$OVERPASS_META" = "xyes" ] ; then + META="--meta" + else + META="" + fi + + if [ ! -d /db/diffs ] ; then + mkdir /db/diffs + fi + + while `true` ; do + ( + if [ ! -e /db/diffs/changes.osm ] ; then + ! /app/venv/bin/pyosmium-get-changes --server $OVERPASS_DIFF_URL -o /db/diffs/changes.osm -f /db/replicate_id + OSMIUM_STATUS=$? + if [ $OSMIUM_STATUS -eq 1 ]; then + echo "There are still some updates remainging" + fi + if [ $OSMIUM_STATUS -eq 2 ]; then + echo "Failure downloading updates" + exit 0 + fi + else + echo "/db/diffs/changes.osm exists. Trying to apply again." + fi + /app/bin/update_from_dir --osc-dir=/db/diffs/ $META + rm /db/diffs/changes.osm + ) + sleep 60 + done +) 2>&1 | tee -a /db/changes.log \ No newline at end of file diff --git a/0.7.54.9/docker-entrypoint.sh b/0.7.54.9/docker-entrypoint.sh index 5a43603..cb69627 100644 --- a/0.7.54.9/docker-entrypoint.sh +++ b/0.7.54.9/docker-entrypoint.sh @@ -6,13 +6,15 @@ OVERPASS_META=${OVERPASS_META:-no} OVERPASS_MODE=${OVERPASS_MODE:-clone} OVERPASS_COMPRESSION=${OVERPASS_COMPRESSION:-gz} +if [[ "$OVERPASS_META" == "attic" ]] ; then + META="--keep-attic" +else + META="--meta" +fi + + if [ ! -d /db/db ] ; then if [ "$OVERPASS_MODE" = "clone" ]; then - if [[ "$OVERPASS_META" == "attic" ]] ; then - META="--keep-attic" - else - META="--meta=$OVERPASS_META" - fi mkdir -p /db/db \ && /app/bin/download_clone.sh --db-dir=/db/db --source=http://dev.overpass-api.de/api_drolbr/ $META \ && cp -r /app/etc/rules /db/db \ @@ -22,10 +24,21 @@ if [ ! -d /db/db ] ; then fi if [ "$OVERPASS_MODE" = "init" ]; then - lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet; exit" \ - && /app/bin/init_osm3s.sh /db/planet /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION" \ - && echo $OVERPASS_PLANET_SEQUENCE_ID > /db/db/replicate_id \ - && rm /db/planet \ + lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet.osm.bz2; exit" \ + && /app/bin/init_osm3s.sh /db/planet.osm.bz2 /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION --map-compression-method=$OVERPASS_COMPRESSION"\ + && echo "Database created. Now updating it." && ( + ! /app/venv/bin/pyosmium-get-changes -O /db/planet.osm.bz2 --server $OVERPASS_DIFF_URL -o /db/changes.osm -f /db/replicate_id + OSMIUM_STATUS=$? + if [ $OSMIUM_STATUS -eq 1 ]; then + echo "There are still some updates remainging" + fi + if [ $OSMIUM_STATUS -eq 2 ]; then + echo "Failure downloading updates" + exit 0 + fi + (cat /db/changes.osm | /app/bin/update_database --db-dir=/db/db $META --compression-method=$OVERPASS_COMPRESSION) 2>&1 | tee -a /db/changes.log + ) \ + && rm /db/planet.osm.bz2 /db/changes.osm \ && cp -r /app/etc/rules /db/db \ && chown -R overpass:overpass /db \ && echo "Overpass ready, you can start your container with docker start" diff --git a/0.7.54.9/etc/nginx-overpass.conf b/0.7.54.9/etc/nginx-overpass.conf index c019615..a19dde4 100644 --- a/0.7.54.9/etc/nginx-overpass.conf +++ b/0.7.54.9/etc/nginx-overpass.conf @@ -25,7 +25,7 @@ http { sendfile on; #tcp_nopush on; - keepalive_timeout 65; + keepalive_timeout 1000s; #gzip on; @@ -55,6 +55,7 @@ http { fastcgi_param SCRIPT_FILENAME /app/$fastcgi_script_name; fastcgi_param PATH_INFO $fastcgi_path_info; fastcgi_pass unix:/nginx/fcgiwrap.socket; + fastcgi_read_timeout 1000s; } location /api/ { diff --git a/0.7.54.9/etc/supervisord.conf b/0.7.54.9/etc/supervisord.conf index 6c4623b..fc67b74 100644 --- a/0.7.54.9/etc/supervisord.conf +++ b/0.7.54.9/etc/supervisord.conf @@ -17,18 +17,12 @@ redirect_stderr=true user=nginx priority=3 -[program:fetch_diff] -command=/app/bin/fetch_osc.sh auto %(ENV_OVERPASS_DIFF_URL)s /db/diffs +[program:update_overpass] +command=/app/bin/update_overpass.sh user=overpass redirect_stderr=true priority=5 -[program:apply_diff] -command=/app/bin/apply_osc_to_db.sh /db/diffs/ auto --meta=%(ENV_OVERPASS_META)s -redirect_stderr=true -user=overpass -priority=4 - [program:dispatcher_areas] command=nice /app/bin/dispatcher --areas --db-dir="/db/db" user=overpass diff --git a/0.7.54.9/requirements.txt b/0.7.54.9/requirements.txt new file mode 100644 index 0000000..8314012 --- /dev/null +++ b/0.7.54.9/requirements.txt @@ -0,0 +1 @@ +osmium diff --git a/0.7.54/Dockerfile b/0.7.54/Dockerfile index b6f8025..85cca11 100644 --- a/0.7.54/Dockerfile +++ b/0.7.54/Dockerfile @@ -1,27 +1,34 @@ -FROM nginx:stable +FROM nginx:1.14 RUN addgroup overpass && adduser --home /db --disabled-password --gecos overpass --ingroup overpass overpass +ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.54.tar.gz /app/src.tar.gz + RUN apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ autoconf \ automake \ + bash \ + bzip2 \ ca-certificates \ expat \ - libexpat1-dev \ + fcgiwrap \ g++ \ + lftp \ + libexpat1-dev \ liblz4-1 \ liblz4-dev \ libtool \ m4 \ make \ + osmium-tool \ + python3 \ + python3-venv \ + supervisor \ + wget \ zlib1g \ - zlib1g-dev - - -ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.54.tar.gz /app/src.tar.gz - -RUN mkdir -p /app/src \ + zlib1g-dev \ + && mkdir -p /app/src \ && cd /app/src \ && tar -x -z --strip-components 1 -f ../src.tar.gz \ && autoscan \ @@ -45,26 +52,28 @@ RUN mkdir -p /app/src \ && apt-get autoremove -y \ && mkdir -p /db/diffs /app/etc \ && cp -r /app/src/rules /app/etc/rules \ - && rm -rf /app/src /app/src.tar.gz + && rm -rf /app/src /app/src.tar.gz \ + && rm -rf /var/lib/apt/lists/* -RUN apt-get install --no-install-recommends --no-install-suggests -y \ - supervisor \ - bash \ - lftp \ - wget \ - fcgiwrap \ - bzip2 +COPY requirements.txt /app/ +RUN python3 -m venv /app/venv \ + && /app/venv/bin/pip install -r /app/requirements.txt RUN mkdir /nginx && chown nginx:nginx /nginx && chown -R overpass:overpass /db + COPY etc/supervisord.conf /etc/supervisor/conf.d/supervisord.conf + COPY etc/nginx-overpass.conf /etc/nginx/nginx.conf -COPY bin/apply_osc_to_db.sh bin/fetch_osc.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/ + +COPY bin/update_overpass.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/ + COPY docker-entrypoint.sh /app/ -RUN chmod a+x /app/docker-entrypoint.sh /app/bin/apply_osc_to_db.sh /app/bin/fetch_osc.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh -VOLUME /db + +RUN chmod a+x /app/docker-entrypoint.sh /app/bin/update_overpass.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh ENV OVERPASS_RULES_LOAD 1 + EXPOSE 80 # CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"] CMD ["/app/docker-entrypoint.sh"] diff --git a/0.7.54/bin/apply_osc_to_db.sh b/0.7.54/bin/apply_osc_to_db.sh deleted file mode 100755 index 94d1879..0000000 --- a/0.7.54/bin/apply_osc_to_db.sh +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht -# -# This file is part of Overpass_API. -# -# Overpass_API is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Overpass_API is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Overpass_API. If not, see . - -if [[ -z $3 ]]; then -{ - echo "Usage: $0 replicate_dir start_id --meta=(attic|yes|no)" - exit 0 -}; fi - -EXEC_DIR="`dirname $0`/" -if [[ ! ${EXEC_DIR:0:1} == "/" ]]; then -{ - EXEC_DIR="`pwd`/$EXEC_DIR" -}; fi - -DB_DIR=`$EXEC_DIR/dispatcher --show-dir` - -if [[ ! -d $DB_DIR ]] ; then - echo "Can't find DB_DIR. Returned value: $DB_DIR" - exit 1; -fi - -REPLICATE_DIR="$1" -if [[ ! ${REPLICATE_DIR:0:1} == "/" ]]; then -{ - REPLICATE_DIR="`pwd`/$REPLICATE_DIR" -}; fi - -START=$2 -META= - -if [[ $3 == "--meta=attic" ]]; then - META="--keep-attic" -elif [[ $3 == "--meta=yes" || $3 == "--meta" ]]; then - META="--meta" -elif [[ $3 == "--meta=no" ]]; then - META= -else -{ - echo "You must specify --meta=yes or --meta=no" - exit 0 -}; fi - -PRODUCE_DIFF= - -get_replicate_filename() -{ - printf -v TDIGIT3 %03u $(($1 % 1000)) - ARG=$(($1 / 1000)) - printf -v TDIGIT2 %03u $(($ARG % 1000)) - ARG=$(($ARG / 1000)) - printf -v TDIGIT1 %03u $ARG - REPLICATE_TRUNK_DIR=$TDIGIT1/$TDIGIT2/ - REPLICATE_FILENAME=$TDIGIT1/$TDIGIT2/$TDIGIT3 -}; - - -collect_minute_diffs() -{ - TEMP_DIR=$1 - TARGET=$(($START + 1)) - - get_replicate_filename $TARGET - - while [[ ( -s $REPLICATE_DIR/$REPLICATE_FILENAME.state.txt ) && ( $(($START + 1440)) -ge $(($TARGET)) ) && ( `du -m $TEMP_DIR | awk '{ print $1; }'` -le 512 ) ]]; - do - { - printf -v TARGET_FILE %09u $TARGET - gunzip <$REPLICATE_DIR/$REPLICATE_FILENAME.osc.gz >$TEMP_DIR/$TARGET_FILE.osc - TARGET=$(($TARGET + 1)) - get_replicate_filename $TARGET - }; - done - TARGET=$(($TARGET - 1)) -}; - - -apply_minute_diffs() -{ - ./update_from_dir --osc-dir=$1 --version=$DATA_VERSION $META --flush-size=0 - EXITCODE=$? - while [[ $EXITCODE -ne 0 ]]; - do - { - sleep 60 - ./update_from_dir --osc-dir=$1 --version=$DATA_VERSION $META --flush-size=0 - EXITCODE=$? - }; - done - DIFF_COUNT=$(($DIFF_COUNT + 1)) -}; - - -update_state() -{ - get_replicate_filename $TARGET - TIMESTAMP_LINE=`grep "^timestamp" <$REPLICATE_DIR/$REPLICATE_FILENAME.state.txt` - while [[ -z $TIMESTAMP_LINE ]]; do - { - sleep 5 - TIMESTAMP_LINE=`grep "^timestamp" <$REPLICATE_DIR/$REPLICATE_FILENAME.state.txt` - }; done - DATA_VERSION=${TIMESTAMP_LINE:10} -}; - - -echo >>$DB_DIR/apply_osc_to_db.log - -mkdir -p $DB_DIR/augmented_diffs/ -DIFF_COUNT=0 - -# update_state - -pushd "$EXEC_DIR" - -while [[ true ]]; do -{ - if [[ $START == "auto" ]]; then - { - START=`cat $DB_DIR/replicate_id` - }; fi - - echo "`date -u '+%F %T'`: updating from $START" >>$DB_DIR/apply_osc_to_db.log - - TEMP_DIR=`mktemp -d /tmp/osm-3s_update_XXXXXX` - collect_minute_diffs $TEMP_DIR - - if [[ $TARGET -gt $START ]]; then - { - echo "`date -u '+%F %T'`: updating to $TARGET" >>$DB_DIR/apply_osc_to_db.log - - update_state - apply_minute_diffs $TEMP_DIR - echo "$TARGET" >$DB_DIR/replicate_id - - echo "`date -u '+%F %T'`: update complete" $TARGET >>$DB_DIR/apply_osc_to_db.log - }; - else - { - sleep 5 - }; fi - - rm -f $TEMP_DIR/* - rmdir $TEMP_DIR - - START=$TARGET -}; done diff --git a/0.7.54/bin/fetch_osc.sh b/0.7.54/bin/fetch_osc.sh deleted file mode 100755 index a9d163f..0000000 --- a/0.7.54/bin/fetch_osc.sh +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht -# -# This file is part of Overpass_API. -# -# Overpass_API is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Overpass_API is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Overpass_API. If not, see . - -if [[ -z $1 ]]; then -{ - echo Usage: $0 Replicate_id Source_dir Local_dir [Sleep] - exit 0 -}; -fi - -REPLICATE_ID=$1 -SOURCE_DIR=$2 -LOCAL_DIR=$3 -SLEEP_BETWEEN_DLS=15 # How long to sleep between download attempts (sec). Default: 15. See also FAIL_COUNTER_ALERT -FAIL_COUNTER_ALERT=20 # After how many sleep cycles do we get nervous -FAIL_COUNTER=0 -FILE_PANIC= - -if [[ ! -d $LOCAL_DIR ]]; - then { - mkdir $LOCAL_DIR -}; -fi - -if [[ $REPLICATE_ID == "auto" ]] ; then - REPLICATE_ID=`find $LOCAL_DIR -type f -name '*state.txt' -not -size 0 | sort | tail -n 1 | sed -e "s#^$LOCAL_DIR##" -e 's#[./]##g' -e 's#statetxt##' -e 's/^0\+//g'` - - EXEC_DIR="`dirname $0`/" - DB_DIR=`$EXEC_DIR/dispatcher --show-dir` - - if [[ ! -d $DB_DIR ]] ; then - echo "Can't find DB_DIR. Returned value: $DB_DIR" - exit 1; - fi - - if [[ "x$REPLICATE_ID" == "x" ]] ; then - REPLICATE_ID=`cat $DB_DIR/replicate_id` - fi - if [[ "x$REPLICATE_ID" == "x" ]] ; then - echo "Could not determine REPLICATE_ID. Exiting" - exit 1 - fi -fi - -# $1 - remote source -# $2 - local destination -fetch_file() -{ - wget -nv -O "$2" "$1" -}; - -retry_fetch_file() -{ - FILE_PANIC= - FAIL_COUNTER=0 - if [[ ! -s "$2" ]]; then { - fetch_file "$1" "$2" - if [[ "$3" == "gzip" ]]; then { - gunzip -t <"$2" - if [[ $? -ne 0 ]]; then { - rm "$2" - }; fi - }; fi - }; fi - until [[ -s "$2" || $FAIL_COUNTER -ge $FAIL_COUNTER_ALERT ]]; do { - FAIL_COUNTER=$(($FAIL_COUNTER+1)) - sleep $SLEEP_BETWEEN_DLS - fetch_file "$1" "$2" - if [[ "$3" == "gzip" ]]; then { - gunzip -t <"$2" - if [[ $? -ne 0 ]]; then { - rm "$2" - }; fi - }; fi - }; done - if [[ $FAIL_COUNTER -ge $FAIL_COUNTER_ALERT ]]; then - FILE_PANIC="true" - fi -}; - -file_panic() -{ - echo "fetch_osc()@"`date -u "+%F %T"`": upstream_delay $REPLICATE_ID" >>$LOCAL_DIR/fetch_osc.log - REPLICATE_ID=$(($REPLICATE_ID - 1)) - - printf -v TDIGIT3 %03u $(($REPLICATE_ID % 1000)) - ARG=$(($REPLICATE_ID / 1000)) - printf -v TDIGIT2 %03u $(($ARG % 1000)) - ARG=$(($ARG / 1000)) - printf -v TDIGIT1 %03u $ARG - - FILE_PANIC="true" - until [[ ! -n $FILE_PANIC ]]; do { - retry_fetch_file "$REMOTE_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.new.osc.gz" "gzip" - }; done - FILE_PANIC="true" - until [[ ! -n $FILE_PANIC ]]; do { - retry_fetch_file "$REMOTE_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.new.state.txt" "text" - }; done - - RES_GZIP=`diff -q "$LOCAL_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.new.osc.gz"` - RES_TEXT=`diff -q "$LOCAL_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.new.state.txt"` - if [[ -n $RES_GZIP || -n $RES_TEXT ]]; then - echo "fetch_osc()@"`date -u "+%F %T"`": file_panic $REPLICATE_ID" >>$LOCAL_DIR/fetch_osc.log - echo "fetch_osc()@"`date -u "+%F %T"`": $RES_GZIP" >>$LOCAL_DIR/fetch_osc.log - echo "fetch_osc()@"`date -u "+%F %T"`": $RES_TEXT" >>$LOCAL_DIR/fetch_osc.log - exit 1 - fi - - rm "$LOCAL_PATH/$TDIGIT3.new.osc.gz" - rm "$LOCAL_PATH/$TDIGIT3.new.state.txt" -}; - -fetch_minute_diff() -{ - printf -v TDIGIT3 %03u $(($REPLICATE_ID % 1000)) - ARG=$(($REPLICATE_ID / 1000)) - printf -v TDIGIT2 %03u $(($ARG % 1000)) - ARG=$(($ARG / 1000)) - printf -v TDIGIT1 %03u $ARG - - LOCAL_PATH="$LOCAL_DIR/$TDIGIT1/$TDIGIT2" - REMOTE_PATH="$SOURCE_DIR/$TDIGIT1/$TDIGIT2" - mkdir -p "$LOCAL_DIR/$TDIGIT1/$TDIGIT2" - - retry_fetch_file "$REMOTE_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.osc.gz" "gzip" - if [[ -n $FILE_PANIC ]]; then - file_panic - fi - retry_fetch_file "$REMOTE_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.state.txt" "text" - if [[ -n $FILE_PANIC ]]; then - file_panic - fi - - TIMESTAMP_LINE=`grep timestamp $LOCAL_DIR/$TDIGIT1/$TDIGIT2/$TDIGIT3.state.txt` - TIMESTAMP=${TIMESTAMP_LINE:10} -}; - -while [[ true ]]; -do -{ - REPLICATE_ID=$(($REPLICATE_ID + 1)) - fetch_minute_diff - echo "fetch_osc()@"`date -u "+%F %T"`": new_replicate_diff $REPLICATE_ID $TIMESTAMP" >>$LOCAL_DIR/fetch_osc.log - sleep 1 -}; -done diff --git a/0.7.54/bin/update_overpass.sh b/0.7.54/bin/update_overpass.sh new file mode 100644 index 0000000..328e375 --- /dev/null +++ b/0.7.54/bin/update_overpass.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env sh + +( + set -e + + if [ "x$OVERPASS_META" = "xattic" ] ; then + META="--keep-attic" + elif [ "x$OVERPASS_META" = "xyes" ] ; then + META="--meta" + else + META="" + fi + + if [ ! -d /db/diffs ] ; then + mkdir /db/diffs + fi + + while `true` ; do + ( + if [ ! -e /db/diffs/changes.osm ] ; then + ! /app/venv/bin/pyosmium-get-changes --server $OVERPASS_DIFF_URL -o /db/diffs/changes.osm -f /db/replicate_id + OSMIUM_STATUS=$? + if [ $OSMIUM_STATUS -eq 1 ]; then + echo "There are still some updates remainging" + fi + if [ $OSMIUM_STATUS -eq 2 ]; then + echo "Failure downloading updates" + exit 0 + fi + else + echo "/db/diffs/changes.osm exists. Trying to apply again." + fi + /app/bin/update_from_dir --osc-dir=/db/diffs/ $META + rm /db/diffs/changes.osm + ) + sleep 60 + done +) 2>&1 | tee -a /db/changes.log \ No newline at end of file diff --git a/0.7.54/docker-entrypoint.sh b/0.7.54/docker-entrypoint.sh index 5a43603..cb69627 100644 --- a/0.7.54/docker-entrypoint.sh +++ b/0.7.54/docker-entrypoint.sh @@ -6,13 +6,15 @@ OVERPASS_META=${OVERPASS_META:-no} OVERPASS_MODE=${OVERPASS_MODE:-clone} OVERPASS_COMPRESSION=${OVERPASS_COMPRESSION:-gz} +if [[ "$OVERPASS_META" == "attic" ]] ; then + META="--keep-attic" +else + META="--meta" +fi + + if [ ! -d /db/db ] ; then if [ "$OVERPASS_MODE" = "clone" ]; then - if [[ "$OVERPASS_META" == "attic" ]] ; then - META="--keep-attic" - else - META="--meta=$OVERPASS_META" - fi mkdir -p /db/db \ && /app/bin/download_clone.sh --db-dir=/db/db --source=http://dev.overpass-api.de/api_drolbr/ $META \ && cp -r /app/etc/rules /db/db \ @@ -22,10 +24,21 @@ if [ ! -d /db/db ] ; then fi if [ "$OVERPASS_MODE" = "init" ]; then - lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet; exit" \ - && /app/bin/init_osm3s.sh /db/planet /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION" \ - && echo $OVERPASS_PLANET_SEQUENCE_ID > /db/db/replicate_id \ - && rm /db/planet \ + lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet.osm.bz2; exit" \ + && /app/bin/init_osm3s.sh /db/planet.osm.bz2 /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION --map-compression-method=$OVERPASS_COMPRESSION"\ + && echo "Database created. Now updating it." && ( + ! /app/venv/bin/pyosmium-get-changes -O /db/planet.osm.bz2 --server $OVERPASS_DIFF_URL -o /db/changes.osm -f /db/replicate_id + OSMIUM_STATUS=$? + if [ $OSMIUM_STATUS -eq 1 ]; then + echo "There are still some updates remainging" + fi + if [ $OSMIUM_STATUS -eq 2 ]; then + echo "Failure downloading updates" + exit 0 + fi + (cat /db/changes.osm | /app/bin/update_database --db-dir=/db/db $META --compression-method=$OVERPASS_COMPRESSION) 2>&1 | tee -a /db/changes.log + ) \ + && rm /db/planet.osm.bz2 /db/changes.osm \ && cp -r /app/etc/rules /db/db \ && chown -R overpass:overpass /db \ && echo "Overpass ready, you can start your container with docker start" diff --git a/0.7.54/etc/nginx-overpass.conf b/0.7.54/etc/nginx-overpass.conf index c019615..a19dde4 100644 --- a/0.7.54/etc/nginx-overpass.conf +++ b/0.7.54/etc/nginx-overpass.conf @@ -25,7 +25,7 @@ http { sendfile on; #tcp_nopush on; - keepalive_timeout 65; + keepalive_timeout 1000s; #gzip on; @@ -55,6 +55,7 @@ http { fastcgi_param SCRIPT_FILENAME /app/$fastcgi_script_name; fastcgi_param PATH_INFO $fastcgi_path_info; fastcgi_pass unix:/nginx/fcgiwrap.socket; + fastcgi_read_timeout 1000s; } location /api/ { diff --git a/0.7.54/etc/supervisord.conf b/0.7.54/etc/supervisord.conf index 6c4623b..fc67b74 100644 --- a/0.7.54/etc/supervisord.conf +++ b/0.7.54/etc/supervisord.conf @@ -17,18 +17,12 @@ redirect_stderr=true user=nginx priority=3 -[program:fetch_diff] -command=/app/bin/fetch_osc.sh auto %(ENV_OVERPASS_DIFF_URL)s /db/diffs +[program:update_overpass] +command=/app/bin/update_overpass.sh user=overpass redirect_stderr=true priority=5 -[program:apply_diff] -command=/app/bin/apply_osc_to_db.sh /db/diffs/ auto --meta=%(ENV_OVERPASS_META)s -redirect_stderr=true -user=overpass -priority=4 - [program:dispatcher_areas] command=nice /app/bin/dispatcher --areas --db-dir="/db/db" user=overpass diff --git a/0.7.54/requirements.txt b/0.7.54/requirements.txt new file mode 100644 index 0000000..8314012 --- /dev/null +++ b/0.7.54/requirements.txt @@ -0,0 +1 @@ +osmium diff --git a/0.7.51/Dockerfile b/0.7.55.1/Dockerfile similarity index 71% rename from 0.7.51/Dockerfile rename to 0.7.55.1/Dockerfile index bc4f5b5..3d01769 100644 --- a/0.7.51/Dockerfile +++ b/0.7.55.1/Dockerfile @@ -1,30 +1,36 @@ -FROM nginx:stable +FROM nginx:1.14 RUN addgroup overpass && adduser --home /db --disabled-password --gecos overpass --ingroup overpass overpass +ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.55.1.tar.gz /app/src.tar.gz + RUN apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ autoconf \ automake \ + bash \ + bzip2 \ ca-certificates \ expat \ - libexpat1-dev \ + fcgiwrap \ g++ \ + lftp \ + libexpat1-dev \ liblz4-1 \ liblz4-dev \ libtool \ m4 \ make \ + osmium-tool \ + python3 \ + python3-venv \ + supervisor \ + wget \ zlib1g \ - zlib1g-dev - - -ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.51.tar.gz /app/src.tar.gz - -RUN mkdir -p /app/src \ + zlib1g-dev \ + && mkdir -p /app/src \ && cd /app/src \ && tar -x -z --strip-components 1 -f ../src.tar.gz \ - && cd src \ && autoscan \ && aclocal \ && autoheader \ @@ -46,26 +52,28 @@ RUN mkdir -p /app/src \ && apt-get autoremove -y \ && mkdir -p /db/diffs /app/etc \ && cp -r /app/src/rules /app/etc/rules \ - && rm -rf /app/src /app/src.tar.gz + && rm -rf /app/src /app/src.tar.gz \ + && rm -rf /var/lib/apt/lists/* -RUN apt-get install --no-install-recommends --no-install-suggests -y \ - supervisor \ - bash \ - lftp \ - wget \ - fcgiwrap \ - bzip2 +COPY requirements.txt /app/ +RUN python3 -m venv /app/venv \ + && /app/venv/bin/pip install -r /app/requirements.txt RUN mkdir /nginx && chown nginx:nginx /nginx && chown -R overpass:overpass /db + COPY etc/supervisord.conf /etc/supervisor/conf.d/supervisord.conf + COPY etc/nginx-overpass.conf /etc/nginx/nginx.conf -COPY bin/apply_osc_to_db.sh bin/fetch_osc.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/ + +COPY bin/update_overpass.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/ + COPY docker-entrypoint.sh /app/ -RUN chmod a+x /app/docker-entrypoint.sh /app/bin/apply_osc_to_db.sh /app/bin/fetch_osc.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh -VOLUME /db + +RUN chmod a+x /app/docker-entrypoint.sh /app/bin/update_overpass.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh ENV OVERPASS_RULES_LOAD 1 + EXPOSE 80 # CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"] CMD ["/app/docker-entrypoint.sh"] diff --git a/0.7.51/bin/dispatcher_start.sh b/0.7.55.1/bin/dispatcher_start.sh old mode 100755 new mode 100644 similarity index 100% rename from 0.7.51/bin/dispatcher_start.sh rename to 0.7.55.1/bin/dispatcher_start.sh diff --git a/0.7.51/bin/rules_loop.sh b/0.7.55.1/bin/rules_loop.sh old mode 100755 new mode 100644 similarity index 100% rename from 0.7.51/bin/rules_loop.sh rename to 0.7.55.1/bin/rules_loop.sh diff --git a/0.7.55.1/bin/update_overpass.sh b/0.7.55.1/bin/update_overpass.sh new file mode 100644 index 0000000..328e375 --- /dev/null +++ b/0.7.55.1/bin/update_overpass.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env sh + +( + set -e + + if [ "x$OVERPASS_META" = "xattic" ] ; then + META="--keep-attic" + elif [ "x$OVERPASS_META" = "xyes" ] ; then + META="--meta" + else + META="" + fi + + if [ ! -d /db/diffs ] ; then + mkdir /db/diffs + fi + + while `true` ; do + ( + if [ ! -e /db/diffs/changes.osm ] ; then + ! /app/venv/bin/pyosmium-get-changes --server $OVERPASS_DIFF_URL -o /db/diffs/changes.osm -f /db/replicate_id + OSMIUM_STATUS=$? + if [ $OSMIUM_STATUS -eq 1 ]; then + echo "There are still some updates remainging" + fi + if [ $OSMIUM_STATUS -eq 2 ]; then + echo "Failure downloading updates" + exit 0 + fi + else + echo "/db/diffs/changes.osm exists. Trying to apply again." + fi + /app/bin/update_from_dir --osc-dir=/db/diffs/ $META + rm /db/diffs/changes.osm + ) + sleep 60 + done +) 2>&1 | tee -a /db/changes.log \ No newline at end of file diff --git a/0.7.55.1/docker-entrypoint.sh b/0.7.55.1/docker-entrypoint.sh new file mode 100644 index 0000000..cb69627 --- /dev/null +++ b/0.7.55.1/docker-entrypoint.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +set -eo pipefail +shopt -s nullglob +OVERPASS_META=${OVERPASS_META:-no} +OVERPASS_MODE=${OVERPASS_MODE:-clone} +OVERPASS_COMPRESSION=${OVERPASS_COMPRESSION:-gz} + +if [[ "$OVERPASS_META" == "attic" ]] ; then + META="--keep-attic" +else + META="--meta" +fi + + +if [ ! -d /db/db ] ; then + if [ "$OVERPASS_MODE" = "clone" ]; then + mkdir -p /db/db \ + && /app/bin/download_clone.sh --db-dir=/db/db --source=http://dev.overpass-api.de/api_drolbr/ $META \ + && cp -r /app/etc/rules /db/db \ + && chown -R overpass:overpass /db \ + && echo "Overpass ready, you can start your container with docker start" + exit + fi + + if [ "$OVERPASS_MODE" = "init" ]; then + lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet.osm.bz2; exit" \ + && /app/bin/init_osm3s.sh /db/planet.osm.bz2 /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION --map-compression-method=$OVERPASS_COMPRESSION"\ + && echo "Database created. Now updating it." && ( + ! /app/venv/bin/pyosmium-get-changes -O /db/planet.osm.bz2 --server $OVERPASS_DIFF_URL -o /db/changes.osm -f /db/replicate_id + OSMIUM_STATUS=$? + if [ $OSMIUM_STATUS -eq 1 ]; then + echo "There are still some updates remainging" + fi + if [ $OSMIUM_STATUS -eq 2 ]; then + echo "Failure downloading updates" + exit 0 + fi + (cat /db/changes.osm | /app/bin/update_database --db-dir=/db/db $META --compression-method=$OVERPASS_COMPRESSION) 2>&1 | tee -a /db/changes.log + ) \ + && rm /db/planet.osm.bz2 /db/changes.osm \ + && cp -r /app/etc/rules /db/db \ + && chown -R overpass:overpass /db \ + && echo "Overpass ready, you can start your container with docker start" + exit + fi +fi + +exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf diff --git a/0.7.54.11/etc/nginx-overpass.conf b/0.7.55.1/etc/nginx-overpass.conf similarity index 95% rename from 0.7.54.11/etc/nginx-overpass.conf rename to 0.7.55.1/etc/nginx-overpass.conf index c019615..a19dde4 100644 --- a/0.7.54.11/etc/nginx-overpass.conf +++ b/0.7.55.1/etc/nginx-overpass.conf @@ -25,7 +25,7 @@ http { sendfile on; #tcp_nopush on; - keepalive_timeout 65; + keepalive_timeout 1000s; #gzip on; @@ -55,6 +55,7 @@ http { fastcgi_param SCRIPT_FILENAME /app/$fastcgi_script_name; fastcgi_param PATH_INFO $fastcgi_path_info; fastcgi_pass unix:/nginx/fcgiwrap.socket; + fastcgi_read_timeout 1000s; } location /api/ { diff --git a/0.7.51/etc/supervisord.conf b/0.7.55.1/etc/supervisord.conf similarity index 74% rename from 0.7.51/etc/supervisord.conf rename to 0.7.55.1/etc/supervisord.conf index 6c4623b..fc67b74 100644 --- a/0.7.51/etc/supervisord.conf +++ b/0.7.55.1/etc/supervisord.conf @@ -17,18 +17,12 @@ redirect_stderr=true user=nginx priority=3 -[program:fetch_diff] -command=/app/bin/fetch_osc.sh auto %(ENV_OVERPASS_DIFF_URL)s /db/diffs +[program:update_overpass] +command=/app/bin/update_overpass.sh user=overpass redirect_stderr=true priority=5 -[program:apply_diff] -command=/app/bin/apply_osc_to_db.sh /db/diffs/ auto --meta=%(ENV_OVERPASS_META)s -redirect_stderr=true -user=overpass -priority=4 - [program:dispatcher_areas] command=nice /app/bin/dispatcher --areas --db-dir="/db/db" user=overpass diff --git a/0.7.55.1/requirements.txt b/0.7.55.1/requirements.txt new file mode 100644 index 0000000..8314012 --- /dev/null +++ b/0.7.55.1/requirements.txt @@ -0,0 +1 @@ +osmium diff --git a/0.7.54.11/Dockerfile b/0.7.55.2/Dockerfile similarity index 71% rename from 0.7.54.11/Dockerfile rename to 0.7.55.2/Dockerfile index 39ad38e..3da8118 100644 --- a/0.7.54.11/Dockerfile +++ b/0.7.55.2/Dockerfile @@ -1,27 +1,34 @@ -FROM nginx:stable +FROM nginx:1.14 RUN addgroup overpass && adduser --home /db --disabled-password --gecos overpass --ingroup overpass overpass +ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.55.2.tar.gz /app/src.tar.gz + RUN apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ autoconf \ automake \ + bash \ + bzip2 \ ca-certificates \ expat \ - libexpat1-dev \ + fcgiwrap \ g++ \ + lftp \ + libexpat1-dev \ liblz4-1 \ liblz4-dev \ libtool \ m4 \ make \ + osmium-tool \ + python3 \ + python3-venv \ + supervisor \ + wget \ zlib1g \ - zlib1g-dev - - -ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.54.11.tar.gz /app/src.tar.gz - -RUN mkdir -p /app/src \ + zlib1g-dev \ + && mkdir -p /app/src \ && cd /app/src \ && tar -x -z --strip-components 1 -f ../src.tar.gz \ && autoscan \ @@ -45,26 +52,28 @@ RUN mkdir -p /app/src \ && apt-get autoremove -y \ && mkdir -p /db/diffs /app/etc \ && cp -r /app/src/rules /app/etc/rules \ - && rm -rf /app/src /app/src.tar.gz + && rm -rf /app/src /app/src.tar.gz \ + && rm -rf /var/lib/apt/lists/* -RUN apt-get install --no-install-recommends --no-install-suggests -y \ - supervisor \ - bash \ - lftp \ - wget \ - fcgiwrap \ - bzip2 +COPY requirements.txt /app/ +RUN python3 -m venv /app/venv \ + && /app/venv/bin/pip install -r /app/requirements.txt RUN mkdir /nginx && chown nginx:nginx /nginx && chown -R overpass:overpass /db + COPY etc/supervisord.conf /etc/supervisor/conf.d/supervisord.conf + COPY etc/nginx-overpass.conf /etc/nginx/nginx.conf -COPY bin/apply_osc_to_db.sh bin/fetch_osc.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/ + +COPY bin/update_overpass.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/ + COPY docker-entrypoint.sh /app/ -RUN chmod a+x /app/docker-entrypoint.sh /app/bin/apply_osc_to_db.sh /app/bin/fetch_osc.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh -VOLUME /db + +RUN chmod a+x /app/docker-entrypoint.sh /app/bin/update_overpass.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh ENV OVERPASS_RULES_LOAD 1 + EXPOSE 80 # CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"] CMD ["/app/docker-entrypoint.sh"] diff --git a/0.7.54.11/bin/dispatcher_start.sh b/0.7.55.2/bin/dispatcher_start.sh old mode 100755 new mode 100644 similarity index 100% rename from 0.7.54.11/bin/dispatcher_start.sh rename to 0.7.55.2/bin/dispatcher_start.sh diff --git a/0.7.54.11/bin/rules_loop.sh b/0.7.55.2/bin/rules_loop.sh old mode 100755 new mode 100644 similarity index 100% rename from 0.7.54.11/bin/rules_loop.sh rename to 0.7.55.2/bin/rules_loop.sh diff --git a/0.7.55.2/bin/update_overpass.sh b/0.7.55.2/bin/update_overpass.sh new file mode 100644 index 0000000..328e375 --- /dev/null +++ b/0.7.55.2/bin/update_overpass.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env sh + +( + set -e + + if [ "x$OVERPASS_META" = "xattic" ] ; then + META="--keep-attic" + elif [ "x$OVERPASS_META" = "xyes" ] ; then + META="--meta" + else + META="" + fi + + if [ ! -d /db/diffs ] ; then + mkdir /db/diffs + fi + + while `true` ; do + ( + if [ ! -e /db/diffs/changes.osm ] ; then + ! /app/venv/bin/pyosmium-get-changes --server $OVERPASS_DIFF_URL -o /db/diffs/changes.osm -f /db/replicate_id + OSMIUM_STATUS=$? + if [ $OSMIUM_STATUS -eq 1 ]; then + echo "There are still some updates remainging" + fi + if [ $OSMIUM_STATUS -eq 2 ]; then + echo "Failure downloading updates" + exit 0 + fi + else + echo "/db/diffs/changes.osm exists. Trying to apply again." + fi + /app/bin/update_from_dir --osc-dir=/db/diffs/ $META + rm /db/diffs/changes.osm + ) + sleep 60 + done +) 2>&1 | tee -a /db/changes.log \ No newline at end of file diff --git a/0.7.55.2/docker-entrypoint.sh b/0.7.55.2/docker-entrypoint.sh new file mode 100644 index 0000000..cb69627 --- /dev/null +++ b/0.7.55.2/docker-entrypoint.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +set -eo pipefail +shopt -s nullglob +OVERPASS_META=${OVERPASS_META:-no} +OVERPASS_MODE=${OVERPASS_MODE:-clone} +OVERPASS_COMPRESSION=${OVERPASS_COMPRESSION:-gz} + +if [[ "$OVERPASS_META" == "attic" ]] ; then + META="--keep-attic" +else + META="--meta" +fi + + +if [ ! -d /db/db ] ; then + if [ "$OVERPASS_MODE" = "clone" ]; then + mkdir -p /db/db \ + && /app/bin/download_clone.sh --db-dir=/db/db --source=http://dev.overpass-api.de/api_drolbr/ $META \ + && cp -r /app/etc/rules /db/db \ + && chown -R overpass:overpass /db \ + && echo "Overpass ready, you can start your container with docker start" + exit + fi + + if [ "$OVERPASS_MODE" = "init" ]; then + lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet.osm.bz2; exit" \ + && /app/bin/init_osm3s.sh /db/planet.osm.bz2 /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION --map-compression-method=$OVERPASS_COMPRESSION"\ + && echo "Database created. Now updating it." && ( + ! /app/venv/bin/pyosmium-get-changes -O /db/planet.osm.bz2 --server $OVERPASS_DIFF_URL -o /db/changes.osm -f /db/replicate_id + OSMIUM_STATUS=$? + if [ $OSMIUM_STATUS -eq 1 ]; then + echo "There are still some updates remainging" + fi + if [ $OSMIUM_STATUS -eq 2 ]; then + echo "Failure downloading updates" + exit 0 + fi + (cat /db/changes.osm | /app/bin/update_database --db-dir=/db/db $META --compression-method=$OVERPASS_COMPRESSION) 2>&1 | tee -a /db/changes.log + ) \ + && rm /db/planet.osm.bz2 /db/changes.osm \ + && cp -r /app/etc/rules /db/db \ + && chown -R overpass:overpass /db \ + && echo "Overpass ready, you can start your container with docker start" + exit + fi +fi + +exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf diff --git a/0.7.51/etc/nginx-overpass.conf b/0.7.55.2/etc/nginx-overpass.conf similarity index 95% rename from 0.7.51/etc/nginx-overpass.conf rename to 0.7.55.2/etc/nginx-overpass.conf index c019615..a19dde4 100644 --- a/0.7.51/etc/nginx-overpass.conf +++ b/0.7.55.2/etc/nginx-overpass.conf @@ -25,7 +25,7 @@ http { sendfile on; #tcp_nopush on; - keepalive_timeout 65; + keepalive_timeout 1000s; #gzip on; @@ -55,6 +55,7 @@ http { fastcgi_param SCRIPT_FILENAME /app/$fastcgi_script_name; fastcgi_param PATH_INFO $fastcgi_path_info; fastcgi_pass unix:/nginx/fcgiwrap.socket; + fastcgi_read_timeout 1000s; } location /api/ { diff --git a/0.7.54.11/etc/supervisord.conf b/0.7.55.2/etc/supervisord.conf similarity index 74% rename from 0.7.54.11/etc/supervisord.conf rename to 0.7.55.2/etc/supervisord.conf index 6c4623b..fc67b74 100644 --- a/0.7.54.11/etc/supervisord.conf +++ b/0.7.55.2/etc/supervisord.conf @@ -17,18 +17,12 @@ redirect_stderr=true user=nginx priority=3 -[program:fetch_diff] -command=/app/bin/fetch_osc.sh auto %(ENV_OVERPASS_DIFF_URL)s /db/diffs +[program:update_overpass] +command=/app/bin/update_overpass.sh user=overpass redirect_stderr=true priority=5 -[program:apply_diff] -command=/app/bin/apply_osc_to_db.sh /db/diffs/ auto --meta=%(ENV_OVERPASS_META)s -redirect_stderr=true -user=overpass -priority=4 - [program:dispatcher_areas] command=nice /app/bin/dispatcher --areas --db-dir="/db/db" user=overpass diff --git a/0.7.55.2/requirements.txt b/0.7.55.2/requirements.txt new file mode 100644 index 0000000..8314012 --- /dev/null +++ b/0.7.55.2/requirements.txt @@ -0,0 +1 @@ +osmium diff --git a/0.7.55.3/Dockerfile b/0.7.55.3/Dockerfile new file mode 100644 index 0000000..7a08746 --- /dev/null +++ b/0.7.55.3/Dockerfile @@ -0,0 +1,79 @@ +FROM nginx:1.14 + +RUN addgroup overpass && adduser --home /db --disabled-password --gecos overpass --ingroup overpass overpass + +ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.55.3.tar.gz /app/src.tar.gz + +RUN apt-get update \ + && apt-get install --no-install-recommends --no-install-suggests -y \ + autoconf \ + automake \ + bash \ + bzip2 \ + ca-certificates \ + expat \ + fcgiwrap \ + g++ \ + lftp \ + libexpat1-dev \ + liblz4-1 \ + liblz4-dev \ + libtool \ + m4 \ + make \ + osmium-tool \ + python3 \ + python3-venv \ + supervisor \ + wget \ + zlib1g \ + zlib1g-dev \ + && mkdir -p /app/src \ + && cd /app/src \ + && tar -x -z --strip-components 1 -f ../src.tar.gz \ + && autoscan \ + && aclocal \ + && autoheader \ + && libtoolize \ + && automake --add-missing \ + && autoconf \ + && CXXFLAGS='-O2' CFLAGS='-O2' ./configure --prefix=/app --enable-lz4 \ + && make -j $(grep -c ^processor /proc/cpuinfo) dist install clean \ + && apt-get remove -y \ + autoconf \ + automake \ + libexpat1-dev \ + g++ \ + libtool \ + liblz4-dev \ + m4 \ + make \ + zlib1g-dev \ + && apt-get autoremove -y \ + && mkdir -p /db/diffs /app/etc \ + && cp -r /app/src/rules /app/etc/rules \ + && rm -rf /app/src /app/src.tar.gz \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt /app/ + +RUN python3 -m venv /app/venv \ + && /app/venv/bin/pip install -r /app/requirements.txt + +RUN mkdir /nginx && chown nginx:nginx /nginx && chown -R overpass:overpass /db + +COPY etc/supervisord.conf /etc/supervisor/conf.d/supervisord.conf + +COPY etc/nginx-overpass.conf /etc/nginx/nginx.conf + +COPY bin/update_overpass.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/ + +COPY docker-entrypoint.sh /app/ + +RUN chmod a+x /app/docker-entrypoint.sh /app/bin/update_overpass.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh + +ENV OVERPASS_RULES_LOAD 1 + +EXPOSE 80 +# CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"] +CMD ["/app/docker-entrypoint.sh"] diff --git a/0.7.55.3/bin/dispatcher_start.sh b/0.7.55.3/bin/dispatcher_start.sh new file mode 100644 index 0000000..19a3c55 --- /dev/null +++ b/0.7.55.3/bin/dispatcher_start.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -e -o pipefail + +if [[ $OVERPASS_META == 'yes' ]] ; then + META_ARG='--meta' +elif [[ $OVERPASS_META == 'attic' ]] ; then + META_ARG='--attic' +else + META_ARG_='' +fi + +find /db/db -type s -print0 | xargs -0 --no-run-if-empty rm && /app/bin/dispatcher --osm-base "${META_ARG}" --db-dir=/db/db + diff --git a/0.7.55.3/bin/rules_loop.sh b/0.7.55.3/bin/rules_loop.sh new file mode 100644 index 0000000..a50f94f --- /dev/null +++ b/0.7.55.3/bin/rules_loop.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht +# +# This file is part of Overpass_API. +# +# Overpass_API is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Overpass_API is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Overpass_API. If not, see . + +if [[ -z $1 ]]; then +{ + echo "Usage: $0 database_dir [desired_cpu_load]" + exit 0 +}; +fi + +CPU_LOAD=${2:-100} +DB_DIR="`pwd`/$1" + +EXEC_DIR="`dirname $0`/" +if [[ ! ${EXEC_DIR:0:1} == "/" ]]; then +{ + EXEC_DIR="`pwd`/$EXEC_DIR" +}; +fi + +pushd "$EXEC_DIR" + +while [[ true ]]; do +{ + START=$(date +%s) + echo "`date '+%F %T'`: update started" >>$DB_DIR/rules_loop.log + ./osm3s_query --progress --rules <$DB_DIR/rules/areas.osm3s + echo "`date '+%F %T'`: update finished" >>$DB_DIR/rules_loop.log + WORK_TIME=$(( $(date +%s) - START )) + SLEEP_TIME=$(( WORK_TIME * 100 / CPU_LOAD - WORK_TIME)) + # let SLEEP_TIME be at least 3 seconds + SLEEP_TIME=$(( SLEEP_TIME < 3 ? 3 : SLEEP_TIME)) + echo "It took $WORK_TIME to run the loop. Desired load is: ${CPU_LOAD}%. Sleeping: $SLEEP_TIME" + sleep $SLEEP_TIME +}; done diff --git a/0.7.55.3/bin/update_overpass.sh b/0.7.55.3/bin/update_overpass.sh new file mode 100644 index 0000000..328e375 --- /dev/null +++ b/0.7.55.3/bin/update_overpass.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env sh + +( + set -e + + if [ "x$OVERPASS_META" = "xattic" ] ; then + META="--keep-attic" + elif [ "x$OVERPASS_META" = "xyes" ] ; then + META="--meta" + else + META="" + fi + + if [ ! -d /db/diffs ] ; then + mkdir /db/diffs + fi + + while `true` ; do + ( + if [ ! -e /db/diffs/changes.osm ] ; then + ! /app/venv/bin/pyosmium-get-changes --server $OVERPASS_DIFF_URL -o /db/diffs/changes.osm -f /db/replicate_id + OSMIUM_STATUS=$? + if [ $OSMIUM_STATUS -eq 1 ]; then + echo "There are still some updates remainging" + fi + if [ $OSMIUM_STATUS -eq 2 ]; then + echo "Failure downloading updates" + exit 0 + fi + else + echo "/db/diffs/changes.osm exists. Trying to apply again." + fi + /app/bin/update_from_dir --osc-dir=/db/diffs/ $META + rm /db/diffs/changes.osm + ) + sleep 60 + done +) 2>&1 | tee -a /db/changes.log \ No newline at end of file diff --git a/0.7.55.3/docker-entrypoint.sh b/0.7.55.3/docker-entrypoint.sh new file mode 100644 index 0000000..cb69627 --- /dev/null +++ b/0.7.55.3/docker-entrypoint.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +set -eo pipefail +shopt -s nullglob +OVERPASS_META=${OVERPASS_META:-no} +OVERPASS_MODE=${OVERPASS_MODE:-clone} +OVERPASS_COMPRESSION=${OVERPASS_COMPRESSION:-gz} + +if [[ "$OVERPASS_META" == "attic" ]] ; then + META="--keep-attic" +else + META="--meta" +fi + + +if [ ! -d /db/db ] ; then + if [ "$OVERPASS_MODE" = "clone" ]; then + mkdir -p /db/db \ + && /app/bin/download_clone.sh --db-dir=/db/db --source=http://dev.overpass-api.de/api_drolbr/ $META \ + && cp -r /app/etc/rules /db/db \ + && chown -R overpass:overpass /db \ + && echo "Overpass ready, you can start your container with docker start" + exit + fi + + if [ "$OVERPASS_MODE" = "init" ]; then + lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet.osm.bz2; exit" \ + && /app/bin/init_osm3s.sh /db/planet.osm.bz2 /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION --map-compression-method=$OVERPASS_COMPRESSION"\ + && echo "Database created. Now updating it." && ( + ! /app/venv/bin/pyosmium-get-changes -O /db/planet.osm.bz2 --server $OVERPASS_DIFF_URL -o /db/changes.osm -f /db/replicate_id + OSMIUM_STATUS=$? + if [ $OSMIUM_STATUS -eq 1 ]; then + echo "There are still some updates remainging" + fi + if [ $OSMIUM_STATUS -eq 2 ]; then + echo "Failure downloading updates" + exit 0 + fi + (cat /db/changes.osm | /app/bin/update_database --db-dir=/db/db $META --compression-method=$OVERPASS_COMPRESSION) 2>&1 | tee -a /db/changes.log + ) \ + && rm /db/planet.osm.bz2 /db/changes.osm \ + && cp -r /app/etc/rules /db/db \ + && chown -R overpass:overpass /db \ + && echo "Overpass ready, you can start your container with docker start" + exit + fi +fi + +exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf diff --git a/0.7.55.3/etc/nginx-overpass.conf b/0.7.55.3/etc/nginx-overpass.conf new file mode 100644 index 0000000..a19dde4 --- /dev/null +++ b/0.7.55.3/etc/nginx-overpass.conf @@ -0,0 +1,67 @@ +daemon off; + +user nginx; +worker_processes 4; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + + +events { + worker_connections 1024; +} + + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 1000s; + + #gzip on; + + server { + listen 80; + server_name localhost; + + #charset koi8-r; + #access_log /var/log/nginx/log/host.access.log main; + + location / { + root /usr/share/nginx/html; + index index.html index.htm; + } + + #error_page 404 /404.html; + + # redirect server error pages to the static page /50x.html + # + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } + + location /cgi-bin/ { + include fastcgi_params; + fastcgi_param SCRIPT_FILENAME /app/$fastcgi_script_name; + fastcgi_param PATH_INFO $fastcgi_path_info; + fastcgi_pass unix:/nginx/fcgiwrap.socket; + fastcgi_read_timeout 1000s; + } + + location /api/ { + rewrite ^/api/(.+)$ /cgi-bin/$1 last; + } + } + + +} diff --git a/0.7.55.3/etc/supervisord.conf b/0.7.55.3/etc/supervisord.conf new file mode 100644 index 0000000..fc67b74 --- /dev/null +++ b/0.7.55.3/etc/supervisord.conf @@ -0,0 +1,36 @@ +[supervisord] +nodaemon=true + +[program:overpass_dispatch] +command=/app/bin/dispatcher_start.sh +user=overpass +redirect_stderr=true +priority=1 + +[program:nginx] +command=nginx +priority=2 + +[program:fcgiwrap] +command=/bin/bash -c "find /nginx -type s -print0 | xargs -0 --no-run-if-empty rm && fcgiwrap -s unix:/nginx/fcgiwrap.socket" +redirect_stderr=true +user=nginx +priority=3 + +[program:update_overpass] +command=/app/bin/update_overpass.sh +user=overpass +redirect_stderr=true +priority=5 + +[program:dispatcher_areas] +command=nice /app/bin/dispatcher --areas --db-dir="/db/db" +user=overpass +redirect_stderr=true +priority=6 + +[program:areas_rules] +command=nice /app/bin/rules_loop.sh /db/db %(ENV_OVERPASS_RULES_LOAD)s +user=overpass +redirect_stderr=true +priority=7 diff --git a/0.7.55.3/requirements.txt b/0.7.55.3/requirements.txt new file mode 100644 index 0000000..8314012 --- /dev/null +++ b/0.7.55.3/requirements.txt @@ -0,0 +1 @@ +osmium diff --git a/0.7.55.4/Dockerfile b/0.7.55.4/Dockerfile new file mode 100644 index 0000000..4662687 --- /dev/null +++ b/0.7.55.4/Dockerfile @@ -0,0 +1,79 @@ +FROM nginx:1.14 + +RUN addgroup overpass && adduser --home /db --disabled-password --gecos overpass --ingroup overpass overpass + +ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.55.4.tar.gz /app/src.tar.gz + +RUN apt-get update \ + && apt-get install --no-install-recommends --no-install-suggests -y \ + autoconf \ + automake \ + bash \ + bzip2 \ + ca-certificates \ + expat \ + fcgiwrap \ + g++ \ + lftp \ + libexpat1-dev \ + liblz4-1 \ + liblz4-dev \ + libtool \ + m4 \ + make \ + osmium-tool \ + python3 \ + python3-venv \ + supervisor \ + wget \ + zlib1g \ + zlib1g-dev \ + && mkdir -p /app/src \ + && cd /app/src \ + && tar -x -z --strip-components 1 -f ../src.tar.gz \ + && autoscan \ + && aclocal \ + && autoheader \ + && libtoolize \ + && automake --add-missing \ + && autoconf \ + && CXXFLAGS='-O2' CFLAGS='-O2' ./configure --prefix=/app --enable-lz4 \ + && make -j $(grep -c ^processor /proc/cpuinfo) dist install clean \ + && apt-get remove -y \ + autoconf \ + automake \ + libexpat1-dev \ + g++ \ + libtool \ + liblz4-dev \ + m4 \ + make \ + zlib1g-dev \ + && apt-get autoremove -y \ + && mkdir -p /db/diffs /app/etc \ + && cp -r /app/src/rules /app/etc/rules \ + && rm -rf /app/src /app/src.tar.gz \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt /app/ + +RUN python3 -m venv /app/venv \ + && /app/venv/bin/pip install -r /app/requirements.txt + +RUN mkdir /nginx && chown nginx:nginx /nginx && chown -R overpass:overpass /db + +COPY etc/supervisord.conf /etc/supervisor/conf.d/supervisord.conf + +COPY etc/nginx-overpass.conf /etc/nginx/nginx.conf + +COPY bin/update_overpass.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/ + +COPY docker-entrypoint.sh /app/ + +RUN chmod a+x /app/docker-entrypoint.sh /app/bin/update_overpass.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh + +ENV OVERPASS_RULES_LOAD 1 + +EXPOSE 80 +# CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"] +CMD ["/app/docker-entrypoint.sh"] diff --git a/0.7.55.4/bin/dispatcher_start.sh b/0.7.55.4/bin/dispatcher_start.sh new file mode 100644 index 0000000..19a3c55 --- /dev/null +++ b/0.7.55.4/bin/dispatcher_start.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -e -o pipefail + +if [[ $OVERPASS_META == 'yes' ]] ; then + META_ARG='--meta' +elif [[ $OVERPASS_META == 'attic' ]] ; then + META_ARG='--attic' +else + META_ARG_='' +fi + +find /db/db -type s -print0 | xargs -0 --no-run-if-empty rm && /app/bin/dispatcher --osm-base "${META_ARG}" --db-dir=/db/db + diff --git a/0.7.55.4/bin/rules_loop.sh b/0.7.55.4/bin/rules_loop.sh new file mode 100644 index 0000000..a50f94f --- /dev/null +++ b/0.7.55.4/bin/rules_loop.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht +# +# This file is part of Overpass_API. +# +# Overpass_API is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Overpass_API is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Overpass_API. If not, see . + +if [[ -z $1 ]]; then +{ + echo "Usage: $0 database_dir [desired_cpu_load]" + exit 0 +}; +fi + +CPU_LOAD=${2:-100} +DB_DIR="`pwd`/$1" + +EXEC_DIR="`dirname $0`/" +if [[ ! ${EXEC_DIR:0:1} == "/" ]]; then +{ + EXEC_DIR="`pwd`/$EXEC_DIR" +}; +fi + +pushd "$EXEC_DIR" + +while [[ true ]]; do +{ + START=$(date +%s) + echo "`date '+%F %T'`: update started" >>$DB_DIR/rules_loop.log + ./osm3s_query --progress --rules <$DB_DIR/rules/areas.osm3s + echo "`date '+%F %T'`: update finished" >>$DB_DIR/rules_loop.log + WORK_TIME=$(( $(date +%s) - START )) + SLEEP_TIME=$(( WORK_TIME * 100 / CPU_LOAD - WORK_TIME)) + # let SLEEP_TIME be at least 3 seconds + SLEEP_TIME=$(( SLEEP_TIME < 3 ? 3 : SLEEP_TIME)) + echo "It took $WORK_TIME to run the loop. Desired load is: ${CPU_LOAD}%. Sleeping: $SLEEP_TIME" + sleep $SLEEP_TIME +}; done diff --git a/0.7.55.4/bin/update_overpass.sh b/0.7.55.4/bin/update_overpass.sh new file mode 100644 index 0000000..328e375 --- /dev/null +++ b/0.7.55.4/bin/update_overpass.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env sh + +( + set -e + + if [ "x$OVERPASS_META" = "xattic" ] ; then + META="--keep-attic" + elif [ "x$OVERPASS_META" = "xyes" ] ; then + META="--meta" + else + META="" + fi + + if [ ! -d /db/diffs ] ; then + mkdir /db/diffs + fi + + while `true` ; do + ( + if [ ! -e /db/diffs/changes.osm ] ; then + ! /app/venv/bin/pyosmium-get-changes --server $OVERPASS_DIFF_URL -o /db/diffs/changes.osm -f /db/replicate_id + OSMIUM_STATUS=$? + if [ $OSMIUM_STATUS -eq 1 ]; then + echo "There are still some updates remainging" + fi + if [ $OSMIUM_STATUS -eq 2 ]; then + echo "Failure downloading updates" + exit 0 + fi + else + echo "/db/diffs/changes.osm exists. Trying to apply again." + fi + /app/bin/update_from_dir --osc-dir=/db/diffs/ $META + rm /db/diffs/changes.osm + ) + sleep 60 + done +) 2>&1 | tee -a /db/changes.log \ No newline at end of file diff --git a/0.7.55.4/docker-entrypoint.sh b/0.7.55.4/docker-entrypoint.sh new file mode 100644 index 0000000..cb69627 --- /dev/null +++ b/0.7.55.4/docker-entrypoint.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +set -eo pipefail +shopt -s nullglob +OVERPASS_META=${OVERPASS_META:-no} +OVERPASS_MODE=${OVERPASS_MODE:-clone} +OVERPASS_COMPRESSION=${OVERPASS_COMPRESSION:-gz} + +if [[ "$OVERPASS_META" == "attic" ]] ; then + META="--keep-attic" +else + META="--meta" +fi + + +if [ ! -d /db/db ] ; then + if [ "$OVERPASS_MODE" = "clone" ]; then + mkdir -p /db/db \ + && /app/bin/download_clone.sh --db-dir=/db/db --source=http://dev.overpass-api.de/api_drolbr/ $META \ + && cp -r /app/etc/rules /db/db \ + && chown -R overpass:overpass /db \ + && echo "Overpass ready, you can start your container with docker start" + exit + fi + + if [ "$OVERPASS_MODE" = "init" ]; then + lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet.osm.bz2; exit" \ + && /app/bin/init_osm3s.sh /db/planet.osm.bz2 /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION --map-compression-method=$OVERPASS_COMPRESSION"\ + && echo "Database created. Now updating it." && ( + ! /app/venv/bin/pyosmium-get-changes -O /db/planet.osm.bz2 --server $OVERPASS_DIFF_URL -o /db/changes.osm -f /db/replicate_id + OSMIUM_STATUS=$? + if [ $OSMIUM_STATUS -eq 1 ]; then + echo "There are still some updates remainging" + fi + if [ $OSMIUM_STATUS -eq 2 ]; then + echo "Failure downloading updates" + exit 0 + fi + (cat /db/changes.osm | /app/bin/update_database --db-dir=/db/db $META --compression-method=$OVERPASS_COMPRESSION) 2>&1 | tee -a /db/changes.log + ) \ + && rm /db/planet.osm.bz2 /db/changes.osm \ + && cp -r /app/etc/rules /db/db \ + && chown -R overpass:overpass /db \ + && echo "Overpass ready, you can start your container with docker start" + exit + fi +fi + +exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf diff --git a/0.7.55.4/etc/nginx-overpass.conf b/0.7.55.4/etc/nginx-overpass.conf new file mode 100644 index 0000000..a19dde4 --- /dev/null +++ b/0.7.55.4/etc/nginx-overpass.conf @@ -0,0 +1,67 @@ +daemon off; + +user nginx; +worker_processes 4; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + + +events { + worker_connections 1024; +} + + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 1000s; + + #gzip on; + + server { + listen 80; + server_name localhost; + + #charset koi8-r; + #access_log /var/log/nginx/log/host.access.log main; + + location / { + root /usr/share/nginx/html; + index index.html index.htm; + } + + #error_page 404 /404.html; + + # redirect server error pages to the static page /50x.html + # + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } + + location /cgi-bin/ { + include fastcgi_params; + fastcgi_param SCRIPT_FILENAME /app/$fastcgi_script_name; + fastcgi_param PATH_INFO $fastcgi_path_info; + fastcgi_pass unix:/nginx/fcgiwrap.socket; + fastcgi_read_timeout 1000s; + } + + location /api/ { + rewrite ^/api/(.+)$ /cgi-bin/$1 last; + } + } + + +} diff --git a/0.7.55.4/etc/supervisord.conf b/0.7.55.4/etc/supervisord.conf new file mode 100644 index 0000000..fc67b74 --- /dev/null +++ b/0.7.55.4/etc/supervisord.conf @@ -0,0 +1,36 @@ +[supervisord] +nodaemon=true + +[program:overpass_dispatch] +command=/app/bin/dispatcher_start.sh +user=overpass +redirect_stderr=true +priority=1 + +[program:nginx] +command=nginx +priority=2 + +[program:fcgiwrap] +command=/bin/bash -c "find /nginx -type s -print0 | xargs -0 --no-run-if-empty rm && fcgiwrap -s unix:/nginx/fcgiwrap.socket" +redirect_stderr=true +user=nginx +priority=3 + +[program:update_overpass] +command=/app/bin/update_overpass.sh +user=overpass +redirect_stderr=true +priority=5 + +[program:dispatcher_areas] +command=nice /app/bin/dispatcher --areas --db-dir="/db/db" +user=overpass +redirect_stderr=true +priority=6 + +[program:areas_rules] +command=nice /app/bin/rules_loop.sh /db/db %(ENV_OVERPASS_RULES_LOAD)s +user=overpass +redirect_stderr=true +priority=7 diff --git a/0.7.55.4/requirements.txt b/0.7.55.4/requirements.txt new file mode 100644 index 0000000..8314012 --- /dev/null +++ b/0.7.55.4/requirements.txt @@ -0,0 +1 @@ +osmium diff --git a/0.7.55.5/Dockerfile b/0.7.55.5/Dockerfile new file mode 100644 index 0000000..c034d99 --- /dev/null +++ b/0.7.55.5/Dockerfile @@ -0,0 +1,79 @@ +FROM nginx:1.14 + +RUN addgroup overpass && adduser --home /db --disabled-password --gecos overpass --ingroup overpass overpass + +ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.55.5.tar.gz /app/src.tar.gz + +RUN apt-get update \ + && apt-get install --no-install-recommends --no-install-suggests -y \ + autoconf \ + automake \ + bash \ + bzip2 \ + ca-certificates \ + expat \ + fcgiwrap \ + g++ \ + lftp \ + libexpat1-dev \ + liblz4-1 \ + liblz4-dev \ + libtool \ + m4 \ + make \ + osmium-tool \ + python3 \ + python3-venv \ + supervisor \ + wget \ + zlib1g \ + zlib1g-dev \ + && mkdir -p /app/src \ + && cd /app/src \ + && tar -x -z --strip-components 1 -f ../src.tar.gz \ + && autoscan \ + && aclocal \ + && autoheader \ + && libtoolize \ + && automake --add-missing \ + && autoconf \ + && CXXFLAGS='-O2' CFLAGS='-O2' ./configure --prefix=/app --enable-lz4 \ + && make -j $(grep -c ^processor /proc/cpuinfo) dist install clean \ + && apt-get remove -y \ + autoconf \ + automake \ + libexpat1-dev \ + g++ \ + libtool \ + liblz4-dev \ + m4 \ + make \ + zlib1g-dev \ + && apt-get autoremove -y \ + && mkdir -p /db/diffs /app/etc \ + && cp -r /app/src/rules /app/etc/rules \ + && rm -rf /app/src /app/src.tar.gz \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt /app/ + +RUN python3 -m venv /app/venv \ + && /app/venv/bin/pip install -r /app/requirements.txt + +RUN mkdir /nginx && chown nginx:nginx /nginx && chown -R overpass:overpass /db + +COPY etc/supervisord.conf /etc/supervisor/conf.d/supervisord.conf + +COPY etc/nginx-overpass.conf /etc/nginx/nginx.conf + +COPY bin/update_overpass.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/ + +COPY docker-entrypoint.sh /app/ + +RUN chmod a+x /app/docker-entrypoint.sh /app/bin/update_overpass.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh + +ENV OVERPASS_RULES_LOAD 1 + +EXPOSE 80 +# CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"] +CMD ["/app/docker-entrypoint.sh"] diff --git a/0.7.55.5/bin/dispatcher_start.sh b/0.7.55.5/bin/dispatcher_start.sh new file mode 100644 index 0000000..19a3c55 --- /dev/null +++ b/0.7.55.5/bin/dispatcher_start.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -e -o pipefail + +if [[ $OVERPASS_META == 'yes' ]] ; then + META_ARG='--meta' +elif [[ $OVERPASS_META == 'attic' ]] ; then + META_ARG='--attic' +else + META_ARG_='' +fi + +find /db/db -type s -print0 | xargs -0 --no-run-if-empty rm && /app/bin/dispatcher --osm-base "${META_ARG}" --db-dir=/db/db + diff --git a/0.7.55.5/bin/rules_loop.sh b/0.7.55.5/bin/rules_loop.sh new file mode 100644 index 0000000..a50f94f --- /dev/null +++ b/0.7.55.5/bin/rules_loop.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht +# +# This file is part of Overpass_API. +# +# Overpass_API is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Overpass_API is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Overpass_API. If not, see . + +if [[ -z $1 ]]; then +{ + echo "Usage: $0 database_dir [desired_cpu_load]" + exit 0 +}; +fi + +CPU_LOAD=${2:-100} +DB_DIR="`pwd`/$1" + +EXEC_DIR="`dirname $0`/" +if [[ ! ${EXEC_DIR:0:1} == "/" ]]; then +{ + EXEC_DIR="`pwd`/$EXEC_DIR" +}; +fi + +pushd "$EXEC_DIR" + +while [[ true ]]; do +{ + START=$(date +%s) + echo "`date '+%F %T'`: update started" >>$DB_DIR/rules_loop.log + ./osm3s_query --progress --rules <$DB_DIR/rules/areas.osm3s + echo "`date '+%F %T'`: update finished" >>$DB_DIR/rules_loop.log + WORK_TIME=$(( $(date +%s) - START )) + SLEEP_TIME=$(( WORK_TIME * 100 / CPU_LOAD - WORK_TIME)) + # let SLEEP_TIME be at least 3 seconds + SLEEP_TIME=$(( SLEEP_TIME < 3 ? 3 : SLEEP_TIME)) + echo "It took $WORK_TIME to run the loop. Desired load is: ${CPU_LOAD}%. Sleeping: $SLEEP_TIME" + sleep $SLEEP_TIME +}; done diff --git a/0.7.55.5/bin/update_overpass.sh b/0.7.55.5/bin/update_overpass.sh new file mode 100644 index 0000000..328e375 --- /dev/null +++ b/0.7.55.5/bin/update_overpass.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env sh + +( + set -e + + if [ "x$OVERPASS_META" = "xattic" ] ; then + META="--keep-attic" + elif [ "x$OVERPASS_META" = "xyes" ] ; then + META="--meta" + else + META="" + fi + + if [ ! -d /db/diffs ] ; then + mkdir /db/diffs + fi + + while `true` ; do + ( + if [ ! -e /db/diffs/changes.osm ] ; then + ! /app/venv/bin/pyosmium-get-changes --server $OVERPASS_DIFF_URL -o /db/diffs/changes.osm -f /db/replicate_id + OSMIUM_STATUS=$? + if [ $OSMIUM_STATUS -eq 1 ]; then + echo "There are still some updates remainging" + fi + if [ $OSMIUM_STATUS -eq 2 ]; then + echo "Failure downloading updates" + exit 0 + fi + else + echo "/db/diffs/changes.osm exists. Trying to apply again." + fi + /app/bin/update_from_dir --osc-dir=/db/diffs/ $META + rm /db/diffs/changes.osm + ) + sleep 60 + done +) 2>&1 | tee -a /db/changes.log \ No newline at end of file diff --git a/0.7.55.5/docker-entrypoint.sh b/0.7.55.5/docker-entrypoint.sh new file mode 100644 index 0000000..cb69627 --- /dev/null +++ b/0.7.55.5/docker-entrypoint.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +set -eo pipefail +shopt -s nullglob +OVERPASS_META=${OVERPASS_META:-no} +OVERPASS_MODE=${OVERPASS_MODE:-clone} +OVERPASS_COMPRESSION=${OVERPASS_COMPRESSION:-gz} + +if [[ "$OVERPASS_META" == "attic" ]] ; then + META="--keep-attic" +else + META="--meta" +fi + + +if [ ! -d /db/db ] ; then + if [ "$OVERPASS_MODE" = "clone" ]; then + mkdir -p /db/db \ + && /app/bin/download_clone.sh --db-dir=/db/db --source=http://dev.overpass-api.de/api_drolbr/ $META \ + && cp -r /app/etc/rules /db/db \ + && chown -R overpass:overpass /db \ + && echo "Overpass ready, you can start your container with docker start" + exit + fi + + if [ "$OVERPASS_MODE" = "init" ]; then + lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet.osm.bz2; exit" \ + && /app/bin/init_osm3s.sh /db/planet.osm.bz2 /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION --map-compression-method=$OVERPASS_COMPRESSION"\ + && echo "Database created. Now updating it." && ( + ! /app/venv/bin/pyosmium-get-changes -O /db/planet.osm.bz2 --server $OVERPASS_DIFF_URL -o /db/changes.osm -f /db/replicate_id + OSMIUM_STATUS=$? + if [ $OSMIUM_STATUS -eq 1 ]; then + echo "There are still some updates remainging" + fi + if [ $OSMIUM_STATUS -eq 2 ]; then + echo "Failure downloading updates" + exit 0 + fi + (cat /db/changes.osm | /app/bin/update_database --db-dir=/db/db $META --compression-method=$OVERPASS_COMPRESSION) 2>&1 | tee -a /db/changes.log + ) \ + && rm /db/planet.osm.bz2 /db/changes.osm \ + && cp -r /app/etc/rules /db/db \ + && chown -R overpass:overpass /db \ + && echo "Overpass ready, you can start your container with docker start" + exit + fi +fi + +exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf diff --git a/0.7.55.5/etc/nginx-overpass.conf b/0.7.55.5/etc/nginx-overpass.conf new file mode 100644 index 0000000..a19dde4 --- /dev/null +++ b/0.7.55.5/etc/nginx-overpass.conf @@ -0,0 +1,67 @@ +daemon off; + +user nginx; +worker_processes 4; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + + +events { + worker_connections 1024; +} + + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 1000s; + + #gzip on; + + server { + listen 80; + server_name localhost; + + #charset koi8-r; + #access_log /var/log/nginx/log/host.access.log main; + + location / { + root /usr/share/nginx/html; + index index.html index.htm; + } + + #error_page 404 /404.html; + + # redirect server error pages to the static page /50x.html + # + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } + + location /cgi-bin/ { + include fastcgi_params; + fastcgi_param SCRIPT_FILENAME /app/$fastcgi_script_name; + fastcgi_param PATH_INFO $fastcgi_path_info; + fastcgi_pass unix:/nginx/fcgiwrap.socket; + fastcgi_read_timeout 1000s; + } + + location /api/ { + rewrite ^/api/(.+)$ /cgi-bin/$1 last; + } + } + + +} diff --git a/0.7.55.5/etc/supervisord.conf b/0.7.55.5/etc/supervisord.conf new file mode 100644 index 0000000..fc67b74 --- /dev/null +++ b/0.7.55.5/etc/supervisord.conf @@ -0,0 +1,36 @@ +[supervisord] +nodaemon=true + +[program:overpass_dispatch] +command=/app/bin/dispatcher_start.sh +user=overpass +redirect_stderr=true +priority=1 + +[program:nginx] +command=nginx +priority=2 + +[program:fcgiwrap] +command=/bin/bash -c "find /nginx -type s -print0 | xargs -0 --no-run-if-empty rm && fcgiwrap -s unix:/nginx/fcgiwrap.socket" +redirect_stderr=true +user=nginx +priority=3 + +[program:update_overpass] +command=/app/bin/update_overpass.sh +user=overpass +redirect_stderr=true +priority=5 + +[program:dispatcher_areas] +command=nice /app/bin/dispatcher --areas --db-dir="/db/db" +user=overpass +redirect_stderr=true +priority=6 + +[program:areas_rules] +command=nice /app/bin/rules_loop.sh /db/db %(ENV_OVERPASS_RULES_LOAD)s +user=overpass +redirect_stderr=true +priority=7 diff --git a/0.7.55.5/requirements.txt b/0.7.55.5/requirements.txt new file mode 100644 index 0000000..8314012 --- /dev/null +++ b/0.7.55.5/requirements.txt @@ -0,0 +1 @@ +osmium diff --git a/0.7.55.6/Dockerfile b/0.7.55.6/Dockerfile new file mode 100644 index 0000000..7e5676b --- /dev/null +++ b/0.7.55.6/Dockerfile @@ -0,0 +1,79 @@ +FROM nginx:1.14 + +RUN addgroup overpass && adduser --home /db --disabled-password --gecos overpass --ingroup overpass overpass + +ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.55.6.tar.gz /app/src.tar.gz + +RUN apt-get update \ + && apt-get install --no-install-recommends --no-install-suggests -y \ + autoconf \ + automake \ + bash \ + bzip2 \ + ca-certificates \ + expat \ + fcgiwrap \ + g++ \ + lftp \ + libexpat1-dev \ + liblz4-1 \ + liblz4-dev \ + libtool \ + m4 \ + make \ + osmium-tool \ + python3 \ + python3-venv \ + supervisor \ + wget \ + zlib1g \ + zlib1g-dev \ + && mkdir -p /app/src \ + && cd /app/src \ + && tar -x -z --strip-components 1 -f ../src.tar.gz \ + && autoscan \ + && aclocal \ + && autoheader \ + && libtoolize \ + && automake --add-missing \ + && autoconf \ + && CXXFLAGS='-O2' CFLAGS='-O2' ./configure --prefix=/app --enable-lz4 \ + && make -j $(grep -c ^processor /proc/cpuinfo) dist install clean \ + && apt-get remove -y \ + autoconf \ + automake \ + libexpat1-dev \ + g++ \ + libtool \ + liblz4-dev \ + m4 \ + make \ + zlib1g-dev \ + && apt-get autoremove -y \ + && mkdir -p /db/diffs /app/etc \ + && cp -r /app/src/rules /app/etc/rules \ + && rm -rf /app/src /app/src.tar.gz \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt /app/ + +RUN python3 -m venv /app/venv \ + && /app/venv/bin/pip install -r /app/requirements.txt + +RUN mkdir /nginx && chown nginx:nginx /nginx && chown -R overpass:overpass /db + +COPY etc/supervisord.conf /etc/supervisor/conf.d/supervisord.conf + +COPY etc/nginx-overpass.conf /etc/nginx/nginx.conf + +COPY bin/update_overpass.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/ + +COPY docker-entrypoint.sh /app/ + +RUN chmod a+x /app/docker-entrypoint.sh /app/bin/update_overpass.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh + +ENV OVERPASS_RULES_LOAD 1 + +EXPOSE 80 +# CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"] +CMD ["/app/docker-entrypoint.sh"] diff --git a/0.7.55.6/bin/dispatcher_start.sh b/0.7.55.6/bin/dispatcher_start.sh new file mode 100644 index 0000000..19a3c55 --- /dev/null +++ b/0.7.55.6/bin/dispatcher_start.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -e -o pipefail + +if [[ $OVERPASS_META == 'yes' ]] ; then + META_ARG='--meta' +elif [[ $OVERPASS_META == 'attic' ]] ; then + META_ARG='--attic' +else + META_ARG_='' +fi + +find /db/db -type s -print0 | xargs -0 --no-run-if-empty rm && /app/bin/dispatcher --osm-base "${META_ARG}" --db-dir=/db/db + diff --git a/0.7.55.6/bin/rules_loop.sh b/0.7.55.6/bin/rules_loop.sh new file mode 100644 index 0000000..a50f94f --- /dev/null +++ b/0.7.55.6/bin/rules_loop.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht +# +# This file is part of Overpass_API. +# +# Overpass_API is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Overpass_API is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Overpass_API. If not, see . + +if [[ -z $1 ]]; then +{ + echo "Usage: $0 database_dir [desired_cpu_load]" + exit 0 +}; +fi + +CPU_LOAD=${2:-100} +DB_DIR="`pwd`/$1" + +EXEC_DIR="`dirname $0`/" +if [[ ! ${EXEC_DIR:0:1} == "/" ]]; then +{ + EXEC_DIR="`pwd`/$EXEC_DIR" +}; +fi + +pushd "$EXEC_DIR" + +while [[ true ]]; do +{ + START=$(date +%s) + echo "`date '+%F %T'`: update started" >>$DB_DIR/rules_loop.log + ./osm3s_query --progress --rules <$DB_DIR/rules/areas.osm3s + echo "`date '+%F %T'`: update finished" >>$DB_DIR/rules_loop.log + WORK_TIME=$(( $(date +%s) - START )) + SLEEP_TIME=$(( WORK_TIME * 100 / CPU_LOAD - WORK_TIME)) + # let SLEEP_TIME be at least 3 seconds + SLEEP_TIME=$(( SLEEP_TIME < 3 ? 3 : SLEEP_TIME)) + echo "It took $WORK_TIME to run the loop. Desired load is: ${CPU_LOAD}%. Sleeping: $SLEEP_TIME" + sleep $SLEEP_TIME +}; done diff --git a/0.7.55.6/bin/update_overpass.sh b/0.7.55.6/bin/update_overpass.sh new file mode 100644 index 0000000..328e375 --- /dev/null +++ b/0.7.55.6/bin/update_overpass.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env sh + +( + set -e + + if [ "x$OVERPASS_META" = "xattic" ] ; then + META="--keep-attic" + elif [ "x$OVERPASS_META" = "xyes" ] ; then + META="--meta" + else + META="" + fi + + if [ ! -d /db/diffs ] ; then + mkdir /db/diffs + fi + + while `true` ; do + ( + if [ ! -e /db/diffs/changes.osm ] ; then + ! /app/venv/bin/pyosmium-get-changes --server $OVERPASS_DIFF_URL -o /db/diffs/changes.osm -f /db/replicate_id + OSMIUM_STATUS=$? + if [ $OSMIUM_STATUS -eq 1 ]; then + echo "There are still some updates remainging" + fi + if [ $OSMIUM_STATUS -eq 2 ]; then + echo "Failure downloading updates" + exit 0 + fi + else + echo "/db/diffs/changes.osm exists. Trying to apply again." + fi + /app/bin/update_from_dir --osc-dir=/db/diffs/ $META + rm /db/diffs/changes.osm + ) + sleep 60 + done +) 2>&1 | tee -a /db/changes.log \ No newline at end of file diff --git a/0.7.55.6/docker-entrypoint.sh b/0.7.55.6/docker-entrypoint.sh new file mode 100644 index 0000000..cb69627 --- /dev/null +++ b/0.7.55.6/docker-entrypoint.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +set -eo pipefail +shopt -s nullglob +OVERPASS_META=${OVERPASS_META:-no} +OVERPASS_MODE=${OVERPASS_MODE:-clone} +OVERPASS_COMPRESSION=${OVERPASS_COMPRESSION:-gz} + +if [[ "$OVERPASS_META" == "attic" ]] ; then + META="--keep-attic" +else + META="--meta" +fi + + +if [ ! -d /db/db ] ; then + if [ "$OVERPASS_MODE" = "clone" ]; then + mkdir -p /db/db \ + && /app/bin/download_clone.sh --db-dir=/db/db --source=http://dev.overpass-api.de/api_drolbr/ $META \ + && cp -r /app/etc/rules /db/db \ + && chown -R overpass:overpass /db \ + && echo "Overpass ready, you can start your container with docker start" + exit + fi + + if [ "$OVERPASS_MODE" = "init" ]; then + lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet.osm.bz2; exit" \ + && /app/bin/init_osm3s.sh /db/planet.osm.bz2 /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION --map-compression-method=$OVERPASS_COMPRESSION"\ + && echo "Database created. Now updating it." && ( + ! /app/venv/bin/pyosmium-get-changes -O /db/planet.osm.bz2 --server $OVERPASS_DIFF_URL -o /db/changes.osm -f /db/replicate_id + OSMIUM_STATUS=$? + if [ $OSMIUM_STATUS -eq 1 ]; then + echo "There are still some updates remainging" + fi + if [ $OSMIUM_STATUS -eq 2 ]; then + echo "Failure downloading updates" + exit 0 + fi + (cat /db/changes.osm | /app/bin/update_database --db-dir=/db/db $META --compression-method=$OVERPASS_COMPRESSION) 2>&1 | tee -a /db/changes.log + ) \ + && rm /db/planet.osm.bz2 /db/changes.osm \ + && cp -r /app/etc/rules /db/db \ + && chown -R overpass:overpass /db \ + && echo "Overpass ready, you can start your container with docker start" + exit + fi +fi + +exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf diff --git a/0.7.55.6/etc/nginx-overpass.conf b/0.7.55.6/etc/nginx-overpass.conf new file mode 100644 index 0000000..a19dde4 --- /dev/null +++ b/0.7.55.6/etc/nginx-overpass.conf @@ -0,0 +1,67 @@ +daemon off; + +user nginx; +worker_processes 4; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + + +events { + worker_connections 1024; +} + + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 1000s; + + #gzip on; + + server { + listen 80; + server_name localhost; + + #charset koi8-r; + #access_log /var/log/nginx/log/host.access.log main; + + location / { + root /usr/share/nginx/html; + index index.html index.htm; + } + + #error_page 404 /404.html; + + # redirect server error pages to the static page /50x.html + # + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } + + location /cgi-bin/ { + include fastcgi_params; + fastcgi_param SCRIPT_FILENAME /app/$fastcgi_script_name; + fastcgi_param PATH_INFO $fastcgi_path_info; + fastcgi_pass unix:/nginx/fcgiwrap.socket; + fastcgi_read_timeout 1000s; + } + + location /api/ { + rewrite ^/api/(.+)$ /cgi-bin/$1 last; + } + } + + +} diff --git a/0.7.55.6/etc/supervisord.conf b/0.7.55.6/etc/supervisord.conf new file mode 100644 index 0000000..fc67b74 --- /dev/null +++ b/0.7.55.6/etc/supervisord.conf @@ -0,0 +1,36 @@ +[supervisord] +nodaemon=true + +[program:overpass_dispatch] +command=/app/bin/dispatcher_start.sh +user=overpass +redirect_stderr=true +priority=1 + +[program:nginx] +command=nginx +priority=2 + +[program:fcgiwrap] +command=/bin/bash -c "find /nginx -type s -print0 | xargs -0 --no-run-if-empty rm && fcgiwrap -s unix:/nginx/fcgiwrap.socket" +redirect_stderr=true +user=nginx +priority=3 + +[program:update_overpass] +command=/app/bin/update_overpass.sh +user=overpass +redirect_stderr=true +priority=5 + +[program:dispatcher_areas] +command=nice /app/bin/dispatcher --areas --db-dir="/db/db" +user=overpass +redirect_stderr=true +priority=6 + +[program:areas_rules] +command=nice /app/bin/rules_loop.sh /db/db %(ENV_OVERPASS_RULES_LOAD)s +user=overpass +redirect_stderr=true +priority=7 diff --git a/0.7.55.6/requirements.txt b/0.7.55.6/requirements.txt new file mode 100644 index 0000000..8314012 --- /dev/null +++ b/0.7.55.6/requirements.txt @@ -0,0 +1 @@ +osmium diff --git a/0.7.55/Dockerfile b/0.7.55/Dockerfile index eb993af..0bf3faa 100644 --- a/0.7.55/Dockerfile +++ b/0.7.55/Dockerfile @@ -2,26 +2,33 @@ FROM nginx:1.14 RUN addgroup overpass && adduser --home /db --disabled-password --gecos overpass --ingroup overpass overpass +ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.55.tar.gz /app/src.tar.gz + RUN apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ autoconf \ automake \ + bash \ + bzip2 \ ca-certificates \ expat \ - libexpat1-dev \ + fcgiwrap \ g++ \ + lftp \ + libexpat1-dev \ liblz4-1 \ liblz4-dev \ libtool \ m4 \ make \ + osmium-tool \ + python3 \ + python3-venv \ + supervisor \ + wget \ zlib1g \ - zlib1g-dev - - -ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.55.tar.gz /app/src.tar.gz - -RUN mkdir -p /app/src \ + zlib1g-dev \ + && mkdir -p /app/src \ && cd /app/src \ && tar -x -z --strip-components 1 -f ../src.tar.gz \ && autoscan \ @@ -45,26 +52,28 @@ RUN mkdir -p /app/src \ && apt-get autoremove -y \ && mkdir -p /db/diffs /app/etc \ && cp -r /app/src/rules /app/etc/rules \ - && rm -rf /app/src /app/src.tar.gz + && rm -rf /app/src /app/src.tar.gz \ + && rm -rf /var/lib/apt/lists/* -RUN apt-get install --no-install-recommends --no-install-suggests -y \ - supervisor \ - bash \ - lftp \ - wget \ - fcgiwrap \ - bzip2 +COPY requirements.txt /app/ +RUN python3 -m venv /app/venv \ + && /app/venv/bin/pip install -r /app/requirements.txt RUN mkdir /nginx && chown nginx:nginx /nginx && chown -R overpass:overpass /db + COPY etc/supervisord.conf /etc/supervisor/conf.d/supervisord.conf + COPY etc/nginx-overpass.conf /etc/nginx/nginx.conf -COPY bin/apply_osc_to_db.sh bin/fetch_osc.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/ + +COPY bin/update_overpass.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/ + COPY docker-entrypoint.sh /app/ -RUN chmod a+x /app/docker-entrypoint.sh /app/bin/apply_osc_to_db.sh /app/bin/fetch_osc.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh -VOLUME /db + +RUN chmod a+x /app/docker-entrypoint.sh /app/bin/update_overpass.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh ENV OVERPASS_RULES_LOAD 1 + EXPOSE 80 # CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"] CMD ["/app/docker-entrypoint.sh"] diff --git a/0.7.55/bin/apply_osc_to_db.sh b/0.7.55/bin/apply_osc_to_db.sh deleted file mode 100755 index 94d1879..0000000 --- a/0.7.55/bin/apply_osc_to_db.sh +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht -# -# This file is part of Overpass_API. -# -# Overpass_API is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Overpass_API is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Overpass_API. If not, see . - -if [[ -z $3 ]]; then -{ - echo "Usage: $0 replicate_dir start_id --meta=(attic|yes|no)" - exit 0 -}; fi - -EXEC_DIR="`dirname $0`/" -if [[ ! ${EXEC_DIR:0:1} == "/" ]]; then -{ - EXEC_DIR="`pwd`/$EXEC_DIR" -}; fi - -DB_DIR=`$EXEC_DIR/dispatcher --show-dir` - -if [[ ! -d $DB_DIR ]] ; then - echo "Can't find DB_DIR. Returned value: $DB_DIR" - exit 1; -fi - -REPLICATE_DIR="$1" -if [[ ! ${REPLICATE_DIR:0:1} == "/" ]]; then -{ - REPLICATE_DIR="`pwd`/$REPLICATE_DIR" -}; fi - -START=$2 -META= - -if [[ $3 == "--meta=attic" ]]; then - META="--keep-attic" -elif [[ $3 == "--meta=yes" || $3 == "--meta" ]]; then - META="--meta" -elif [[ $3 == "--meta=no" ]]; then - META= -else -{ - echo "You must specify --meta=yes or --meta=no" - exit 0 -}; fi - -PRODUCE_DIFF= - -get_replicate_filename() -{ - printf -v TDIGIT3 %03u $(($1 % 1000)) - ARG=$(($1 / 1000)) - printf -v TDIGIT2 %03u $(($ARG % 1000)) - ARG=$(($ARG / 1000)) - printf -v TDIGIT1 %03u $ARG - REPLICATE_TRUNK_DIR=$TDIGIT1/$TDIGIT2/ - REPLICATE_FILENAME=$TDIGIT1/$TDIGIT2/$TDIGIT3 -}; - - -collect_minute_diffs() -{ - TEMP_DIR=$1 - TARGET=$(($START + 1)) - - get_replicate_filename $TARGET - - while [[ ( -s $REPLICATE_DIR/$REPLICATE_FILENAME.state.txt ) && ( $(($START + 1440)) -ge $(($TARGET)) ) && ( `du -m $TEMP_DIR | awk '{ print $1; }'` -le 512 ) ]]; - do - { - printf -v TARGET_FILE %09u $TARGET - gunzip <$REPLICATE_DIR/$REPLICATE_FILENAME.osc.gz >$TEMP_DIR/$TARGET_FILE.osc - TARGET=$(($TARGET + 1)) - get_replicate_filename $TARGET - }; - done - TARGET=$(($TARGET - 1)) -}; - - -apply_minute_diffs() -{ - ./update_from_dir --osc-dir=$1 --version=$DATA_VERSION $META --flush-size=0 - EXITCODE=$? - while [[ $EXITCODE -ne 0 ]]; - do - { - sleep 60 - ./update_from_dir --osc-dir=$1 --version=$DATA_VERSION $META --flush-size=0 - EXITCODE=$? - }; - done - DIFF_COUNT=$(($DIFF_COUNT + 1)) -}; - - -update_state() -{ - get_replicate_filename $TARGET - TIMESTAMP_LINE=`grep "^timestamp" <$REPLICATE_DIR/$REPLICATE_FILENAME.state.txt` - while [[ -z $TIMESTAMP_LINE ]]; do - { - sleep 5 - TIMESTAMP_LINE=`grep "^timestamp" <$REPLICATE_DIR/$REPLICATE_FILENAME.state.txt` - }; done - DATA_VERSION=${TIMESTAMP_LINE:10} -}; - - -echo >>$DB_DIR/apply_osc_to_db.log - -mkdir -p $DB_DIR/augmented_diffs/ -DIFF_COUNT=0 - -# update_state - -pushd "$EXEC_DIR" - -while [[ true ]]; do -{ - if [[ $START == "auto" ]]; then - { - START=`cat $DB_DIR/replicate_id` - }; fi - - echo "`date -u '+%F %T'`: updating from $START" >>$DB_DIR/apply_osc_to_db.log - - TEMP_DIR=`mktemp -d /tmp/osm-3s_update_XXXXXX` - collect_minute_diffs $TEMP_DIR - - if [[ $TARGET -gt $START ]]; then - { - echo "`date -u '+%F %T'`: updating to $TARGET" >>$DB_DIR/apply_osc_to_db.log - - update_state - apply_minute_diffs $TEMP_DIR - echo "$TARGET" >$DB_DIR/replicate_id - - echo "`date -u '+%F %T'`: update complete" $TARGET >>$DB_DIR/apply_osc_to_db.log - }; - else - { - sleep 5 - }; fi - - rm -f $TEMP_DIR/* - rmdir $TEMP_DIR - - START=$TARGET -}; done diff --git a/0.7.55/bin/fetch_osc.sh b/0.7.55/bin/fetch_osc.sh deleted file mode 100755 index a9d163f..0000000 --- a/0.7.55/bin/fetch_osc.sh +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht -# -# This file is part of Overpass_API. -# -# Overpass_API is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Overpass_API is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Overpass_API. If not, see . - -if [[ -z $1 ]]; then -{ - echo Usage: $0 Replicate_id Source_dir Local_dir [Sleep] - exit 0 -}; -fi - -REPLICATE_ID=$1 -SOURCE_DIR=$2 -LOCAL_DIR=$3 -SLEEP_BETWEEN_DLS=15 # How long to sleep between download attempts (sec). Default: 15. See also FAIL_COUNTER_ALERT -FAIL_COUNTER_ALERT=20 # After how many sleep cycles do we get nervous -FAIL_COUNTER=0 -FILE_PANIC= - -if [[ ! -d $LOCAL_DIR ]]; - then { - mkdir $LOCAL_DIR -}; -fi - -if [[ $REPLICATE_ID == "auto" ]] ; then - REPLICATE_ID=`find $LOCAL_DIR -type f -name '*state.txt' -not -size 0 | sort | tail -n 1 | sed -e "s#^$LOCAL_DIR##" -e 's#[./]##g' -e 's#statetxt##' -e 's/^0\+//g'` - - EXEC_DIR="`dirname $0`/" - DB_DIR=`$EXEC_DIR/dispatcher --show-dir` - - if [[ ! -d $DB_DIR ]] ; then - echo "Can't find DB_DIR. Returned value: $DB_DIR" - exit 1; - fi - - if [[ "x$REPLICATE_ID" == "x" ]] ; then - REPLICATE_ID=`cat $DB_DIR/replicate_id` - fi - if [[ "x$REPLICATE_ID" == "x" ]] ; then - echo "Could not determine REPLICATE_ID. Exiting" - exit 1 - fi -fi - -# $1 - remote source -# $2 - local destination -fetch_file() -{ - wget -nv -O "$2" "$1" -}; - -retry_fetch_file() -{ - FILE_PANIC= - FAIL_COUNTER=0 - if [[ ! -s "$2" ]]; then { - fetch_file "$1" "$2" - if [[ "$3" == "gzip" ]]; then { - gunzip -t <"$2" - if [[ $? -ne 0 ]]; then { - rm "$2" - }; fi - }; fi - }; fi - until [[ -s "$2" || $FAIL_COUNTER -ge $FAIL_COUNTER_ALERT ]]; do { - FAIL_COUNTER=$(($FAIL_COUNTER+1)) - sleep $SLEEP_BETWEEN_DLS - fetch_file "$1" "$2" - if [[ "$3" == "gzip" ]]; then { - gunzip -t <"$2" - if [[ $? -ne 0 ]]; then { - rm "$2" - }; fi - }; fi - }; done - if [[ $FAIL_COUNTER -ge $FAIL_COUNTER_ALERT ]]; then - FILE_PANIC="true" - fi -}; - -file_panic() -{ - echo "fetch_osc()@"`date -u "+%F %T"`": upstream_delay $REPLICATE_ID" >>$LOCAL_DIR/fetch_osc.log - REPLICATE_ID=$(($REPLICATE_ID - 1)) - - printf -v TDIGIT3 %03u $(($REPLICATE_ID % 1000)) - ARG=$(($REPLICATE_ID / 1000)) - printf -v TDIGIT2 %03u $(($ARG % 1000)) - ARG=$(($ARG / 1000)) - printf -v TDIGIT1 %03u $ARG - - FILE_PANIC="true" - until [[ ! -n $FILE_PANIC ]]; do { - retry_fetch_file "$REMOTE_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.new.osc.gz" "gzip" - }; done - FILE_PANIC="true" - until [[ ! -n $FILE_PANIC ]]; do { - retry_fetch_file "$REMOTE_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.new.state.txt" "text" - }; done - - RES_GZIP=`diff -q "$LOCAL_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.new.osc.gz"` - RES_TEXT=`diff -q "$LOCAL_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.new.state.txt"` - if [[ -n $RES_GZIP || -n $RES_TEXT ]]; then - echo "fetch_osc()@"`date -u "+%F %T"`": file_panic $REPLICATE_ID" >>$LOCAL_DIR/fetch_osc.log - echo "fetch_osc()@"`date -u "+%F %T"`": $RES_GZIP" >>$LOCAL_DIR/fetch_osc.log - echo "fetch_osc()@"`date -u "+%F %T"`": $RES_TEXT" >>$LOCAL_DIR/fetch_osc.log - exit 1 - fi - - rm "$LOCAL_PATH/$TDIGIT3.new.osc.gz" - rm "$LOCAL_PATH/$TDIGIT3.new.state.txt" -}; - -fetch_minute_diff() -{ - printf -v TDIGIT3 %03u $(($REPLICATE_ID % 1000)) - ARG=$(($REPLICATE_ID / 1000)) - printf -v TDIGIT2 %03u $(($ARG % 1000)) - ARG=$(($ARG / 1000)) - printf -v TDIGIT1 %03u $ARG - - LOCAL_PATH="$LOCAL_DIR/$TDIGIT1/$TDIGIT2" - REMOTE_PATH="$SOURCE_DIR/$TDIGIT1/$TDIGIT2" - mkdir -p "$LOCAL_DIR/$TDIGIT1/$TDIGIT2" - - retry_fetch_file "$REMOTE_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.osc.gz" "gzip" - if [[ -n $FILE_PANIC ]]; then - file_panic - fi - retry_fetch_file "$REMOTE_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.state.txt" "text" - if [[ -n $FILE_PANIC ]]; then - file_panic - fi - - TIMESTAMP_LINE=`grep timestamp $LOCAL_DIR/$TDIGIT1/$TDIGIT2/$TDIGIT3.state.txt` - TIMESTAMP=${TIMESTAMP_LINE:10} -}; - -while [[ true ]]; -do -{ - REPLICATE_ID=$(($REPLICATE_ID + 1)) - fetch_minute_diff - echo "fetch_osc()@"`date -u "+%F %T"`": new_replicate_diff $REPLICATE_ID $TIMESTAMP" >>$LOCAL_DIR/fetch_osc.log - sleep 1 -}; -done diff --git a/0.7.55/bin/update_overpass.sh b/0.7.55/bin/update_overpass.sh new file mode 100644 index 0000000..328e375 --- /dev/null +++ b/0.7.55/bin/update_overpass.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env sh + +( + set -e + + if [ "x$OVERPASS_META" = "xattic" ] ; then + META="--keep-attic" + elif [ "x$OVERPASS_META" = "xyes" ] ; then + META="--meta" + else + META="" + fi + + if [ ! -d /db/diffs ] ; then + mkdir /db/diffs + fi + + while `true` ; do + ( + if [ ! -e /db/diffs/changes.osm ] ; then + ! /app/venv/bin/pyosmium-get-changes --server $OVERPASS_DIFF_URL -o /db/diffs/changes.osm -f /db/replicate_id + OSMIUM_STATUS=$? + if [ $OSMIUM_STATUS -eq 1 ]; then + echo "There are still some updates remainging" + fi + if [ $OSMIUM_STATUS -eq 2 ]; then + echo "Failure downloading updates" + exit 0 + fi + else + echo "/db/diffs/changes.osm exists. Trying to apply again." + fi + /app/bin/update_from_dir --osc-dir=/db/diffs/ $META + rm /db/diffs/changes.osm + ) + sleep 60 + done +) 2>&1 | tee -a /db/changes.log \ No newline at end of file diff --git a/0.7.55/docker-entrypoint.sh b/0.7.55/docker-entrypoint.sh index 5a43603..cb69627 100644 --- a/0.7.55/docker-entrypoint.sh +++ b/0.7.55/docker-entrypoint.sh @@ -6,13 +6,15 @@ OVERPASS_META=${OVERPASS_META:-no} OVERPASS_MODE=${OVERPASS_MODE:-clone} OVERPASS_COMPRESSION=${OVERPASS_COMPRESSION:-gz} +if [[ "$OVERPASS_META" == "attic" ]] ; then + META="--keep-attic" +else + META="--meta" +fi + + if [ ! -d /db/db ] ; then if [ "$OVERPASS_MODE" = "clone" ]; then - if [[ "$OVERPASS_META" == "attic" ]] ; then - META="--keep-attic" - else - META="--meta=$OVERPASS_META" - fi mkdir -p /db/db \ && /app/bin/download_clone.sh --db-dir=/db/db --source=http://dev.overpass-api.de/api_drolbr/ $META \ && cp -r /app/etc/rules /db/db \ @@ -22,10 +24,21 @@ if [ ! -d /db/db ] ; then fi if [ "$OVERPASS_MODE" = "init" ]; then - lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet; exit" \ - && /app/bin/init_osm3s.sh /db/planet /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION" \ - && echo $OVERPASS_PLANET_SEQUENCE_ID > /db/db/replicate_id \ - && rm /db/planet \ + lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet.osm.bz2; exit" \ + && /app/bin/init_osm3s.sh /db/planet.osm.bz2 /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION --map-compression-method=$OVERPASS_COMPRESSION"\ + && echo "Database created. Now updating it." && ( + ! /app/venv/bin/pyosmium-get-changes -O /db/planet.osm.bz2 --server $OVERPASS_DIFF_URL -o /db/changes.osm -f /db/replicate_id + OSMIUM_STATUS=$? + if [ $OSMIUM_STATUS -eq 1 ]; then + echo "There are still some updates remainging" + fi + if [ $OSMIUM_STATUS -eq 2 ]; then + echo "Failure downloading updates" + exit 0 + fi + (cat /db/changes.osm | /app/bin/update_database --db-dir=/db/db $META --compression-method=$OVERPASS_COMPRESSION) 2>&1 | tee -a /db/changes.log + ) \ + && rm /db/planet.osm.bz2 /db/changes.osm \ && cp -r /app/etc/rules /db/db \ && chown -R overpass:overpass /db \ && echo "Overpass ready, you can start your container with docker start" diff --git a/0.7.55/etc/supervisord.conf b/0.7.55/etc/supervisord.conf index 6c4623b..fc67b74 100644 --- a/0.7.55/etc/supervisord.conf +++ b/0.7.55/etc/supervisord.conf @@ -17,18 +17,12 @@ redirect_stderr=true user=nginx priority=3 -[program:fetch_diff] -command=/app/bin/fetch_osc.sh auto %(ENV_OVERPASS_DIFF_URL)s /db/diffs +[program:update_overpass] +command=/app/bin/update_overpass.sh user=overpass redirect_stderr=true priority=5 -[program:apply_diff] -command=/app/bin/apply_osc_to_db.sh /db/diffs/ auto --meta=%(ENV_OVERPASS_META)s -redirect_stderr=true -user=overpass -priority=4 - [program:dispatcher_areas] command=nice /app/bin/dispatcher --areas --db-dir="/db/db" user=overpass diff --git a/0.7.55/requirements.txt b/0.7.55/requirements.txt new file mode 100644 index 0000000..8314012 --- /dev/null +++ b/0.7.55/requirements.txt @@ -0,0 +1 @@ +osmium diff --git a/Dockerfile.template b/Dockerfile.template index 637b452..4eecf18 100644 --- a/Dockerfile.template +++ b/Dockerfile.template @@ -2,26 +2,33 @@ FROM nginx:1.14 RUN addgroup overpass && adduser --home /db --disabled-password --gecos overpass --ingroup overpass overpass +ADD http://dev.overpass-api.de/releases/osm-3s_v{version}.tar.gz /app/src.tar.gz + RUN apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ autoconf \ automake \ + bash \ + bzip2 \ ca-certificates \ expat \ - libexpat1-dev \ + fcgiwrap \ g++ \ + lftp \ + libexpat1-dev \ liblz4-1 \ liblz4-dev \ libtool \ m4 \ make \ + osmium-tool \ + python3 \ + python3-venv \ + supervisor \ + wget \ zlib1g \ - zlib1g-dev - - -ADD http://dev.overpass-api.de/releases/osm-3s_v{version}.tar.gz /app/src.tar.gz - -RUN mkdir -p /app/src \ + zlib1g-dev \ + && mkdir -p /app/src \ && cd /app/src \ && tar -x -z --strip-components 1 -f ../src.tar.gz \ && autoscan \ @@ -45,26 +52,28 @@ RUN mkdir -p /app/src \ && apt-get autoremove -y \ && mkdir -p /db/diffs /app/etc \ && cp -r /app/src/rules /app/etc/rules \ - && rm -rf /app/src /app/src.tar.gz + && rm -rf /app/src /app/src.tar.gz \ + && rm -rf /var/lib/apt/lists/* -RUN apt-get install --no-install-recommends --no-install-suggests -y \ - supervisor \ - bash \ - lftp \ - wget \ - fcgiwrap \ - bzip2 +COPY requirements.txt /app/ +RUN python3 -m venv /app/venv \ + && /app/venv/bin/pip install -r /app/requirements.txt RUN mkdir /nginx && chown nginx:nginx /nginx && chown -R overpass:overpass /db + COPY etc/supervisord.conf /etc/supervisor/conf.d/supervisord.conf + COPY etc/nginx-overpass.conf /etc/nginx/nginx.conf -COPY bin/apply_osc_to_db.sh bin/fetch_osc.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/ + +COPY bin/update_overpass.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/ + COPY docker-entrypoint.sh /app/ -RUN chmod a+x /app/docker-entrypoint.sh /app/bin/apply_osc_to_db.sh /app/bin/fetch_osc.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh -VOLUME /db + +RUN chmod a+x /app/docker-entrypoint.sh /app/bin/update_overpass.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh ENV OVERPASS_RULES_LOAD 1 + EXPOSE 80 # CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"] CMD ["/app/docker-entrypoint.sh"] diff --git a/README.md b/README.md index f6805ba..f08eb16 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,6 @@ To do it properly it needs a bit of guidance in form of enviroment variables. * `OVERPASS_META` - `yes`, `no` or `attic` - passed to Overpass as `--meta` or `--keep-attic` * `OVERPASS_DIFF_URL` - url to diff's directory for updating the instance (eg. https://planet.openstreetmap.org/replication/minute/) * `OVERPASS_PLANET_URL` - url to "planet" file in `init` mode -* `OVERPASS_PLANET_SEQUENCE_ID` - sequence identifier corresponding to planet file above. All files after this one will be applied * `OVERPASS_COMPRESSION` - takes values of `no`, `gz` or `lz4`. Specifies compression mode of the Overpass database. Ony useful in `init` mode. Defaults to `gz` * `OVERPASS_RULES_LOAD` - desired load generated by areas generation. Controls how long the script will sleep before regenerating @@ -19,9 +18,7 @@ then data is copied from http://dev.overpass-api.de/api_drolbr/ and then updated covering whole world. This mode works only with minute diffs. In `init` mode you need to point `OVERPASS_PLANET_URL` to address with planet (partial) dump. This file will be downloaded, -indexed by Overpass and later - updated using `OVERPASS_PLANET_SEQUENCE_ID` and `OVERPASS_DIFF_URL`. You need to check which -sequence number is for your planet file. Take it from desctiption or just take a sequence number a day before your planet -file is dated. +indexed by Overpass and later - updated using `OVERPASS_DIFF_URL`. Start initalization mode with `-i` and `-t` options to `docker run` so you will have a chance to monitor the progress of file downloads etc. @@ -43,7 +40,6 @@ docker run \ -e OVERPASS_MODE=init \ -e OVERPASS_PLANET_URL=http://download.geofabrik.de/europe/monaco-latest.osm.bz2 \ -e OVERPASS_DIFF_URL=http://download.openstreetmap.fr/replication/europe/monaco/minute/ \ - -e OVERPASS_PLANET_SEQUENCE_ID=2946200 \ -e OVERPASS_RULES_LOAD=10 \ -v /big/docker/overpass_db/:/db \ -p 12345:80 \ diff --git a/bin/apply_osc_to_db.sh b/bin/apply_osc_to_db.sh deleted file mode 100755 index 94d1879..0000000 --- a/bin/apply_osc_to_db.sh +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht -# -# This file is part of Overpass_API. -# -# Overpass_API is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Overpass_API is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Overpass_API. If not, see . - -if [[ -z $3 ]]; then -{ - echo "Usage: $0 replicate_dir start_id --meta=(attic|yes|no)" - exit 0 -}; fi - -EXEC_DIR="`dirname $0`/" -if [[ ! ${EXEC_DIR:0:1} == "/" ]]; then -{ - EXEC_DIR="`pwd`/$EXEC_DIR" -}; fi - -DB_DIR=`$EXEC_DIR/dispatcher --show-dir` - -if [[ ! -d $DB_DIR ]] ; then - echo "Can't find DB_DIR. Returned value: $DB_DIR" - exit 1; -fi - -REPLICATE_DIR="$1" -if [[ ! ${REPLICATE_DIR:0:1} == "/" ]]; then -{ - REPLICATE_DIR="`pwd`/$REPLICATE_DIR" -}; fi - -START=$2 -META= - -if [[ $3 == "--meta=attic" ]]; then - META="--keep-attic" -elif [[ $3 == "--meta=yes" || $3 == "--meta" ]]; then - META="--meta" -elif [[ $3 == "--meta=no" ]]; then - META= -else -{ - echo "You must specify --meta=yes or --meta=no" - exit 0 -}; fi - -PRODUCE_DIFF= - -get_replicate_filename() -{ - printf -v TDIGIT3 %03u $(($1 % 1000)) - ARG=$(($1 / 1000)) - printf -v TDIGIT2 %03u $(($ARG % 1000)) - ARG=$(($ARG / 1000)) - printf -v TDIGIT1 %03u $ARG - REPLICATE_TRUNK_DIR=$TDIGIT1/$TDIGIT2/ - REPLICATE_FILENAME=$TDIGIT1/$TDIGIT2/$TDIGIT3 -}; - - -collect_minute_diffs() -{ - TEMP_DIR=$1 - TARGET=$(($START + 1)) - - get_replicate_filename $TARGET - - while [[ ( -s $REPLICATE_DIR/$REPLICATE_FILENAME.state.txt ) && ( $(($START + 1440)) -ge $(($TARGET)) ) && ( `du -m $TEMP_DIR | awk '{ print $1; }'` -le 512 ) ]]; - do - { - printf -v TARGET_FILE %09u $TARGET - gunzip <$REPLICATE_DIR/$REPLICATE_FILENAME.osc.gz >$TEMP_DIR/$TARGET_FILE.osc - TARGET=$(($TARGET + 1)) - get_replicate_filename $TARGET - }; - done - TARGET=$(($TARGET - 1)) -}; - - -apply_minute_diffs() -{ - ./update_from_dir --osc-dir=$1 --version=$DATA_VERSION $META --flush-size=0 - EXITCODE=$? - while [[ $EXITCODE -ne 0 ]]; - do - { - sleep 60 - ./update_from_dir --osc-dir=$1 --version=$DATA_VERSION $META --flush-size=0 - EXITCODE=$? - }; - done - DIFF_COUNT=$(($DIFF_COUNT + 1)) -}; - - -update_state() -{ - get_replicate_filename $TARGET - TIMESTAMP_LINE=`grep "^timestamp" <$REPLICATE_DIR/$REPLICATE_FILENAME.state.txt` - while [[ -z $TIMESTAMP_LINE ]]; do - { - sleep 5 - TIMESTAMP_LINE=`grep "^timestamp" <$REPLICATE_DIR/$REPLICATE_FILENAME.state.txt` - }; done - DATA_VERSION=${TIMESTAMP_LINE:10} -}; - - -echo >>$DB_DIR/apply_osc_to_db.log - -mkdir -p $DB_DIR/augmented_diffs/ -DIFF_COUNT=0 - -# update_state - -pushd "$EXEC_DIR" - -while [[ true ]]; do -{ - if [[ $START == "auto" ]]; then - { - START=`cat $DB_DIR/replicate_id` - }; fi - - echo "`date -u '+%F %T'`: updating from $START" >>$DB_DIR/apply_osc_to_db.log - - TEMP_DIR=`mktemp -d /tmp/osm-3s_update_XXXXXX` - collect_minute_diffs $TEMP_DIR - - if [[ $TARGET -gt $START ]]; then - { - echo "`date -u '+%F %T'`: updating to $TARGET" >>$DB_DIR/apply_osc_to_db.log - - update_state - apply_minute_diffs $TEMP_DIR - echo "$TARGET" >$DB_DIR/replicate_id - - echo "`date -u '+%F %T'`: update complete" $TARGET >>$DB_DIR/apply_osc_to_db.log - }; - else - { - sleep 5 - }; fi - - rm -f $TEMP_DIR/* - rmdir $TEMP_DIR - - START=$TARGET -}; done diff --git a/bin/fetch_osc.sh b/bin/fetch_osc.sh deleted file mode 100755 index a9d163f..0000000 --- a/bin/fetch_osc.sh +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht -# -# This file is part of Overpass_API. -# -# Overpass_API is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Overpass_API is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Overpass_API. If not, see . - -if [[ -z $1 ]]; then -{ - echo Usage: $0 Replicate_id Source_dir Local_dir [Sleep] - exit 0 -}; -fi - -REPLICATE_ID=$1 -SOURCE_DIR=$2 -LOCAL_DIR=$3 -SLEEP_BETWEEN_DLS=15 # How long to sleep between download attempts (sec). Default: 15. See also FAIL_COUNTER_ALERT -FAIL_COUNTER_ALERT=20 # After how many sleep cycles do we get nervous -FAIL_COUNTER=0 -FILE_PANIC= - -if [[ ! -d $LOCAL_DIR ]]; - then { - mkdir $LOCAL_DIR -}; -fi - -if [[ $REPLICATE_ID == "auto" ]] ; then - REPLICATE_ID=`find $LOCAL_DIR -type f -name '*state.txt' -not -size 0 | sort | tail -n 1 | sed -e "s#^$LOCAL_DIR##" -e 's#[./]##g' -e 's#statetxt##' -e 's/^0\+//g'` - - EXEC_DIR="`dirname $0`/" - DB_DIR=`$EXEC_DIR/dispatcher --show-dir` - - if [[ ! -d $DB_DIR ]] ; then - echo "Can't find DB_DIR. Returned value: $DB_DIR" - exit 1; - fi - - if [[ "x$REPLICATE_ID" == "x" ]] ; then - REPLICATE_ID=`cat $DB_DIR/replicate_id` - fi - if [[ "x$REPLICATE_ID" == "x" ]] ; then - echo "Could not determine REPLICATE_ID. Exiting" - exit 1 - fi -fi - -# $1 - remote source -# $2 - local destination -fetch_file() -{ - wget -nv -O "$2" "$1" -}; - -retry_fetch_file() -{ - FILE_PANIC= - FAIL_COUNTER=0 - if [[ ! -s "$2" ]]; then { - fetch_file "$1" "$2" - if [[ "$3" == "gzip" ]]; then { - gunzip -t <"$2" - if [[ $? -ne 0 ]]; then { - rm "$2" - }; fi - }; fi - }; fi - until [[ -s "$2" || $FAIL_COUNTER -ge $FAIL_COUNTER_ALERT ]]; do { - FAIL_COUNTER=$(($FAIL_COUNTER+1)) - sleep $SLEEP_BETWEEN_DLS - fetch_file "$1" "$2" - if [[ "$3" == "gzip" ]]; then { - gunzip -t <"$2" - if [[ $? -ne 0 ]]; then { - rm "$2" - }; fi - }; fi - }; done - if [[ $FAIL_COUNTER -ge $FAIL_COUNTER_ALERT ]]; then - FILE_PANIC="true" - fi -}; - -file_panic() -{ - echo "fetch_osc()@"`date -u "+%F %T"`": upstream_delay $REPLICATE_ID" >>$LOCAL_DIR/fetch_osc.log - REPLICATE_ID=$(($REPLICATE_ID - 1)) - - printf -v TDIGIT3 %03u $(($REPLICATE_ID % 1000)) - ARG=$(($REPLICATE_ID / 1000)) - printf -v TDIGIT2 %03u $(($ARG % 1000)) - ARG=$(($ARG / 1000)) - printf -v TDIGIT1 %03u $ARG - - FILE_PANIC="true" - until [[ ! -n $FILE_PANIC ]]; do { - retry_fetch_file "$REMOTE_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.new.osc.gz" "gzip" - }; done - FILE_PANIC="true" - until [[ ! -n $FILE_PANIC ]]; do { - retry_fetch_file "$REMOTE_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.new.state.txt" "text" - }; done - - RES_GZIP=`diff -q "$LOCAL_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.new.osc.gz"` - RES_TEXT=`diff -q "$LOCAL_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.new.state.txt"` - if [[ -n $RES_GZIP || -n $RES_TEXT ]]; then - echo "fetch_osc()@"`date -u "+%F %T"`": file_panic $REPLICATE_ID" >>$LOCAL_DIR/fetch_osc.log - echo "fetch_osc()@"`date -u "+%F %T"`": $RES_GZIP" >>$LOCAL_DIR/fetch_osc.log - echo "fetch_osc()@"`date -u "+%F %T"`": $RES_TEXT" >>$LOCAL_DIR/fetch_osc.log - exit 1 - fi - - rm "$LOCAL_PATH/$TDIGIT3.new.osc.gz" - rm "$LOCAL_PATH/$TDIGIT3.new.state.txt" -}; - -fetch_minute_diff() -{ - printf -v TDIGIT3 %03u $(($REPLICATE_ID % 1000)) - ARG=$(($REPLICATE_ID / 1000)) - printf -v TDIGIT2 %03u $(($ARG % 1000)) - ARG=$(($ARG / 1000)) - printf -v TDIGIT1 %03u $ARG - - LOCAL_PATH="$LOCAL_DIR/$TDIGIT1/$TDIGIT2" - REMOTE_PATH="$SOURCE_DIR/$TDIGIT1/$TDIGIT2" - mkdir -p "$LOCAL_DIR/$TDIGIT1/$TDIGIT2" - - retry_fetch_file "$REMOTE_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.osc.gz" "gzip" - if [[ -n $FILE_PANIC ]]; then - file_panic - fi - retry_fetch_file "$REMOTE_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.state.txt" "text" - if [[ -n $FILE_PANIC ]]; then - file_panic - fi - - TIMESTAMP_LINE=`grep timestamp $LOCAL_DIR/$TDIGIT1/$TDIGIT2/$TDIGIT3.state.txt` - TIMESTAMP=${TIMESTAMP_LINE:10} -}; - -while [[ true ]]; -do -{ - REPLICATE_ID=$(($REPLICATE_ID + 1)) - fetch_minute_diff - echo "fetch_osc()@"`date -u "+%F %T"`": new_replicate_diff $REPLICATE_ID $TIMESTAMP" >>$LOCAL_DIR/fetch_osc.log - sleep 1 -}; -done diff --git a/bin/update_overpass.sh b/bin/update_overpass.sh new file mode 100644 index 0000000..328e375 --- /dev/null +++ b/bin/update_overpass.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env sh + +( + set -e + + if [ "x$OVERPASS_META" = "xattic" ] ; then + META="--keep-attic" + elif [ "x$OVERPASS_META" = "xyes" ] ; then + META="--meta" + else + META="" + fi + + if [ ! -d /db/diffs ] ; then + mkdir /db/diffs + fi + + while `true` ; do + ( + if [ ! -e /db/diffs/changes.osm ] ; then + ! /app/venv/bin/pyosmium-get-changes --server $OVERPASS_DIFF_URL -o /db/diffs/changes.osm -f /db/replicate_id + OSMIUM_STATUS=$? + if [ $OSMIUM_STATUS -eq 1 ]; then + echo "There are still some updates remainging" + fi + if [ $OSMIUM_STATUS -eq 2 ]; then + echo "Failure downloading updates" + exit 0 + fi + else + echo "/db/diffs/changes.osm exists. Trying to apply again." + fi + /app/bin/update_from_dir --osc-dir=/db/diffs/ $META + rm /db/diffs/changes.osm + ) + sleep 60 + done +) 2>&1 | tee -a /db/changes.log \ No newline at end of file diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 5a43603..cb69627 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -6,13 +6,15 @@ OVERPASS_META=${OVERPASS_META:-no} OVERPASS_MODE=${OVERPASS_MODE:-clone} OVERPASS_COMPRESSION=${OVERPASS_COMPRESSION:-gz} +if [[ "$OVERPASS_META" == "attic" ]] ; then + META="--keep-attic" +else + META="--meta" +fi + + if [ ! -d /db/db ] ; then if [ "$OVERPASS_MODE" = "clone" ]; then - if [[ "$OVERPASS_META" == "attic" ]] ; then - META="--keep-attic" - else - META="--meta=$OVERPASS_META" - fi mkdir -p /db/db \ && /app/bin/download_clone.sh --db-dir=/db/db --source=http://dev.overpass-api.de/api_drolbr/ $META \ && cp -r /app/etc/rules /db/db \ @@ -22,10 +24,21 @@ if [ ! -d /db/db ] ; then fi if [ "$OVERPASS_MODE" = "init" ]; then - lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet; exit" \ - && /app/bin/init_osm3s.sh /db/planet /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION" \ - && echo $OVERPASS_PLANET_SEQUENCE_ID > /db/db/replicate_id \ - && rm /db/planet \ + lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet.osm.bz2; exit" \ + && /app/bin/init_osm3s.sh /db/planet.osm.bz2 /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION --map-compression-method=$OVERPASS_COMPRESSION"\ + && echo "Database created. Now updating it." && ( + ! /app/venv/bin/pyosmium-get-changes -O /db/planet.osm.bz2 --server $OVERPASS_DIFF_URL -o /db/changes.osm -f /db/replicate_id + OSMIUM_STATUS=$? + if [ $OSMIUM_STATUS -eq 1 ]; then + echo "There are still some updates remainging" + fi + if [ $OSMIUM_STATUS -eq 2 ]; then + echo "Failure downloading updates" + exit 0 + fi + (cat /db/changes.osm | /app/bin/update_database --db-dir=/db/db $META --compression-method=$OVERPASS_COMPRESSION) 2>&1 | tee -a /db/changes.log + ) \ + && rm /db/planet.osm.bz2 /db/changes.osm \ && cp -r /app/etc/rules /db/db \ && chown -R overpass:overpass /db \ && echo "Overpass ready, you can start your container with docker start" diff --git a/etc/supervisord.conf b/etc/supervisord.conf index 6c4623b..fc67b74 100644 --- a/etc/supervisord.conf +++ b/etc/supervisord.conf @@ -17,18 +17,12 @@ redirect_stderr=true user=nginx priority=3 -[program:fetch_diff] -command=/app/bin/fetch_osc.sh auto %(ENV_OVERPASS_DIFF_URL)s /db/diffs +[program:update_overpass] +command=/app/bin/update_overpass.sh user=overpass redirect_stderr=true priority=5 -[program:apply_diff] -command=/app/bin/apply_osc_to_db.sh /db/diffs/ auto --meta=%(ENV_OVERPASS_META)s -redirect_stderr=true -user=overpass -priority=4 - [program:dispatcher_areas] command=nice /app/bin/dispatcher --areas --db-dir="/db/db" user=overpass diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..8314012 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +osmium diff --git a/update.py b/update.py index 55105b5..d8209b6 100644 --- a/update.py +++ b/update.py @@ -8,6 +8,9 @@ class VersionFinder(html.parser.HTMLParser): + def error(self, message): + raise RuntimeError(message) + def __init__(self): super().__init__() self.versions = [] @@ -28,12 +31,23 @@ def main(): with open("Dockerfile.template") as f: template = f.read() for ver in parser.versions: + if any((ver.startswith(x) for x in ('0.6', 'eta', '0.7.1', '0.7.2', '0.7.3', '0.7.4', '0.7.50', '0.7.52', + '0.7.54.11', # invalid CRC in archive + '0.7.51', # no autoconf + ))) or \ + ver == '0.7': + # ignore old releases + continue + if os.path.exists(ver): + shutil.rmtree(ver) os.mkdir(ver) with open(pathlib.Path(ver) / "Dockerfile", "w+") as f: f.write(template.format(version=ver)) for i in ("etc", "bin"): - shutil.copytree(i, pathlib.Path(ver) / i) + shutil.copytree(i, pathlib.Path(ver) / i) shutil.copyfile("docker-entrypoint.sh", pathlib.Path(ver) / "docker-entrypoint.sh") + shutil.copyfile("requirements.txt", pathlib.Path(ver) / "requirements.txt") + if __name__ == '__main__': main()