diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..485dee6
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+.idea
diff --git a/0.7.51/bin/apply_osc_to_db.sh b/0.7.51/bin/apply_osc_to_db.sh
deleted file mode 100755
index 94d1879..0000000
--- a/0.7.51/bin/apply_osc_to_db.sh
+++ /dev/null
@@ -1,163 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht
-#
-# This file is part of Overpass_API.
-#
-# Overpass_API is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# Overpass_API is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with Overpass_API. If not, see .
-
-if [[ -z $3 ]]; then
-{
- echo "Usage: $0 replicate_dir start_id --meta=(attic|yes|no)"
- exit 0
-}; fi
-
-EXEC_DIR="`dirname $0`/"
-if [[ ! ${EXEC_DIR:0:1} == "/" ]]; then
-{
- EXEC_DIR="`pwd`/$EXEC_DIR"
-}; fi
-
-DB_DIR=`$EXEC_DIR/dispatcher --show-dir`
-
-if [[ ! -d $DB_DIR ]] ; then
- echo "Can't find DB_DIR. Returned value: $DB_DIR"
- exit 1;
-fi
-
-REPLICATE_DIR="$1"
-if [[ ! ${REPLICATE_DIR:0:1} == "/" ]]; then
-{
- REPLICATE_DIR="`pwd`/$REPLICATE_DIR"
-}; fi
-
-START=$2
-META=
-
-if [[ $3 == "--meta=attic" ]]; then
- META="--keep-attic"
-elif [[ $3 == "--meta=yes" || $3 == "--meta" ]]; then
- META="--meta"
-elif [[ $3 == "--meta=no" ]]; then
- META=
-else
-{
- echo "You must specify --meta=yes or --meta=no"
- exit 0
-}; fi
-
-PRODUCE_DIFF=
-
-get_replicate_filename()
-{
- printf -v TDIGIT3 %03u $(($1 % 1000))
- ARG=$(($1 / 1000))
- printf -v TDIGIT2 %03u $(($ARG % 1000))
- ARG=$(($ARG / 1000))
- printf -v TDIGIT1 %03u $ARG
- REPLICATE_TRUNK_DIR=$TDIGIT1/$TDIGIT2/
- REPLICATE_FILENAME=$TDIGIT1/$TDIGIT2/$TDIGIT3
-};
-
-
-collect_minute_diffs()
-{
- TEMP_DIR=$1
- TARGET=$(($START + 1))
-
- get_replicate_filename $TARGET
-
- while [[ ( -s $REPLICATE_DIR/$REPLICATE_FILENAME.state.txt ) && ( $(($START + 1440)) -ge $(($TARGET)) ) && ( `du -m $TEMP_DIR | awk '{ print $1; }'` -le 512 ) ]];
- do
- {
- printf -v TARGET_FILE %09u $TARGET
- gunzip <$REPLICATE_DIR/$REPLICATE_FILENAME.osc.gz >$TEMP_DIR/$TARGET_FILE.osc
- TARGET=$(($TARGET + 1))
- get_replicate_filename $TARGET
- };
- done
- TARGET=$(($TARGET - 1))
-};
-
-
-apply_minute_diffs()
-{
- ./update_from_dir --osc-dir=$1 --version=$DATA_VERSION $META --flush-size=0
- EXITCODE=$?
- while [[ $EXITCODE -ne 0 ]];
- do
- {
- sleep 60
- ./update_from_dir --osc-dir=$1 --version=$DATA_VERSION $META --flush-size=0
- EXITCODE=$?
- };
- done
- DIFF_COUNT=$(($DIFF_COUNT + 1))
-};
-
-
-update_state()
-{
- get_replicate_filename $TARGET
- TIMESTAMP_LINE=`grep "^timestamp" <$REPLICATE_DIR/$REPLICATE_FILENAME.state.txt`
- while [[ -z $TIMESTAMP_LINE ]]; do
- {
- sleep 5
- TIMESTAMP_LINE=`grep "^timestamp" <$REPLICATE_DIR/$REPLICATE_FILENAME.state.txt`
- }; done
- DATA_VERSION=${TIMESTAMP_LINE:10}
-};
-
-
-echo >>$DB_DIR/apply_osc_to_db.log
-
-mkdir -p $DB_DIR/augmented_diffs/
-DIFF_COUNT=0
-
-# update_state
-
-pushd "$EXEC_DIR"
-
-while [[ true ]]; do
-{
- if [[ $START == "auto" ]]; then
- {
- START=`cat $DB_DIR/replicate_id`
- }; fi
-
- echo "`date -u '+%F %T'`: updating from $START" >>$DB_DIR/apply_osc_to_db.log
-
- TEMP_DIR=`mktemp -d /tmp/osm-3s_update_XXXXXX`
- collect_minute_diffs $TEMP_DIR
-
- if [[ $TARGET -gt $START ]]; then
- {
- echo "`date -u '+%F %T'`: updating to $TARGET" >>$DB_DIR/apply_osc_to_db.log
-
- update_state
- apply_minute_diffs $TEMP_DIR
- echo "$TARGET" >$DB_DIR/replicate_id
-
- echo "`date -u '+%F %T'`: update complete" $TARGET >>$DB_DIR/apply_osc_to_db.log
- };
- else
- {
- sleep 5
- }; fi
-
- rm -f $TEMP_DIR/*
- rmdir $TEMP_DIR
-
- START=$TARGET
-}; done
diff --git a/0.7.51/bin/fetch_osc.sh b/0.7.51/bin/fetch_osc.sh
deleted file mode 100755
index a9d163f..0000000
--- a/0.7.51/bin/fetch_osc.sh
+++ /dev/null
@@ -1,163 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht
-#
-# This file is part of Overpass_API.
-#
-# Overpass_API is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# Overpass_API is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with Overpass_API. If not, see .
-
-if [[ -z $1 ]]; then
-{
- echo Usage: $0 Replicate_id Source_dir Local_dir [Sleep]
- exit 0
-};
-fi
-
-REPLICATE_ID=$1
-SOURCE_DIR=$2
-LOCAL_DIR=$3
-SLEEP_BETWEEN_DLS=15 # How long to sleep between download attempts (sec). Default: 15. See also FAIL_COUNTER_ALERT
-FAIL_COUNTER_ALERT=20 # After how many sleep cycles do we get nervous
-FAIL_COUNTER=0
-FILE_PANIC=
-
-if [[ ! -d $LOCAL_DIR ]];
- then {
- mkdir $LOCAL_DIR
-};
-fi
-
-if [[ $REPLICATE_ID == "auto" ]] ; then
- REPLICATE_ID=`find $LOCAL_DIR -type f -name '*state.txt' -not -size 0 | sort | tail -n 1 | sed -e "s#^$LOCAL_DIR##" -e 's#[./]##g' -e 's#statetxt##' -e 's/^0\+//g'`
-
- EXEC_DIR="`dirname $0`/"
- DB_DIR=`$EXEC_DIR/dispatcher --show-dir`
-
- if [[ ! -d $DB_DIR ]] ; then
- echo "Can't find DB_DIR. Returned value: $DB_DIR"
- exit 1;
- fi
-
- if [[ "x$REPLICATE_ID" == "x" ]] ; then
- REPLICATE_ID=`cat $DB_DIR/replicate_id`
- fi
- if [[ "x$REPLICATE_ID" == "x" ]] ; then
- echo "Could not determine REPLICATE_ID. Exiting"
- exit 1
- fi
-fi
-
-# $1 - remote source
-# $2 - local destination
-fetch_file()
-{
- wget -nv -O "$2" "$1"
-};
-
-retry_fetch_file()
-{
- FILE_PANIC=
- FAIL_COUNTER=0
- if [[ ! -s "$2" ]]; then {
- fetch_file "$1" "$2"
- if [[ "$3" == "gzip" ]]; then {
- gunzip -t <"$2"
- if [[ $? -ne 0 ]]; then {
- rm "$2"
- }; fi
- }; fi
- }; fi
- until [[ -s "$2" || $FAIL_COUNTER -ge $FAIL_COUNTER_ALERT ]]; do {
- FAIL_COUNTER=$(($FAIL_COUNTER+1))
- sleep $SLEEP_BETWEEN_DLS
- fetch_file "$1" "$2"
- if [[ "$3" == "gzip" ]]; then {
- gunzip -t <"$2"
- if [[ $? -ne 0 ]]; then {
- rm "$2"
- }; fi
- }; fi
- }; done
- if [[ $FAIL_COUNTER -ge $FAIL_COUNTER_ALERT ]]; then
- FILE_PANIC="true"
- fi
-};
-
-file_panic()
-{
- echo "fetch_osc()@"`date -u "+%F %T"`": upstream_delay $REPLICATE_ID" >>$LOCAL_DIR/fetch_osc.log
- REPLICATE_ID=$(($REPLICATE_ID - 1))
-
- printf -v TDIGIT3 %03u $(($REPLICATE_ID % 1000))
- ARG=$(($REPLICATE_ID / 1000))
- printf -v TDIGIT2 %03u $(($ARG % 1000))
- ARG=$(($ARG / 1000))
- printf -v TDIGIT1 %03u $ARG
-
- FILE_PANIC="true"
- until [[ ! -n $FILE_PANIC ]]; do {
- retry_fetch_file "$REMOTE_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.new.osc.gz" "gzip"
- }; done
- FILE_PANIC="true"
- until [[ ! -n $FILE_PANIC ]]; do {
- retry_fetch_file "$REMOTE_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.new.state.txt" "text"
- }; done
-
- RES_GZIP=`diff -q "$LOCAL_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.new.osc.gz"`
- RES_TEXT=`diff -q "$LOCAL_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.new.state.txt"`
- if [[ -n $RES_GZIP || -n $RES_TEXT ]]; then
- echo "fetch_osc()@"`date -u "+%F %T"`": file_panic $REPLICATE_ID" >>$LOCAL_DIR/fetch_osc.log
- echo "fetch_osc()@"`date -u "+%F %T"`": $RES_GZIP" >>$LOCAL_DIR/fetch_osc.log
- echo "fetch_osc()@"`date -u "+%F %T"`": $RES_TEXT" >>$LOCAL_DIR/fetch_osc.log
- exit 1
- fi
-
- rm "$LOCAL_PATH/$TDIGIT3.new.osc.gz"
- rm "$LOCAL_PATH/$TDIGIT3.new.state.txt"
-};
-
-fetch_minute_diff()
-{
- printf -v TDIGIT3 %03u $(($REPLICATE_ID % 1000))
- ARG=$(($REPLICATE_ID / 1000))
- printf -v TDIGIT2 %03u $(($ARG % 1000))
- ARG=$(($ARG / 1000))
- printf -v TDIGIT1 %03u $ARG
-
- LOCAL_PATH="$LOCAL_DIR/$TDIGIT1/$TDIGIT2"
- REMOTE_PATH="$SOURCE_DIR/$TDIGIT1/$TDIGIT2"
- mkdir -p "$LOCAL_DIR/$TDIGIT1/$TDIGIT2"
-
- retry_fetch_file "$REMOTE_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.osc.gz" "gzip"
- if [[ -n $FILE_PANIC ]]; then
- file_panic
- fi
- retry_fetch_file "$REMOTE_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.state.txt" "text"
- if [[ -n $FILE_PANIC ]]; then
- file_panic
- fi
-
- TIMESTAMP_LINE=`grep timestamp $LOCAL_DIR/$TDIGIT1/$TDIGIT2/$TDIGIT3.state.txt`
- TIMESTAMP=${TIMESTAMP_LINE:10}
-};
-
-while [[ true ]];
-do
-{
- REPLICATE_ID=$(($REPLICATE_ID + 1))
- fetch_minute_diff
- echo "fetch_osc()@"`date -u "+%F %T"`": new_replicate_diff $REPLICATE_ID $TIMESTAMP" >>$LOCAL_DIR/fetch_osc.log
- sleep 1
-};
-done
diff --git a/0.7.51/docker-entrypoint.sh b/0.7.51/docker-entrypoint.sh
deleted file mode 100644
index 5a43603..0000000
--- a/0.7.51/docker-entrypoint.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/bin/bash
-
-set -eo pipefail
-shopt -s nullglob
-OVERPASS_META=${OVERPASS_META:-no}
-OVERPASS_MODE=${OVERPASS_MODE:-clone}
-OVERPASS_COMPRESSION=${OVERPASS_COMPRESSION:-gz}
-
-if [ ! -d /db/db ] ; then
- if [ "$OVERPASS_MODE" = "clone" ]; then
- if [[ "$OVERPASS_META" == "attic" ]] ; then
- META="--keep-attic"
- else
- META="--meta=$OVERPASS_META"
- fi
- mkdir -p /db/db \
- && /app/bin/download_clone.sh --db-dir=/db/db --source=http://dev.overpass-api.de/api_drolbr/ $META \
- && cp -r /app/etc/rules /db/db \
- && chown -R overpass:overpass /db \
- && echo "Overpass ready, you can start your container with docker start"
- exit
- fi
-
- if [ "$OVERPASS_MODE" = "init" ]; then
- lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet; exit" \
- && /app/bin/init_osm3s.sh /db/planet /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION" \
- && echo $OVERPASS_PLANET_SEQUENCE_ID > /db/db/replicate_id \
- && rm /db/planet \
- && cp -r /app/etc/rules /db/db \
- && chown -R overpass:overpass /db \
- && echo "Overpass ready, you can start your container with docker start"
- exit
- fi
-fi
-
-exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf
diff --git a/0.7.53/Dockerfile b/0.7.53/Dockerfile
index f14622a..adc83bb 100644
--- a/0.7.53/Dockerfile
+++ b/0.7.53/Dockerfile
@@ -1,27 +1,34 @@
-FROM nginx:stable
+FROM nginx:1.14
RUN addgroup overpass && adduser --home /db --disabled-password --gecos overpass --ingroup overpass overpass
+ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.53.tar.gz /app/src.tar.gz
+
RUN apt-get update \
&& apt-get install --no-install-recommends --no-install-suggests -y \
autoconf \
automake \
+ bash \
+ bzip2 \
ca-certificates \
expat \
- libexpat1-dev \
+ fcgiwrap \
g++ \
+ lftp \
+ libexpat1-dev \
liblz4-1 \
liblz4-dev \
libtool \
m4 \
make \
+ osmium-tool \
+ python3 \
+ python3-venv \
+ supervisor \
+ wget \
zlib1g \
- zlib1g-dev
-
-
-ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.53.tar.gz /app/src.tar.gz
-
-RUN mkdir -p /app/src \
+ zlib1g-dev \
+ && mkdir -p /app/src \
&& cd /app/src \
&& tar -x -z --strip-components 1 -f ../src.tar.gz \
&& autoscan \
@@ -45,26 +52,28 @@ RUN mkdir -p /app/src \
&& apt-get autoremove -y \
&& mkdir -p /db/diffs /app/etc \
&& cp -r /app/src/rules /app/etc/rules \
- && rm -rf /app/src /app/src.tar.gz
+ && rm -rf /app/src /app/src.tar.gz \
+ && rm -rf /var/lib/apt/lists/*
-RUN apt-get install --no-install-recommends --no-install-suggests -y \
- supervisor \
- bash \
- lftp \
- wget \
- fcgiwrap \
- bzip2
+COPY requirements.txt /app/
+RUN python3 -m venv /app/venv \
+ && /app/venv/bin/pip install -r /app/requirements.txt
RUN mkdir /nginx && chown nginx:nginx /nginx && chown -R overpass:overpass /db
+
COPY etc/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+
COPY etc/nginx-overpass.conf /etc/nginx/nginx.conf
-COPY bin/apply_osc_to_db.sh bin/fetch_osc.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/
+
+COPY bin/update_overpass.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/
+
COPY docker-entrypoint.sh /app/
-RUN chmod a+x /app/docker-entrypoint.sh /app/bin/apply_osc_to_db.sh /app/bin/fetch_osc.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh
-VOLUME /db
+
+RUN chmod a+x /app/docker-entrypoint.sh /app/bin/update_overpass.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh
ENV OVERPASS_RULES_LOAD 1
+
EXPOSE 80
# CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"]
CMD ["/app/docker-entrypoint.sh"]
diff --git a/0.7.53/bin/apply_osc_to_db.sh b/0.7.53/bin/apply_osc_to_db.sh
deleted file mode 100755
index 94d1879..0000000
--- a/0.7.53/bin/apply_osc_to_db.sh
+++ /dev/null
@@ -1,163 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht
-#
-# This file is part of Overpass_API.
-#
-# Overpass_API is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# Overpass_API is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with Overpass_API. If not, see .
-
-if [[ -z $3 ]]; then
-{
- echo "Usage: $0 replicate_dir start_id --meta=(attic|yes|no)"
- exit 0
-}; fi
-
-EXEC_DIR="`dirname $0`/"
-if [[ ! ${EXEC_DIR:0:1} == "/" ]]; then
-{
- EXEC_DIR="`pwd`/$EXEC_DIR"
-}; fi
-
-DB_DIR=`$EXEC_DIR/dispatcher --show-dir`
-
-if [[ ! -d $DB_DIR ]] ; then
- echo "Can't find DB_DIR. Returned value: $DB_DIR"
- exit 1;
-fi
-
-REPLICATE_DIR="$1"
-if [[ ! ${REPLICATE_DIR:0:1} == "/" ]]; then
-{
- REPLICATE_DIR="`pwd`/$REPLICATE_DIR"
-}; fi
-
-START=$2
-META=
-
-if [[ $3 == "--meta=attic" ]]; then
- META="--keep-attic"
-elif [[ $3 == "--meta=yes" || $3 == "--meta" ]]; then
- META="--meta"
-elif [[ $3 == "--meta=no" ]]; then
- META=
-else
-{
- echo "You must specify --meta=yes or --meta=no"
- exit 0
-}; fi
-
-PRODUCE_DIFF=
-
-get_replicate_filename()
-{
- printf -v TDIGIT3 %03u $(($1 % 1000))
- ARG=$(($1 / 1000))
- printf -v TDIGIT2 %03u $(($ARG % 1000))
- ARG=$(($ARG / 1000))
- printf -v TDIGIT1 %03u $ARG
- REPLICATE_TRUNK_DIR=$TDIGIT1/$TDIGIT2/
- REPLICATE_FILENAME=$TDIGIT1/$TDIGIT2/$TDIGIT3
-};
-
-
-collect_minute_diffs()
-{
- TEMP_DIR=$1
- TARGET=$(($START + 1))
-
- get_replicate_filename $TARGET
-
- while [[ ( -s $REPLICATE_DIR/$REPLICATE_FILENAME.state.txt ) && ( $(($START + 1440)) -ge $(($TARGET)) ) && ( `du -m $TEMP_DIR | awk '{ print $1; }'` -le 512 ) ]];
- do
- {
- printf -v TARGET_FILE %09u $TARGET
- gunzip <$REPLICATE_DIR/$REPLICATE_FILENAME.osc.gz >$TEMP_DIR/$TARGET_FILE.osc
- TARGET=$(($TARGET + 1))
- get_replicate_filename $TARGET
- };
- done
- TARGET=$(($TARGET - 1))
-};
-
-
-apply_minute_diffs()
-{
- ./update_from_dir --osc-dir=$1 --version=$DATA_VERSION $META --flush-size=0
- EXITCODE=$?
- while [[ $EXITCODE -ne 0 ]];
- do
- {
- sleep 60
- ./update_from_dir --osc-dir=$1 --version=$DATA_VERSION $META --flush-size=0
- EXITCODE=$?
- };
- done
- DIFF_COUNT=$(($DIFF_COUNT + 1))
-};
-
-
-update_state()
-{
- get_replicate_filename $TARGET
- TIMESTAMP_LINE=`grep "^timestamp" <$REPLICATE_DIR/$REPLICATE_FILENAME.state.txt`
- while [[ -z $TIMESTAMP_LINE ]]; do
- {
- sleep 5
- TIMESTAMP_LINE=`grep "^timestamp" <$REPLICATE_DIR/$REPLICATE_FILENAME.state.txt`
- }; done
- DATA_VERSION=${TIMESTAMP_LINE:10}
-};
-
-
-echo >>$DB_DIR/apply_osc_to_db.log
-
-mkdir -p $DB_DIR/augmented_diffs/
-DIFF_COUNT=0
-
-# update_state
-
-pushd "$EXEC_DIR"
-
-while [[ true ]]; do
-{
- if [[ $START == "auto" ]]; then
- {
- START=`cat $DB_DIR/replicate_id`
- }; fi
-
- echo "`date -u '+%F %T'`: updating from $START" >>$DB_DIR/apply_osc_to_db.log
-
- TEMP_DIR=`mktemp -d /tmp/osm-3s_update_XXXXXX`
- collect_minute_diffs $TEMP_DIR
-
- if [[ $TARGET -gt $START ]]; then
- {
- echo "`date -u '+%F %T'`: updating to $TARGET" >>$DB_DIR/apply_osc_to_db.log
-
- update_state
- apply_minute_diffs $TEMP_DIR
- echo "$TARGET" >$DB_DIR/replicate_id
-
- echo "`date -u '+%F %T'`: update complete" $TARGET >>$DB_DIR/apply_osc_to_db.log
- };
- else
- {
- sleep 5
- }; fi
-
- rm -f $TEMP_DIR/*
- rmdir $TEMP_DIR
-
- START=$TARGET
-}; done
diff --git a/0.7.53/bin/fetch_osc.sh b/0.7.53/bin/fetch_osc.sh
deleted file mode 100755
index a9d163f..0000000
--- a/0.7.53/bin/fetch_osc.sh
+++ /dev/null
@@ -1,163 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht
-#
-# This file is part of Overpass_API.
-#
-# Overpass_API is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# Overpass_API is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with Overpass_API. If not, see .
-
-if [[ -z $1 ]]; then
-{
- echo Usage: $0 Replicate_id Source_dir Local_dir [Sleep]
- exit 0
-};
-fi
-
-REPLICATE_ID=$1
-SOURCE_DIR=$2
-LOCAL_DIR=$3
-SLEEP_BETWEEN_DLS=15 # How long to sleep between download attempts (sec). Default: 15. See also FAIL_COUNTER_ALERT
-FAIL_COUNTER_ALERT=20 # After how many sleep cycles do we get nervous
-FAIL_COUNTER=0
-FILE_PANIC=
-
-if [[ ! -d $LOCAL_DIR ]];
- then {
- mkdir $LOCAL_DIR
-};
-fi
-
-if [[ $REPLICATE_ID == "auto" ]] ; then
- REPLICATE_ID=`find $LOCAL_DIR -type f -name '*state.txt' -not -size 0 | sort | tail -n 1 | sed -e "s#^$LOCAL_DIR##" -e 's#[./]##g' -e 's#statetxt##' -e 's/^0\+//g'`
-
- EXEC_DIR="`dirname $0`/"
- DB_DIR=`$EXEC_DIR/dispatcher --show-dir`
-
- if [[ ! -d $DB_DIR ]] ; then
- echo "Can't find DB_DIR. Returned value: $DB_DIR"
- exit 1;
- fi
-
- if [[ "x$REPLICATE_ID" == "x" ]] ; then
- REPLICATE_ID=`cat $DB_DIR/replicate_id`
- fi
- if [[ "x$REPLICATE_ID" == "x" ]] ; then
- echo "Could not determine REPLICATE_ID. Exiting"
- exit 1
- fi
-fi
-
-# $1 - remote source
-# $2 - local destination
-fetch_file()
-{
- wget -nv -O "$2" "$1"
-};
-
-retry_fetch_file()
-{
- FILE_PANIC=
- FAIL_COUNTER=0
- if [[ ! -s "$2" ]]; then {
- fetch_file "$1" "$2"
- if [[ "$3" == "gzip" ]]; then {
- gunzip -t <"$2"
- if [[ $? -ne 0 ]]; then {
- rm "$2"
- }; fi
- }; fi
- }; fi
- until [[ -s "$2" || $FAIL_COUNTER -ge $FAIL_COUNTER_ALERT ]]; do {
- FAIL_COUNTER=$(($FAIL_COUNTER+1))
- sleep $SLEEP_BETWEEN_DLS
- fetch_file "$1" "$2"
- if [[ "$3" == "gzip" ]]; then {
- gunzip -t <"$2"
- if [[ $? -ne 0 ]]; then {
- rm "$2"
- }; fi
- }; fi
- }; done
- if [[ $FAIL_COUNTER -ge $FAIL_COUNTER_ALERT ]]; then
- FILE_PANIC="true"
- fi
-};
-
-file_panic()
-{
- echo "fetch_osc()@"`date -u "+%F %T"`": upstream_delay $REPLICATE_ID" >>$LOCAL_DIR/fetch_osc.log
- REPLICATE_ID=$(($REPLICATE_ID - 1))
-
- printf -v TDIGIT3 %03u $(($REPLICATE_ID % 1000))
- ARG=$(($REPLICATE_ID / 1000))
- printf -v TDIGIT2 %03u $(($ARG % 1000))
- ARG=$(($ARG / 1000))
- printf -v TDIGIT1 %03u $ARG
-
- FILE_PANIC="true"
- until [[ ! -n $FILE_PANIC ]]; do {
- retry_fetch_file "$REMOTE_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.new.osc.gz" "gzip"
- }; done
- FILE_PANIC="true"
- until [[ ! -n $FILE_PANIC ]]; do {
- retry_fetch_file "$REMOTE_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.new.state.txt" "text"
- }; done
-
- RES_GZIP=`diff -q "$LOCAL_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.new.osc.gz"`
- RES_TEXT=`diff -q "$LOCAL_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.new.state.txt"`
- if [[ -n $RES_GZIP || -n $RES_TEXT ]]; then
- echo "fetch_osc()@"`date -u "+%F %T"`": file_panic $REPLICATE_ID" >>$LOCAL_DIR/fetch_osc.log
- echo "fetch_osc()@"`date -u "+%F %T"`": $RES_GZIP" >>$LOCAL_DIR/fetch_osc.log
- echo "fetch_osc()@"`date -u "+%F %T"`": $RES_TEXT" >>$LOCAL_DIR/fetch_osc.log
- exit 1
- fi
-
- rm "$LOCAL_PATH/$TDIGIT3.new.osc.gz"
- rm "$LOCAL_PATH/$TDIGIT3.new.state.txt"
-};
-
-fetch_minute_diff()
-{
- printf -v TDIGIT3 %03u $(($REPLICATE_ID % 1000))
- ARG=$(($REPLICATE_ID / 1000))
- printf -v TDIGIT2 %03u $(($ARG % 1000))
- ARG=$(($ARG / 1000))
- printf -v TDIGIT1 %03u $ARG
-
- LOCAL_PATH="$LOCAL_DIR/$TDIGIT1/$TDIGIT2"
- REMOTE_PATH="$SOURCE_DIR/$TDIGIT1/$TDIGIT2"
- mkdir -p "$LOCAL_DIR/$TDIGIT1/$TDIGIT2"
-
- retry_fetch_file "$REMOTE_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.osc.gz" "gzip"
- if [[ -n $FILE_PANIC ]]; then
- file_panic
- fi
- retry_fetch_file "$REMOTE_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.state.txt" "text"
- if [[ -n $FILE_PANIC ]]; then
- file_panic
- fi
-
- TIMESTAMP_LINE=`grep timestamp $LOCAL_DIR/$TDIGIT1/$TDIGIT2/$TDIGIT3.state.txt`
- TIMESTAMP=${TIMESTAMP_LINE:10}
-};
-
-while [[ true ]];
-do
-{
- REPLICATE_ID=$(($REPLICATE_ID + 1))
- fetch_minute_diff
- echo "fetch_osc()@"`date -u "+%F %T"`": new_replicate_diff $REPLICATE_ID $TIMESTAMP" >>$LOCAL_DIR/fetch_osc.log
- sleep 1
-};
-done
diff --git a/0.7.53/bin/update_overpass.sh b/0.7.53/bin/update_overpass.sh
new file mode 100644
index 0000000..328e375
--- /dev/null
+++ b/0.7.53/bin/update_overpass.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env sh
+
+(
+ set -e
+
+ if [ "x$OVERPASS_META" = "xattic" ] ; then
+ META="--keep-attic"
+ elif [ "x$OVERPASS_META" = "xyes" ] ; then
+ META="--meta"
+ else
+ META=""
+ fi
+
+ if [ ! -d /db/diffs ] ; then
+ mkdir /db/diffs
+ fi
+
+ while `true` ; do
+ (
+ if [ ! -e /db/diffs/changes.osm ] ; then
+ ! /app/venv/bin/pyosmium-get-changes --server $OVERPASS_DIFF_URL -o /db/diffs/changes.osm -f /db/replicate_id
+ OSMIUM_STATUS=$?
+ if [ $OSMIUM_STATUS -eq 1 ]; then
+ echo "There are still some updates remainging"
+ fi
+ if [ $OSMIUM_STATUS -eq 2 ]; then
+ echo "Failure downloading updates"
+ exit 0
+ fi
+ else
+ echo "/db/diffs/changes.osm exists. Trying to apply again."
+ fi
+ /app/bin/update_from_dir --osc-dir=/db/diffs/ $META
+ rm /db/diffs/changes.osm
+ )
+ sleep 60
+ done
+) 2>&1 | tee -a /db/changes.log
\ No newline at end of file
diff --git a/0.7.53/docker-entrypoint.sh b/0.7.53/docker-entrypoint.sh
index 5a43603..cb69627 100644
--- a/0.7.53/docker-entrypoint.sh
+++ b/0.7.53/docker-entrypoint.sh
@@ -6,13 +6,15 @@ OVERPASS_META=${OVERPASS_META:-no}
OVERPASS_MODE=${OVERPASS_MODE:-clone}
OVERPASS_COMPRESSION=${OVERPASS_COMPRESSION:-gz}
+if [[ "$OVERPASS_META" == "attic" ]] ; then
+ META="--keep-attic"
+else
+ META="--meta"
+fi
+
+
if [ ! -d /db/db ] ; then
if [ "$OVERPASS_MODE" = "clone" ]; then
- if [[ "$OVERPASS_META" == "attic" ]] ; then
- META="--keep-attic"
- else
- META="--meta=$OVERPASS_META"
- fi
mkdir -p /db/db \
&& /app/bin/download_clone.sh --db-dir=/db/db --source=http://dev.overpass-api.de/api_drolbr/ $META \
&& cp -r /app/etc/rules /db/db \
@@ -22,10 +24,21 @@ if [ ! -d /db/db ] ; then
fi
if [ "$OVERPASS_MODE" = "init" ]; then
- lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet; exit" \
- && /app/bin/init_osm3s.sh /db/planet /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION" \
- && echo $OVERPASS_PLANET_SEQUENCE_ID > /db/db/replicate_id \
- && rm /db/planet \
+ lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet.osm.bz2; exit" \
+ && /app/bin/init_osm3s.sh /db/planet.osm.bz2 /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION --map-compression-method=$OVERPASS_COMPRESSION"\
+ && echo "Database created. Now updating it." && (
+ ! /app/venv/bin/pyosmium-get-changes -O /db/planet.osm.bz2 --server $OVERPASS_DIFF_URL -o /db/changes.osm -f /db/replicate_id
+ OSMIUM_STATUS=$?
+ if [ $OSMIUM_STATUS -eq 1 ]; then
+ echo "There are still some updates remainging"
+ fi
+ if [ $OSMIUM_STATUS -eq 2 ]; then
+ echo "Failure downloading updates"
+ exit 0
+ fi
+ (cat /db/changes.osm | /app/bin/update_database --db-dir=/db/db $META --compression-method=$OVERPASS_COMPRESSION) 2>&1 | tee -a /db/changes.log
+ ) \
+ && rm /db/planet.osm.bz2 /db/changes.osm \
&& cp -r /app/etc/rules /db/db \
&& chown -R overpass:overpass /db \
&& echo "Overpass ready, you can start your container with docker start"
diff --git a/0.7.53/etc/nginx-overpass.conf b/0.7.53/etc/nginx-overpass.conf
index c019615..a19dde4 100644
--- a/0.7.53/etc/nginx-overpass.conf
+++ b/0.7.53/etc/nginx-overpass.conf
@@ -25,7 +25,7 @@ http {
sendfile on;
#tcp_nopush on;
- keepalive_timeout 65;
+ keepalive_timeout 1000s;
#gzip on;
@@ -55,6 +55,7 @@ http {
fastcgi_param SCRIPT_FILENAME /app/$fastcgi_script_name;
fastcgi_param PATH_INFO $fastcgi_path_info;
fastcgi_pass unix:/nginx/fcgiwrap.socket;
+ fastcgi_read_timeout 1000s;
}
location /api/ {
diff --git a/0.7.53/etc/supervisord.conf b/0.7.53/etc/supervisord.conf
index 6c4623b..fc67b74 100644
--- a/0.7.53/etc/supervisord.conf
+++ b/0.7.53/etc/supervisord.conf
@@ -17,18 +17,12 @@ redirect_stderr=true
user=nginx
priority=3
-[program:fetch_diff]
-command=/app/bin/fetch_osc.sh auto %(ENV_OVERPASS_DIFF_URL)s /db/diffs
+[program:update_overpass]
+command=/app/bin/update_overpass.sh
user=overpass
redirect_stderr=true
priority=5
-[program:apply_diff]
-command=/app/bin/apply_osc_to_db.sh /db/diffs/ auto --meta=%(ENV_OVERPASS_META)s
-redirect_stderr=true
-user=overpass
-priority=4
-
[program:dispatcher_areas]
command=nice /app/bin/dispatcher --areas --db-dir="/db/db"
user=overpass
diff --git a/0.7.53/requirements.txt b/0.7.53/requirements.txt
new file mode 100644
index 0000000..8314012
--- /dev/null
+++ b/0.7.53/requirements.txt
@@ -0,0 +1 @@
+osmium
diff --git a/0.7.54.10/Dockerfile b/0.7.54.10/Dockerfile
index 5cd623f..ae1e6aa 100644
--- a/0.7.54.10/Dockerfile
+++ b/0.7.54.10/Dockerfile
@@ -1,27 +1,34 @@
-FROM nginx:stable
+FROM nginx:1.14
RUN addgroup overpass && adduser --home /db --disabled-password --gecos overpass --ingroup overpass overpass
+ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.54.10.tar.gz /app/src.tar.gz
+
RUN apt-get update \
&& apt-get install --no-install-recommends --no-install-suggests -y \
autoconf \
automake \
+ bash \
+ bzip2 \
ca-certificates \
expat \
- libexpat1-dev \
+ fcgiwrap \
g++ \
+ lftp \
+ libexpat1-dev \
liblz4-1 \
liblz4-dev \
libtool \
m4 \
make \
+ osmium-tool \
+ python3 \
+ python3-venv \
+ supervisor \
+ wget \
zlib1g \
- zlib1g-dev
-
-
-ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.54.10.tar.gz /app/src.tar.gz
-
-RUN mkdir -p /app/src \
+ zlib1g-dev \
+ && mkdir -p /app/src \
&& cd /app/src \
&& tar -x -z --strip-components 1 -f ../src.tar.gz \
&& autoscan \
@@ -45,26 +52,28 @@ RUN mkdir -p /app/src \
&& apt-get autoremove -y \
&& mkdir -p /db/diffs /app/etc \
&& cp -r /app/src/rules /app/etc/rules \
- && rm -rf /app/src /app/src.tar.gz
+ && rm -rf /app/src /app/src.tar.gz \
+ && rm -rf /var/lib/apt/lists/*
-RUN apt-get install --no-install-recommends --no-install-suggests -y \
- supervisor \
- bash \
- lftp \
- wget \
- fcgiwrap \
- bzip2
+COPY requirements.txt /app/
+RUN python3 -m venv /app/venv \
+ && /app/venv/bin/pip install -r /app/requirements.txt
RUN mkdir /nginx && chown nginx:nginx /nginx && chown -R overpass:overpass /db
+
COPY etc/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+
COPY etc/nginx-overpass.conf /etc/nginx/nginx.conf
-COPY bin/apply_osc_to_db.sh bin/fetch_osc.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/
+
+COPY bin/update_overpass.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/
+
COPY docker-entrypoint.sh /app/
-RUN chmod a+x /app/docker-entrypoint.sh /app/bin/apply_osc_to_db.sh /app/bin/fetch_osc.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh
-VOLUME /db
+
+RUN chmod a+x /app/docker-entrypoint.sh /app/bin/update_overpass.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh
ENV OVERPASS_RULES_LOAD 1
+
EXPOSE 80
# CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"]
CMD ["/app/docker-entrypoint.sh"]
diff --git a/0.7.54.10/bin/apply_osc_to_db.sh b/0.7.54.10/bin/apply_osc_to_db.sh
deleted file mode 100755
index 94d1879..0000000
--- a/0.7.54.10/bin/apply_osc_to_db.sh
+++ /dev/null
@@ -1,163 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht
-#
-# This file is part of Overpass_API.
-#
-# Overpass_API is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# Overpass_API is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with Overpass_API. If not, see .
-
-if [[ -z $3 ]]; then
-{
- echo "Usage: $0 replicate_dir start_id --meta=(attic|yes|no)"
- exit 0
-}; fi
-
-EXEC_DIR="`dirname $0`/"
-if [[ ! ${EXEC_DIR:0:1} == "/" ]]; then
-{
- EXEC_DIR="`pwd`/$EXEC_DIR"
-}; fi
-
-DB_DIR=`$EXEC_DIR/dispatcher --show-dir`
-
-if [[ ! -d $DB_DIR ]] ; then
- echo "Can't find DB_DIR. Returned value: $DB_DIR"
- exit 1;
-fi
-
-REPLICATE_DIR="$1"
-if [[ ! ${REPLICATE_DIR:0:1} == "/" ]]; then
-{
- REPLICATE_DIR="`pwd`/$REPLICATE_DIR"
-}; fi
-
-START=$2
-META=
-
-if [[ $3 == "--meta=attic" ]]; then
- META="--keep-attic"
-elif [[ $3 == "--meta=yes" || $3 == "--meta" ]]; then
- META="--meta"
-elif [[ $3 == "--meta=no" ]]; then
- META=
-else
-{
- echo "You must specify --meta=yes or --meta=no"
- exit 0
-}; fi
-
-PRODUCE_DIFF=
-
-get_replicate_filename()
-{
- printf -v TDIGIT3 %03u $(($1 % 1000))
- ARG=$(($1 / 1000))
- printf -v TDIGIT2 %03u $(($ARG % 1000))
- ARG=$(($ARG / 1000))
- printf -v TDIGIT1 %03u $ARG
- REPLICATE_TRUNK_DIR=$TDIGIT1/$TDIGIT2/
- REPLICATE_FILENAME=$TDIGIT1/$TDIGIT2/$TDIGIT3
-};
-
-
-collect_minute_diffs()
-{
- TEMP_DIR=$1
- TARGET=$(($START + 1))
-
- get_replicate_filename $TARGET
-
- while [[ ( -s $REPLICATE_DIR/$REPLICATE_FILENAME.state.txt ) && ( $(($START + 1440)) -ge $(($TARGET)) ) && ( `du -m $TEMP_DIR | awk '{ print $1; }'` -le 512 ) ]];
- do
- {
- printf -v TARGET_FILE %09u $TARGET
- gunzip <$REPLICATE_DIR/$REPLICATE_FILENAME.osc.gz >$TEMP_DIR/$TARGET_FILE.osc
- TARGET=$(($TARGET + 1))
- get_replicate_filename $TARGET
- };
- done
- TARGET=$(($TARGET - 1))
-};
-
-
-apply_minute_diffs()
-{
- ./update_from_dir --osc-dir=$1 --version=$DATA_VERSION $META --flush-size=0
- EXITCODE=$?
- while [[ $EXITCODE -ne 0 ]];
- do
- {
- sleep 60
- ./update_from_dir --osc-dir=$1 --version=$DATA_VERSION $META --flush-size=0
- EXITCODE=$?
- };
- done
- DIFF_COUNT=$(($DIFF_COUNT + 1))
-};
-
-
-update_state()
-{
- get_replicate_filename $TARGET
- TIMESTAMP_LINE=`grep "^timestamp" <$REPLICATE_DIR/$REPLICATE_FILENAME.state.txt`
- while [[ -z $TIMESTAMP_LINE ]]; do
- {
- sleep 5
- TIMESTAMP_LINE=`grep "^timestamp" <$REPLICATE_DIR/$REPLICATE_FILENAME.state.txt`
- }; done
- DATA_VERSION=${TIMESTAMP_LINE:10}
-};
-
-
-echo >>$DB_DIR/apply_osc_to_db.log
-
-mkdir -p $DB_DIR/augmented_diffs/
-DIFF_COUNT=0
-
-# update_state
-
-pushd "$EXEC_DIR"
-
-while [[ true ]]; do
-{
- if [[ $START == "auto" ]]; then
- {
- START=`cat $DB_DIR/replicate_id`
- }; fi
-
- echo "`date -u '+%F %T'`: updating from $START" >>$DB_DIR/apply_osc_to_db.log
-
- TEMP_DIR=`mktemp -d /tmp/osm-3s_update_XXXXXX`
- collect_minute_diffs $TEMP_DIR
-
- if [[ $TARGET -gt $START ]]; then
- {
- echo "`date -u '+%F %T'`: updating to $TARGET" >>$DB_DIR/apply_osc_to_db.log
-
- update_state
- apply_minute_diffs $TEMP_DIR
- echo "$TARGET" >$DB_DIR/replicate_id
-
- echo "`date -u '+%F %T'`: update complete" $TARGET >>$DB_DIR/apply_osc_to_db.log
- };
- else
- {
- sleep 5
- }; fi
-
- rm -f $TEMP_DIR/*
- rmdir $TEMP_DIR
-
- START=$TARGET
-}; done
diff --git a/0.7.54.10/bin/fetch_osc.sh b/0.7.54.10/bin/fetch_osc.sh
deleted file mode 100755
index a9d163f..0000000
--- a/0.7.54.10/bin/fetch_osc.sh
+++ /dev/null
@@ -1,163 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht
-#
-# This file is part of Overpass_API.
-#
-# Overpass_API is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# Overpass_API is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with Overpass_API. If not, see .
-
-if [[ -z $1 ]]; then
-{
- echo Usage: $0 Replicate_id Source_dir Local_dir [Sleep]
- exit 0
-};
-fi
-
-REPLICATE_ID=$1
-SOURCE_DIR=$2
-LOCAL_DIR=$3
-SLEEP_BETWEEN_DLS=15 # How long to sleep between download attempts (sec). Default: 15. See also FAIL_COUNTER_ALERT
-FAIL_COUNTER_ALERT=20 # After how many sleep cycles do we get nervous
-FAIL_COUNTER=0
-FILE_PANIC=
-
-if [[ ! -d $LOCAL_DIR ]];
- then {
- mkdir $LOCAL_DIR
-};
-fi
-
-if [[ $REPLICATE_ID == "auto" ]] ; then
- REPLICATE_ID=`find $LOCAL_DIR -type f -name '*state.txt' -not -size 0 | sort | tail -n 1 | sed -e "s#^$LOCAL_DIR##" -e 's#[./]##g' -e 's#statetxt##' -e 's/^0\+//g'`
-
- EXEC_DIR="`dirname $0`/"
- DB_DIR=`$EXEC_DIR/dispatcher --show-dir`
-
- if [[ ! -d $DB_DIR ]] ; then
- echo "Can't find DB_DIR. Returned value: $DB_DIR"
- exit 1;
- fi
-
- if [[ "x$REPLICATE_ID" == "x" ]] ; then
- REPLICATE_ID=`cat $DB_DIR/replicate_id`
- fi
- if [[ "x$REPLICATE_ID" == "x" ]] ; then
- echo "Could not determine REPLICATE_ID. Exiting"
- exit 1
- fi
-fi
-
-# $1 - remote source
-# $2 - local destination
-fetch_file()
-{
- wget -nv -O "$2" "$1"
-};
-
-retry_fetch_file()
-{
- FILE_PANIC=
- FAIL_COUNTER=0
- if [[ ! -s "$2" ]]; then {
- fetch_file "$1" "$2"
- if [[ "$3" == "gzip" ]]; then {
- gunzip -t <"$2"
- if [[ $? -ne 0 ]]; then {
- rm "$2"
- }; fi
- }; fi
- }; fi
- until [[ -s "$2" || $FAIL_COUNTER -ge $FAIL_COUNTER_ALERT ]]; do {
- FAIL_COUNTER=$(($FAIL_COUNTER+1))
- sleep $SLEEP_BETWEEN_DLS
- fetch_file "$1" "$2"
- if [[ "$3" == "gzip" ]]; then {
- gunzip -t <"$2"
- if [[ $? -ne 0 ]]; then {
- rm "$2"
- }; fi
- }; fi
- }; done
- if [[ $FAIL_COUNTER -ge $FAIL_COUNTER_ALERT ]]; then
- FILE_PANIC="true"
- fi
-};
-
-file_panic()
-{
- echo "fetch_osc()@"`date -u "+%F %T"`": upstream_delay $REPLICATE_ID" >>$LOCAL_DIR/fetch_osc.log
- REPLICATE_ID=$(($REPLICATE_ID - 1))
-
- printf -v TDIGIT3 %03u $(($REPLICATE_ID % 1000))
- ARG=$(($REPLICATE_ID / 1000))
- printf -v TDIGIT2 %03u $(($ARG % 1000))
- ARG=$(($ARG / 1000))
- printf -v TDIGIT1 %03u $ARG
-
- FILE_PANIC="true"
- until [[ ! -n $FILE_PANIC ]]; do {
- retry_fetch_file "$REMOTE_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.new.osc.gz" "gzip"
- }; done
- FILE_PANIC="true"
- until [[ ! -n $FILE_PANIC ]]; do {
- retry_fetch_file "$REMOTE_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.new.state.txt" "text"
- }; done
-
- RES_GZIP=`diff -q "$LOCAL_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.new.osc.gz"`
- RES_TEXT=`diff -q "$LOCAL_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.new.state.txt"`
- if [[ -n $RES_GZIP || -n $RES_TEXT ]]; then
- echo "fetch_osc()@"`date -u "+%F %T"`": file_panic $REPLICATE_ID" >>$LOCAL_DIR/fetch_osc.log
- echo "fetch_osc()@"`date -u "+%F %T"`": $RES_GZIP" >>$LOCAL_DIR/fetch_osc.log
- echo "fetch_osc()@"`date -u "+%F %T"`": $RES_TEXT" >>$LOCAL_DIR/fetch_osc.log
- exit 1
- fi
-
- rm "$LOCAL_PATH/$TDIGIT3.new.osc.gz"
- rm "$LOCAL_PATH/$TDIGIT3.new.state.txt"
-};
-
-fetch_minute_diff()
-{
- printf -v TDIGIT3 %03u $(($REPLICATE_ID % 1000))
- ARG=$(($REPLICATE_ID / 1000))
- printf -v TDIGIT2 %03u $(($ARG % 1000))
- ARG=$(($ARG / 1000))
- printf -v TDIGIT1 %03u $ARG
-
- LOCAL_PATH="$LOCAL_DIR/$TDIGIT1/$TDIGIT2"
- REMOTE_PATH="$SOURCE_DIR/$TDIGIT1/$TDIGIT2"
- mkdir -p "$LOCAL_DIR/$TDIGIT1/$TDIGIT2"
-
- retry_fetch_file "$REMOTE_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.osc.gz" "gzip"
- if [[ -n $FILE_PANIC ]]; then
- file_panic
- fi
- retry_fetch_file "$REMOTE_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.state.txt" "text"
- if [[ -n $FILE_PANIC ]]; then
- file_panic
- fi
-
- TIMESTAMP_LINE=`grep timestamp $LOCAL_DIR/$TDIGIT1/$TDIGIT2/$TDIGIT3.state.txt`
- TIMESTAMP=${TIMESTAMP_LINE:10}
-};
-
-while [[ true ]];
-do
-{
- REPLICATE_ID=$(($REPLICATE_ID + 1))
- fetch_minute_diff
- echo "fetch_osc()@"`date -u "+%F %T"`": new_replicate_diff $REPLICATE_ID $TIMESTAMP" >>$LOCAL_DIR/fetch_osc.log
- sleep 1
-};
-done
diff --git a/0.7.54.10/bin/update_overpass.sh b/0.7.54.10/bin/update_overpass.sh
new file mode 100644
index 0000000..328e375
--- /dev/null
+++ b/0.7.54.10/bin/update_overpass.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env sh
+
+(
+ set -e
+
+ if [ "x$OVERPASS_META" = "xattic" ] ; then
+ META="--keep-attic"
+ elif [ "x$OVERPASS_META" = "xyes" ] ; then
+ META="--meta"
+ else
+ META=""
+ fi
+
+ if [ ! -d /db/diffs ] ; then
+ mkdir /db/diffs
+ fi
+
+ while `true` ; do
+ (
+ if [ ! -e /db/diffs/changes.osm ] ; then
+ ! /app/venv/bin/pyosmium-get-changes --server $OVERPASS_DIFF_URL -o /db/diffs/changes.osm -f /db/replicate_id
+ OSMIUM_STATUS=$?
+ if [ $OSMIUM_STATUS -eq 1 ]; then
+ echo "There are still some updates remainging"
+ fi
+ if [ $OSMIUM_STATUS -eq 2 ]; then
+ echo "Failure downloading updates"
+ exit 0
+ fi
+ else
+ echo "/db/diffs/changes.osm exists. Trying to apply again."
+ fi
+ /app/bin/update_from_dir --osc-dir=/db/diffs/ $META
+ rm /db/diffs/changes.osm
+ )
+ sleep 60
+ done
+) 2>&1 | tee -a /db/changes.log
\ No newline at end of file
diff --git a/0.7.54.10/docker-entrypoint.sh b/0.7.54.10/docker-entrypoint.sh
index 5a43603..cb69627 100644
--- a/0.7.54.10/docker-entrypoint.sh
+++ b/0.7.54.10/docker-entrypoint.sh
@@ -6,13 +6,15 @@ OVERPASS_META=${OVERPASS_META:-no}
OVERPASS_MODE=${OVERPASS_MODE:-clone}
OVERPASS_COMPRESSION=${OVERPASS_COMPRESSION:-gz}
+if [[ "$OVERPASS_META" == "attic" ]] ; then
+ META="--keep-attic"
+else
+ META="--meta"
+fi
+
+
if [ ! -d /db/db ] ; then
if [ "$OVERPASS_MODE" = "clone" ]; then
- if [[ "$OVERPASS_META" == "attic" ]] ; then
- META="--keep-attic"
- else
- META="--meta=$OVERPASS_META"
- fi
mkdir -p /db/db \
&& /app/bin/download_clone.sh --db-dir=/db/db --source=http://dev.overpass-api.de/api_drolbr/ $META \
&& cp -r /app/etc/rules /db/db \
@@ -22,10 +24,21 @@ if [ ! -d /db/db ] ; then
fi
if [ "$OVERPASS_MODE" = "init" ]; then
- lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet; exit" \
- && /app/bin/init_osm3s.sh /db/planet /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION" \
- && echo $OVERPASS_PLANET_SEQUENCE_ID > /db/db/replicate_id \
- && rm /db/planet \
+ lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet.osm.bz2; exit" \
+ && /app/bin/init_osm3s.sh /db/planet.osm.bz2 /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION --map-compression-method=$OVERPASS_COMPRESSION"\
+ && echo "Database created. Now updating it." && (
+ ! /app/venv/bin/pyosmium-get-changes -O /db/planet.osm.bz2 --server $OVERPASS_DIFF_URL -o /db/changes.osm -f /db/replicate_id
+ OSMIUM_STATUS=$?
+ if [ $OSMIUM_STATUS -eq 1 ]; then
+ echo "There are still some updates remainging"
+ fi
+ if [ $OSMIUM_STATUS -eq 2 ]; then
+ echo "Failure downloading updates"
+ exit 0
+ fi
+ (cat /db/changes.osm | /app/bin/update_database --db-dir=/db/db $META --compression-method=$OVERPASS_COMPRESSION) 2>&1 | tee -a /db/changes.log
+ ) \
+ && rm /db/planet.osm.bz2 /db/changes.osm \
&& cp -r /app/etc/rules /db/db \
&& chown -R overpass:overpass /db \
&& echo "Overpass ready, you can start your container with docker start"
diff --git a/0.7.54.10/etc/nginx-overpass.conf b/0.7.54.10/etc/nginx-overpass.conf
index c019615..a19dde4 100644
--- a/0.7.54.10/etc/nginx-overpass.conf
+++ b/0.7.54.10/etc/nginx-overpass.conf
@@ -25,7 +25,7 @@ http {
sendfile on;
#tcp_nopush on;
- keepalive_timeout 65;
+ keepalive_timeout 1000s;
#gzip on;
@@ -55,6 +55,7 @@ http {
fastcgi_param SCRIPT_FILENAME /app/$fastcgi_script_name;
fastcgi_param PATH_INFO $fastcgi_path_info;
fastcgi_pass unix:/nginx/fcgiwrap.socket;
+ fastcgi_read_timeout 1000s;
}
location /api/ {
diff --git a/0.7.54.10/etc/supervisord.conf b/0.7.54.10/etc/supervisord.conf
index 6c4623b..fc67b74 100644
--- a/0.7.54.10/etc/supervisord.conf
+++ b/0.7.54.10/etc/supervisord.conf
@@ -17,18 +17,12 @@ redirect_stderr=true
user=nginx
priority=3
-[program:fetch_diff]
-command=/app/bin/fetch_osc.sh auto %(ENV_OVERPASS_DIFF_URL)s /db/diffs
+[program:update_overpass]
+command=/app/bin/update_overpass.sh
user=overpass
redirect_stderr=true
priority=5
-[program:apply_diff]
-command=/app/bin/apply_osc_to_db.sh /db/diffs/ auto --meta=%(ENV_OVERPASS_META)s
-redirect_stderr=true
-user=overpass
-priority=4
-
[program:dispatcher_areas]
command=nice /app/bin/dispatcher --areas --db-dir="/db/db"
user=overpass
diff --git a/0.7.54.10/requirements.txt b/0.7.54.10/requirements.txt
new file mode 100644
index 0000000..8314012
--- /dev/null
+++ b/0.7.54.10/requirements.txt
@@ -0,0 +1 @@
+osmium
diff --git a/0.7.54.11/bin/apply_osc_to_db.sh b/0.7.54.11/bin/apply_osc_to_db.sh
deleted file mode 100755
index 94d1879..0000000
--- a/0.7.54.11/bin/apply_osc_to_db.sh
+++ /dev/null
@@ -1,163 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht
-#
-# This file is part of Overpass_API.
-#
-# Overpass_API is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# Overpass_API is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with Overpass_API. If not, see .
-
-if [[ -z $3 ]]; then
-{
- echo "Usage: $0 replicate_dir start_id --meta=(attic|yes|no)"
- exit 0
-}; fi
-
-EXEC_DIR="`dirname $0`/"
-if [[ ! ${EXEC_DIR:0:1} == "/" ]]; then
-{
- EXEC_DIR="`pwd`/$EXEC_DIR"
-}; fi
-
-DB_DIR=`$EXEC_DIR/dispatcher --show-dir`
-
-if [[ ! -d $DB_DIR ]] ; then
- echo "Can't find DB_DIR. Returned value: $DB_DIR"
- exit 1;
-fi
-
-REPLICATE_DIR="$1"
-if [[ ! ${REPLICATE_DIR:0:1} == "/" ]]; then
-{
- REPLICATE_DIR="`pwd`/$REPLICATE_DIR"
-}; fi
-
-START=$2
-META=
-
-if [[ $3 == "--meta=attic" ]]; then
- META="--keep-attic"
-elif [[ $3 == "--meta=yes" || $3 == "--meta" ]]; then
- META="--meta"
-elif [[ $3 == "--meta=no" ]]; then
- META=
-else
-{
- echo "You must specify --meta=yes or --meta=no"
- exit 0
-}; fi
-
-PRODUCE_DIFF=
-
-get_replicate_filename()
-{
- printf -v TDIGIT3 %03u $(($1 % 1000))
- ARG=$(($1 / 1000))
- printf -v TDIGIT2 %03u $(($ARG % 1000))
- ARG=$(($ARG / 1000))
- printf -v TDIGIT1 %03u $ARG
- REPLICATE_TRUNK_DIR=$TDIGIT1/$TDIGIT2/
- REPLICATE_FILENAME=$TDIGIT1/$TDIGIT2/$TDIGIT3
-};
-
-
-collect_minute_diffs()
-{
- TEMP_DIR=$1
- TARGET=$(($START + 1))
-
- get_replicate_filename $TARGET
-
- while [[ ( -s $REPLICATE_DIR/$REPLICATE_FILENAME.state.txt ) && ( $(($START + 1440)) -ge $(($TARGET)) ) && ( `du -m $TEMP_DIR | awk '{ print $1; }'` -le 512 ) ]];
- do
- {
- printf -v TARGET_FILE %09u $TARGET
- gunzip <$REPLICATE_DIR/$REPLICATE_FILENAME.osc.gz >$TEMP_DIR/$TARGET_FILE.osc
- TARGET=$(($TARGET + 1))
- get_replicate_filename $TARGET
- };
- done
- TARGET=$(($TARGET - 1))
-};
-
-
-apply_minute_diffs()
-{
- ./update_from_dir --osc-dir=$1 --version=$DATA_VERSION $META --flush-size=0
- EXITCODE=$?
- while [[ $EXITCODE -ne 0 ]];
- do
- {
- sleep 60
- ./update_from_dir --osc-dir=$1 --version=$DATA_VERSION $META --flush-size=0
- EXITCODE=$?
- };
- done
- DIFF_COUNT=$(($DIFF_COUNT + 1))
-};
-
-
-update_state()
-{
- get_replicate_filename $TARGET
- TIMESTAMP_LINE=`grep "^timestamp" <$REPLICATE_DIR/$REPLICATE_FILENAME.state.txt`
- while [[ -z $TIMESTAMP_LINE ]]; do
- {
- sleep 5
- TIMESTAMP_LINE=`grep "^timestamp" <$REPLICATE_DIR/$REPLICATE_FILENAME.state.txt`
- }; done
- DATA_VERSION=${TIMESTAMP_LINE:10}
-};
-
-
-echo >>$DB_DIR/apply_osc_to_db.log
-
-mkdir -p $DB_DIR/augmented_diffs/
-DIFF_COUNT=0
-
-# update_state
-
-pushd "$EXEC_DIR"
-
-while [[ true ]]; do
-{
- if [[ $START == "auto" ]]; then
- {
- START=`cat $DB_DIR/replicate_id`
- }; fi
-
- echo "`date -u '+%F %T'`: updating from $START" >>$DB_DIR/apply_osc_to_db.log
-
- TEMP_DIR=`mktemp -d /tmp/osm-3s_update_XXXXXX`
- collect_minute_diffs $TEMP_DIR
-
- if [[ $TARGET -gt $START ]]; then
- {
- echo "`date -u '+%F %T'`: updating to $TARGET" >>$DB_DIR/apply_osc_to_db.log
-
- update_state
- apply_minute_diffs $TEMP_DIR
- echo "$TARGET" >$DB_DIR/replicate_id
-
- echo "`date -u '+%F %T'`: update complete" $TARGET >>$DB_DIR/apply_osc_to_db.log
- };
- else
- {
- sleep 5
- }; fi
-
- rm -f $TEMP_DIR/*
- rmdir $TEMP_DIR
-
- START=$TARGET
-}; done
diff --git a/0.7.54.11/bin/fetch_osc.sh b/0.7.54.11/bin/fetch_osc.sh
deleted file mode 100755
index a9d163f..0000000
--- a/0.7.54.11/bin/fetch_osc.sh
+++ /dev/null
@@ -1,163 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht
-#
-# This file is part of Overpass_API.
-#
-# Overpass_API is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# Overpass_API is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with Overpass_API. If not, see .
-
-if [[ -z $1 ]]; then
-{
- echo Usage: $0 Replicate_id Source_dir Local_dir [Sleep]
- exit 0
-};
-fi
-
-REPLICATE_ID=$1
-SOURCE_DIR=$2
-LOCAL_DIR=$3
-SLEEP_BETWEEN_DLS=15 # How long to sleep between download attempts (sec). Default: 15. See also FAIL_COUNTER_ALERT
-FAIL_COUNTER_ALERT=20 # After how many sleep cycles do we get nervous
-FAIL_COUNTER=0
-FILE_PANIC=
-
-if [[ ! -d $LOCAL_DIR ]];
- then {
- mkdir $LOCAL_DIR
-};
-fi
-
-if [[ $REPLICATE_ID == "auto" ]] ; then
- REPLICATE_ID=`find $LOCAL_DIR -type f -name '*state.txt' -not -size 0 | sort | tail -n 1 | sed -e "s#^$LOCAL_DIR##" -e 's#[./]##g' -e 's#statetxt##' -e 's/^0\+//g'`
-
- EXEC_DIR="`dirname $0`/"
- DB_DIR=`$EXEC_DIR/dispatcher --show-dir`
-
- if [[ ! -d $DB_DIR ]] ; then
- echo "Can't find DB_DIR. Returned value: $DB_DIR"
- exit 1;
- fi
-
- if [[ "x$REPLICATE_ID" == "x" ]] ; then
- REPLICATE_ID=`cat $DB_DIR/replicate_id`
- fi
- if [[ "x$REPLICATE_ID" == "x" ]] ; then
- echo "Could not determine REPLICATE_ID. Exiting"
- exit 1
- fi
-fi
-
-# $1 - remote source
-# $2 - local destination
-fetch_file()
-{
- wget -nv -O "$2" "$1"
-};
-
-retry_fetch_file()
-{
- FILE_PANIC=
- FAIL_COUNTER=0
- if [[ ! -s "$2" ]]; then {
- fetch_file "$1" "$2"
- if [[ "$3" == "gzip" ]]; then {
- gunzip -t <"$2"
- if [[ $? -ne 0 ]]; then {
- rm "$2"
- }; fi
- }; fi
- }; fi
- until [[ -s "$2" || $FAIL_COUNTER -ge $FAIL_COUNTER_ALERT ]]; do {
- FAIL_COUNTER=$(($FAIL_COUNTER+1))
- sleep $SLEEP_BETWEEN_DLS
- fetch_file "$1" "$2"
- if [[ "$3" == "gzip" ]]; then {
- gunzip -t <"$2"
- if [[ $? -ne 0 ]]; then {
- rm "$2"
- }; fi
- }; fi
- }; done
- if [[ $FAIL_COUNTER -ge $FAIL_COUNTER_ALERT ]]; then
- FILE_PANIC="true"
- fi
-};
-
-file_panic()
-{
- echo "fetch_osc()@"`date -u "+%F %T"`": upstream_delay $REPLICATE_ID" >>$LOCAL_DIR/fetch_osc.log
- REPLICATE_ID=$(($REPLICATE_ID - 1))
-
- printf -v TDIGIT3 %03u $(($REPLICATE_ID % 1000))
- ARG=$(($REPLICATE_ID / 1000))
- printf -v TDIGIT2 %03u $(($ARG % 1000))
- ARG=$(($ARG / 1000))
- printf -v TDIGIT1 %03u $ARG
-
- FILE_PANIC="true"
- until [[ ! -n $FILE_PANIC ]]; do {
- retry_fetch_file "$REMOTE_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.new.osc.gz" "gzip"
- }; done
- FILE_PANIC="true"
- until [[ ! -n $FILE_PANIC ]]; do {
- retry_fetch_file "$REMOTE_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.new.state.txt" "text"
- }; done
-
- RES_GZIP=`diff -q "$LOCAL_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.new.osc.gz"`
- RES_TEXT=`diff -q "$LOCAL_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.new.state.txt"`
- if [[ -n $RES_GZIP || -n $RES_TEXT ]]; then
- echo "fetch_osc()@"`date -u "+%F %T"`": file_panic $REPLICATE_ID" >>$LOCAL_DIR/fetch_osc.log
- echo "fetch_osc()@"`date -u "+%F %T"`": $RES_GZIP" >>$LOCAL_DIR/fetch_osc.log
- echo "fetch_osc()@"`date -u "+%F %T"`": $RES_TEXT" >>$LOCAL_DIR/fetch_osc.log
- exit 1
- fi
-
- rm "$LOCAL_PATH/$TDIGIT3.new.osc.gz"
- rm "$LOCAL_PATH/$TDIGIT3.new.state.txt"
-};
-
-fetch_minute_diff()
-{
- printf -v TDIGIT3 %03u $(($REPLICATE_ID % 1000))
- ARG=$(($REPLICATE_ID / 1000))
- printf -v TDIGIT2 %03u $(($ARG % 1000))
- ARG=$(($ARG / 1000))
- printf -v TDIGIT1 %03u $ARG
-
- LOCAL_PATH="$LOCAL_DIR/$TDIGIT1/$TDIGIT2"
- REMOTE_PATH="$SOURCE_DIR/$TDIGIT1/$TDIGIT2"
- mkdir -p "$LOCAL_DIR/$TDIGIT1/$TDIGIT2"
-
- retry_fetch_file "$REMOTE_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.osc.gz" "gzip"
- if [[ -n $FILE_PANIC ]]; then
- file_panic
- fi
- retry_fetch_file "$REMOTE_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.state.txt" "text"
- if [[ -n $FILE_PANIC ]]; then
- file_panic
- fi
-
- TIMESTAMP_LINE=`grep timestamp $LOCAL_DIR/$TDIGIT1/$TDIGIT2/$TDIGIT3.state.txt`
- TIMESTAMP=${TIMESTAMP_LINE:10}
-};
-
-while [[ true ]];
-do
-{
- REPLICATE_ID=$(($REPLICATE_ID + 1))
- fetch_minute_diff
- echo "fetch_osc()@"`date -u "+%F %T"`": new_replicate_diff $REPLICATE_ID $TIMESTAMP" >>$LOCAL_DIR/fetch_osc.log
- sleep 1
-};
-done
diff --git a/0.7.54.11/docker-entrypoint.sh b/0.7.54.11/docker-entrypoint.sh
deleted file mode 100644
index 5a43603..0000000
--- a/0.7.54.11/docker-entrypoint.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/bin/bash
-
-set -eo pipefail
-shopt -s nullglob
-OVERPASS_META=${OVERPASS_META:-no}
-OVERPASS_MODE=${OVERPASS_MODE:-clone}
-OVERPASS_COMPRESSION=${OVERPASS_COMPRESSION:-gz}
-
-if [ ! -d /db/db ] ; then
- if [ "$OVERPASS_MODE" = "clone" ]; then
- if [[ "$OVERPASS_META" == "attic" ]] ; then
- META="--keep-attic"
- else
- META="--meta=$OVERPASS_META"
- fi
- mkdir -p /db/db \
- && /app/bin/download_clone.sh --db-dir=/db/db --source=http://dev.overpass-api.de/api_drolbr/ $META \
- && cp -r /app/etc/rules /db/db \
- && chown -R overpass:overpass /db \
- && echo "Overpass ready, you can start your container with docker start"
- exit
- fi
-
- if [ "$OVERPASS_MODE" = "init" ]; then
- lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet; exit" \
- && /app/bin/init_osm3s.sh /db/planet /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION" \
- && echo $OVERPASS_PLANET_SEQUENCE_ID > /db/db/replicate_id \
- && rm /db/planet \
- && cp -r /app/etc/rules /db/db \
- && chown -R overpass:overpass /db \
- && echo "Overpass ready, you can start your container with docker start"
- exit
- fi
-fi
-
-exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf
diff --git a/0.7.54.12/Dockerfile b/0.7.54.12/Dockerfile
index c5eccd1..f1543f2 100644
--- a/0.7.54.12/Dockerfile
+++ b/0.7.54.12/Dockerfile
@@ -1,27 +1,34 @@
-FROM nginx:stable
+FROM nginx:1.14
RUN addgroup overpass && adduser --home /db --disabled-password --gecos overpass --ingroup overpass overpass
+ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.54.12.tar.gz /app/src.tar.gz
+
RUN apt-get update \
&& apt-get install --no-install-recommends --no-install-suggests -y \
autoconf \
automake \
+ bash \
+ bzip2 \
ca-certificates \
expat \
- libexpat1-dev \
+ fcgiwrap \
g++ \
+ lftp \
+ libexpat1-dev \
liblz4-1 \
liblz4-dev \
libtool \
m4 \
make \
+ osmium-tool \
+ python3 \
+ python3-venv \
+ supervisor \
+ wget \
zlib1g \
- zlib1g-dev
-
-
-ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.54.12.tar.gz /app/src.tar.gz
-
-RUN mkdir -p /app/src \
+ zlib1g-dev \
+ && mkdir -p /app/src \
&& cd /app/src \
&& tar -x -z --strip-components 1 -f ../src.tar.gz \
&& autoscan \
@@ -45,26 +52,28 @@ RUN mkdir -p /app/src \
&& apt-get autoremove -y \
&& mkdir -p /db/diffs /app/etc \
&& cp -r /app/src/rules /app/etc/rules \
- && rm -rf /app/src /app/src.tar.gz
+ && rm -rf /app/src /app/src.tar.gz \
+ && rm -rf /var/lib/apt/lists/*
-RUN apt-get install --no-install-recommends --no-install-suggests -y \
- supervisor \
- bash \
- lftp \
- wget \
- fcgiwrap \
- bzip2
+COPY requirements.txt /app/
+RUN python3 -m venv /app/venv \
+ && /app/venv/bin/pip install -r /app/requirements.txt
RUN mkdir /nginx && chown nginx:nginx /nginx && chown -R overpass:overpass /db
+
COPY etc/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+
COPY etc/nginx-overpass.conf /etc/nginx/nginx.conf
-COPY bin/apply_osc_to_db.sh bin/fetch_osc.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/
+
+COPY bin/update_overpass.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/
+
COPY docker-entrypoint.sh /app/
-RUN chmod a+x /app/docker-entrypoint.sh /app/bin/apply_osc_to_db.sh /app/bin/fetch_osc.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh
-VOLUME /db
+
+RUN chmod a+x /app/docker-entrypoint.sh /app/bin/update_overpass.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh
ENV OVERPASS_RULES_LOAD 1
+
EXPOSE 80
# CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"]
CMD ["/app/docker-entrypoint.sh"]
diff --git a/0.7.54.12/bin/apply_osc_to_db.sh b/0.7.54.12/bin/apply_osc_to_db.sh
deleted file mode 100755
index 94d1879..0000000
--- a/0.7.54.12/bin/apply_osc_to_db.sh
+++ /dev/null
@@ -1,163 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht
-#
-# This file is part of Overpass_API.
-#
-# Overpass_API is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# Overpass_API is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with Overpass_API. If not, see .
-
-if [[ -z $3 ]]; then
-{
- echo "Usage: $0 replicate_dir start_id --meta=(attic|yes|no)"
- exit 0
-}; fi
-
-EXEC_DIR="`dirname $0`/"
-if [[ ! ${EXEC_DIR:0:1} == "/" ]]; then
-{
- EXEC_DIR="`pwd`/$EXEC_DIR"
-}; fi
-
-DB_DIR=`$EXEC_DIR/dispatcher --show-dir`
-
-if [[ ! -d $DB_DIR ]] ; then
- echo "Can't find DB_DIR. Returned value: $DB_DIR"
- exit 1;
-fi
-
-REPLICATE_DIR="$1"
-if [[ ! ${REPLICATE_DIR:0:1} == "/" ]]; then
-{
- REPLICATE_DIR="`pwd`/$REPLICATE_DIR"
-}; fi
-
-START=$2
-META=
-
-if [[ $3 == "--meta=attic" ]]; then
- META="--keep-attic"
-elif [[ $3 == "--meta=yes" || $3 == "--meta" ]]; then
- META="--meta"
-elif [[ $3 == "--meta=no" ]]; then
- META=
-else
-{
- echo "You must specify --meta=yes or --meta=no"
- exit 0
-}; fi
-
-PRODUCE_DIFF=
-
-get_replicate_filename()
-{
- printf -v TDIGIT3 %03u $(($1 % 1000))
- ARG=$(($1 / 1000))
- printf -v TDIGIT2 %03u $(($ARG % 1000))
- ARG=$(($ARG / 1000))
- printf -v TDIGIT1 %03u $ARG
- REPLICATE_TRUNK_DIR=$TDIGIT1/$TDIGIT2/
- REPLICATE_FILENAME=$TDIGIT1/$TDIGIT2/$TDIGIT3
-};
-
-
-collect_minute_diffs()
-{
- TEMP_DIR=$1
- TARGET=$(($START + 1))
-
- get_replicate_filename $TARGET
-
- while [[ ( -s $REPLICATE_DIR/$REPLICATE_FILENAME.state.txt ) && ( $(($START + 1440)) -ge $(($TARGET)) ) && ( `du -m $TEMP_DIR | awk '{ print $1; }'` -le 512 ) ]];
- do
- {
- printf -v TARGET_FILE %09u $TARGET
- gunzip <$REPLICATE_DIR/$REPLICATE_FILENAME.osc.gz >$TEMP_DIR/$TARGET_FILE.osc
- TARGET=$(($TARGET + 1))
- get_replicate_filename $TARGET
- };
- done
- TARGET=$(($TARGET - 1))
-};
-
-
-apply_minute_diffs()
-{
- ./update_from_dir --osc-dir=$1 --version=$DATA_VERSION $META --flush-size=0
- EXITCODE=$?
- while [[ $EXITCODE -ne 0 ]];
- do
- {
- sleep 60
- ./update_from_dir --osc-dir=$1 --version=$DATA_VERSION $META --flush-size=0
- EXITCODE=$?
- };
- done
- DIFF_COUNT=$(($DIFF_COUNT + 1))
-};
-
-
-update_state()
-{
- get_replicate_filename $TARGET
- TIMESTAMP_LINE=`grep "^timestamp" <$REPLICATE_DIR/$REPLICATE_FILENAME.state.txt`
- while [[ -z $TIMESTAMP_LINE ]]; do
- {
- sleep 5
- TIMESTAMP_LINE=`grep "^timestamp" <$REPLICATE_DIR/$REPLICATE_FILENAME.state.txt`
- }; done
- DATA_VERSION=${TIMESTAMP_LINE:10}
-};
-
-
-echo >>$DB_DIR/apply_osc_to_db.log
-
-mkdir -p $DB_DIR/augmented_diffs/
-DIFF_COUNT=0
-
-# update_state
-
-pushd "$EXEC_DIR"
-
-while [[ true ]]; do
-{
- if [[ $START == "auto" ]]; then
- {
- START=`cat $DB_DIR/replicate_id`
- }; fi
-
- echo "`date -u '+%F %T'`: updating from $START" >>$DB_DIR/apply_osc_to_db.log
-
- TEMP_DIR=`mktemp -d /tmp/osm-3s_update_XXXXXX`
- collect_minute_diffs $TEMP_DIR
-
- if [[ $TARGET -gt $START ]]; then
- {
- echo "`date -u '+%F %T'`: updating to $TARGET" >>$DB_DIR/apply_osc_to_db.log
-
- update_state
- apply_minute_diffs $TEMP_DIR
- echo "$TARGET" >$DB_DIR/replicate_id
-
- echo "`date -u '+%F %T'`: update complete" $TARGET >>$DB_DIR/apply_osc_to_db.log
- };
- else
- {
- sleep 5
- }; fi
-
- rm -f $TEMP_DIR/*
- rmdir $TEMP_DIR
-
- START=$TARGET
-}; done
diff --git a/0.7.54.12/bin/fetch_osc.sh b/0.7.54.12/bin/fetch_osc.sh
deleted file mode 100755
index a9d163f..0000000
--- a/0.7.54.12/bin/fetch_osc.sh
+++ /dev/null
@@ -1,163 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht
-#
-# This file is part of Overpass_API.
-#
-# Overpass_API is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# Overpass_API is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with Overpass_API. If not, see .
-
-if [[ -z $1 ]]; then
-{
- echo Usage: $0 Replicate_id Source_dir Local_dir [Sleep]
- exit 0
-};
-fi
-
-REPLICATE_ID=$1
-SOURCE_DIR=$2
-LOCAL_DIR=$3
-SLEEP_BETWEEN_DLS=15 # How long to sleep between download attempts (sec). Default: 15. See also FAIL_COUNTER_ALERT
-FAIL_COUNTER_ALERT=20 # After how many sleep cycles do we get nervous
-FAIL_COUNTER=0
-FILE_PANIC=
-
-if [[ ! -d $LOCAL_DIR ]];
- then {
- mkdir $LOCAL_DIR
-};
-fi
-
-if [[ $REPLICATE_ID == "auto" ]] ; then
- REPLICATE_ID=`find $LOCAL_DIR -type f -name '*state.txt' -not -size 0 | sort | tail -n 1 | sed -e "s#^$LOCAL_DIR##" -e 's#[./]##g' -e 's#statetxt##' -e 's/^0\+//g'`
-
- EXEC_DIR="`dirname $0`/"
- DB_DIR=`$EXEC_DIR/dispatcher --show-dir`
-
- if [[ ! -d $DB_DIR ]] ; then
- echo "Can't find DB_DIR. Returned value: $DB_DIR"
- exit 1;
- fi
-
- if [[ "x$REPLICATE_ID" == "x" ]] ; then
- REPLICATE_ID=`cat $DB_DIR/replicate_id`
- fi
- if [[ "x$REPLICATE_ID" == "x" ]] ; then
- echo "Could not determine REPLICATE_ID. Exiting"
- exit 1
- fi
-fi
-
-# $1 - remote source
-# $2 - local destination
-fetch_file()
-{
- wget -nv -O "$2" "$1"
-};
-
-retry_fetch_file()
-{
- FILE_PANIC=
- FAIL_COUNTER=0
- if [[ ! -s "$2" ]]; then {
- fetch_file "$1" "$2"
- if [[ "$3" == "gzip" ]]; then {
- gunzip -t <"$2"
- if [[ $? -ne 0 ]]; then {
- rm "$2"
- }; fi
- }; fi
- }; fi
- until [[ -s "$2" || $FAIL_COUNTER -ge $FAIL_COUNTER_ALERT ]]; do {
- FAIL_COUNTER=$(($FAIL_COUNTER+1))
- sleep $SLEEP_BETWEEN_DLS
- fetch_file "$1" "$2"
- if [[ "$3" == "gzip" ]]; then {
- gunzip -t <"$2"
- if [[ $? -ne 0 ]]; then {
- rm "$2"
- }; fi
- }; fi
- }; done
- if [[ $FAIL_COUNTER -ge $FAIL_COUNTER_ALERT ]]; then
- FILE_PANIC="true"
- fi
-};
-
-file_panic()
-{
- echo "fetch_osc()@"`date -u "+%F %T"`": upstream_delay $REPLICATE_ID" >>$LOCAL_DIR/fetch_osc.log
- REPLICATE_ID=$(($REPLICATE_ID - 1))
-
- printf -v TDIGIT3 %03u $(($REPLICATE_ID % 1000))
- ARG=$(($REPLICATE_ID / 1000))
- printf -v TDIGIT2 %03u $(($ARG % 1000))
- ARG=$(($ARG / 1000))
- printf -v TDIGIT1 %03u $ARG
-
- FILE_PANIC="true"
- until [[ ! -n $FILE_PANIC ]]; do {
- retry_fetch_file "$REMOTE_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.new.osc.gz" "gzip"
- }; done
- FILE_PANIC="true"
- until [[ ! -n $FILE_PANIC ]]; do {
- retry_fetch_file "$REMOTE_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.new.state.txt" "text"
- }; done
-
- RES_GZIP=`diff -q "$LOCAL_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.new.osc.gz"`
- RES_TEXT=`diff -q "$LOCAL_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.new.state.txt"`
- if [[ -n $RES_GZIP || -n $RES_TEXT ]]; then
- echo "fetch_osc()@"`date -u "+%F %T"`": file_panic $REPLICATE_ID" >>$LOCAL_DIR/fetch_osc.log
- echo "fetch_osc()@"`date -u "+%F %T"`": $RES_GZIP" >>$LOCAL_DIR/fetch_osc.log
- echo "fetch_osc()@"`date -u "+%F %T"`": $RES_TEXT" >>$LOCAL_DIR/fetch_osc.log
- exit 1
- fi
-
- rm "$LOCAL_PATH/$TDIGIT3.new.osc.gz"
- rm "$LOCAL_PATH/$TDIGIT3.new.state.txt"
-};
-
-fetch_minute_diff()
-{
- printf -v TDIGIT3 %03u $(($REPLICATE_ID % 1000))
- ARG=$(($REPLICATE_ID / 1000))
- printf -v TDIGIT2 %03u $(($ARG % 1000))
- ARG=$(($ARG / 1000))
- printf -v TDIGIT1 %03u $ARG
-
- LOCAL_PATH="$LOCAL_DIR/$TDIGIT1/$TDIGIT2"
- REMOTE_PATH="$SOURCE_DIR/$TDIGIT1/$TDIGIT2"
- mkdir -p "$LOCAL_DIR/$TDIGIT1/$TDIGIT2"
-
- retry_fetch_file "$REMOTE_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.osc.gz" "gzip"
- if [[ -n $FILE_PANIC ]]; then
- file_panic
- fi
- retry_fetch_file "$REMOTE_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.state.txt" "text"
- if [[ -n $FILE_PANIC ]]; then
- file_panic
- fi
-
- TIMESTAMP_LINE=`grep timestamp $LOCAL_DIR/$TDIGIT1/$TDIGIT2/$TDIGIT3.state.txt`
- TIMESTAMP=${TIMESTAMP_LINE:10}
-};
-
-while [[ true ]];
-do
-{
- REPLICATE_ID=$(($REPLICATE_ID + 1))
- fetch_minute_diff
- echo "fetch_osc()@"`date -u "+%F %T"`": new_replicate_diff $REPLICATE_ID $TIMESTAMP" >>$LOCAL_DIR/fetch_osc.log
- sleep 1
-};
-done
diff --git a/0.7.54.12/bin/update_overpass.sh b/0.7.54.12/bin/update_overpass.sh
new file mode 100644
index 0000000..328e375
--- /dev/null
+++ b/0.7.54.12/bin/update_overpass.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env sh
+
+(
+ set -e
+
+ if [ "x$OVERPASS_META" = "xattic" ] ; then
+ META="--keep-attic"
+ elif [ "x$OVERPASS_META" = "xyes" ] ; then
+ META="--meta"
+ else
+ META=""
+ fi
+
+ if [ ! -d /db/diffs ] ; then
+ mkdir /db/diffs
+ fi
+
+ while `true` ; do
+ (
+ if [ ! -e /db/diffs/changes.osm ] ; then
+ ! /app/venv/bin/pyosmium-get-changes --server $OVERPASS_DIFF_URL -o /db/diffs/changes.osm -f /db/replicate_id
+ OSMIUM_STATUS=$?
+ if [ $OSMIUM_STATUS -eq 1 ]; then
+ echo "There are still some updates remainging"
+ fi
+ if [ $OSMIUM_STATUS -eq 2 ]; then
+ echo "Failure downloading updates"
+ exit 0
+ fi
+ else
+ echo "/db/diffs/changes.osm exists. Trying to apply again."
+ fi
+ /app/bin/update_from_dir --osc-dir=/db/diffs/ $META
+ rm /db/diffs/changes.osm
+ )
+ sleep 60
+ done
+) 2>&1 | tee -a /db/changes.log
\ No newline at end of file
diff --git a/0.7.54.12/docker-entrypoint.sh b/0.7.54.12/docker-entrypoint.sh
index 5a43603..cb69627 100644
--- a/0.7.54.12/docker-entrypoint.sh
+++ b/0.7.54.12/docker-entrypoint.sh
@@ -6,13 +6,15 @@ OVERPASS_META=${OVERPASS_META:-no}
OVERPASS_MODE=${OVERPASS_MODE:-clone}
OVERPASS_COMPRESSION=${OVERPASS_COMPRESSION:-gz}
+if [[ "$OVERPASS_META" == "attic" ]] ; then
+ META="--keep-attic"
+else
+ META="--meta"
+fi
+
+
if [ ! -d /db/db ] ; then
if [ "$OVERPASS_MODE" = "clone" ]; then
- if [[ "$OVERPASS_META" == "attic" ]] ; then
- META="--keep-attic"
- else
- META="--meta=$OVERPASS_META"
- fi
mkdir -p /db/db \
&& /app/bin/download_clone.sh --db-dir=/db/db --source=http://dev.overpass-api.de/api_drolbr/ $META \
&& cp -r /app/etc/rules /db/db \
@@ -22,10 +24,21 @@ if [ ! -d /db/db ] ; then
fi
if [ "$OVERPASS_MODE" = "init" ]; then
- lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet; exit" \
- && /app/bin/init_osm3s.sh /db/planet /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION" \
- && echo $OVERPASS_PLANET_SEQUENCE_ID > /db/db/replicate_id \
- && rm /db/planet \
+ lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet.osm.bz2; exit" \
+ && /app/bin/init_osm3s.sh /db/planet.osm.bz2 /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION --map-compression-method=$OVERPASS_COMPRESSION"\
+ && echo "Database created. Now updating it." && (
+ ! /app/venv/bin/pyosmium-get-changes -O /db/planet.osm.bz2 --server $OVERPASS_DIFF_URL -o /db/changes.osm -f /db/replicate_id
+ OSMIUM_STATUS=$?
+ if [ $OSMIUM_STATUS -eq 1 ]; then
+ echo "There are still some updates remainging"
+ fi
+ if [ $OSMIUM_STATUS -eq 2 ]; then
+ echo "Failure downloading updates"
+ exit 0
+ fi
+ (cat /db/changes.osm | /app/bin/update_database --db-dir=/db/db $META --compression-method=$OVERPASS_COMPRESSION) 2>&1 | tee -a /db/changes.log
+ ) \
+ && rm /db/planet.osm.bz2 /db/changes.osm \
&& cp -r /app/etc/rules /db/db \
&& chown -R overpass:overpass /db \
&& echo "Overpass ready, you can start your container with docker start"
diff --git a/0.7.54.12/etc/nginx-overpass.conf b/0.7.54.12/etc/nginx-overpass.conf
index c019615..a19dde4 100644
--- a/0.7.54.12/etc/nginx-overpass.conf
+++ b/0.7.54.12/etc/nginx-overpass.conf
@@ -25,7 +25,7 @@ http {
sendfile on;
#tcp_nopush on;
- keepalive_timeout 65;
+ keepalive_timeout 1000s;
#gzip on;
@@ -55,6 +55,7 @@ http {
fastcgi_param SCRIPT_FILENAME /app/$fastcgi_script_name;
fastcgi_param PATH_INFO $fastcgi_path_info;
fastcgi_pass unix:/nginx/fcgiwrap.socket;
+ fastcgi_read_timeout 1000s;
}
location /api/ {
diff --git a/0.7.54.12/etc/supervisord.conf b/0.7.54.12/etc/supervisord.conf
index 6c4623b..fc67b74 100644
--- a/0.7.54.12/etc/supervisord.conf
+++ b/0.7.54.12/etc/supervisord.conf
@@ -17,18 +17,12 @@ redirect_stderr=true
user=nginx
priority=3
-[program:fetch_diff]
-command=/app/bin/fetch_osc.sh auto %(ENV_OVERPASS_DIFF_URL)s /db/diffs
+[program:update_overpass]
+command=/app/bin/update_overpass.sh
user=overpass
redirect_stderr=true
priority=5
-[program:apply_diff]
-command=/app/bin/apply_osc_to_db.sh /db/diffs/ auto --meta=%(ENV_OVERPASS_META)s
-redirect_stderr=true
-user=overpass
-priority=4
-
[program:dispatcher_areas]
command=nice /app/bin/dispatcher --areas --db-dir="/db/db"
user=overpass
diff --git a/0.7.54.12/requirements.txt b/0.7.54.12/requirements.txt
new file mode 100644
index 0000000..8314012
--- /dev/null
+++ b/0.7.54.12/requirements.txt
@@ -0,0 +1 @@
+osmium
diff --git a/0.7.54.13/Dockerfile b/0.7.54.13/Dockerfile
index 1e1796c..221a5e8 100644
--- a/0.7.54.13/Dockerfile
+++ b/0.7.54.13/Dockerfile
@@ -1,27 +1,34 @@
-FROM nginx:stable
+FROM nginx:1.14
RUN addgroup overpass && adduser --home /db --disabled-password --gecos overpass --ingroup overpass overpass
+ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.54.13.tar.gz /app/src.tar.gz
+
RUN apt-get update \
&& apt-get install --no-install-recommends --no-install-suggests -y \
autoconf \
automake \
+ bash \
+ bzip2 \
ca-certificates \
expat \
- libexpat1-dev \
+ fcgiwrap \
g++ \
+ lftp \
+ libexpat1-dev \
liblz4-1 \
liblz4-dev \
libtool \
m4 \
make \
+ osmium-tool \
+ python3 \
+ python3-venv \
+ supervisor \
+ wget \
zlib1g \
- zlib1g-dev
-
-
-ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.54.13.tar.gz /app/src.tar.gz
-
-RUN mkdir -p /app/src \
+ zlib1g-dev \
+ && mkdir -p /app/src \
&& cd /app/src \
&& tar -x -z --strip-components 1 -f ../src.tar.gz \
&& autoscan \
@@ -45,26 +52,28 @@ RUN mkdir -p /app/src \
&& apt-get autoremove -y \
&& mkdir -p /db/diffs /app/etc \
&& cp -r /app/src/rules /app/etc/rules \
- && rm -rf /app/src /app/src.tar.gz
+ && rm -rf /app/src /app/src.tar.gz \
+ && rm -rf /var/lib/apt/lists/*
-RUN apt-get install --no-install-recommends --no-install-suggests -y \
- supervisor \
- bash \
- lftp \
- wget \
- fcgiwrap \
- bzip2
+COPY requirements.txt /app/
+RUN python3 -m venv /app/venv \
+ && /app/venv/bin/pip install -r /app/requirements.txt
RUN mkdir /nginx && chown nginx:nginx /nginx && chown -R overpass:overpass /db
+
COPY etc/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+
COPY etc/nginx-overpass.conf /etc/nginx/nginx.conf
-COPY bin/apply_osc_to_db.sh bin/fetch_osc.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/
+
+COPY bin/update_overpass.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/
+
COPY docker-entrypoint.sh /app/
-RUN chmod a+x /app/docker-entrypoint.sh /app/bin/apply_osc_to_db.sh /app/bin/fetch_osc.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh
-VOLUME /db
+
+RUN chmod a+x /app/docker-entrypoint.sh /app/bin/update_overpass.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh
ENV OVERPASS_RULES_LOAD 1
+
EXPOSE 80
# CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"]
CMD ["/app/docker-entrypoint.sh"]
diff --git a/0.7.54.13/bin/apply_osc_to_db.sh b/0.7.54.13/bin/apply_osc_to_db.sh
deleted file mode 100755
index 94d1879..0000000
--- a/0.7.54.13/bin/apply_osc_to_db.sh
+++ /dev/null
@@ -1,163 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht
-#
-# This file is part of Overpass_API.
-#
-# Overpass_API is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# Overpass_API is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with Overpass_API. If not, see .
-
-if [[ -z $3 ]]; then
-{
- echo "Usage: $0 replicate_dir start_id --meta=(attic|yes|no)"
- exit 0
-}; fi
-
-EXEC_DIR="`dirname $0`/"
-if [[ ! ${EXEC_DIR:0:1} == "/" ]]; then
-{
- EXEC_DIR="`pwd`/$EXEC_DIR"
-}; fi
-
-DB_DIR=`$EXEC_DIR/dispatcher --show-dir`
-
-if [[ ! -d $DB_DIR ]] ; then
- echo "Can't find DB_DIR. Returned value: $DB_DIR"
- exit 1;
-fi
-
-REPLICATE_DIR="$1"
-if [[ ! ${REPLICATE_DIR:0:1} == "/" ]]; then
-{
- REPLICATE_DIR="`pwd`/$REPLICATE_DIR"
-}; fi
-
-START=$2
-META=
-
-if [[ $3 == "--meta=attic" ]]; then
- META="--keep-attic"
-elif [[ $3 == "--meta=yes" || $3 == "--meta" ]]; then
- META="--meta"
-elif [[ $3 == "--meta=no" ]]; then
- META=
-else
-{
- echo "You must specify --meta=yes or --meta=no"
- exit 0
-}; fi
-
-PRODUCE_DIFF=
-
-get_replicate_filename()
-{
- printf -v TDIGIT3 %03u $(($1 % 1000))
- ARG=$(($1 / 1000))
- printf -v TDIGIT2 %03u $(($ARG % 1000))
- ARG=$(($ARG / 1000))
- printf -v TDIGIT1 %03u $ARG
- REPLICATE_TRUNK_DIR=$TDIGIT1/$TDIGIT2/
- REPLICATE_FILENAME=$TDIGIT1/$TDIGIT2/$TDIGIT3
-};
-
-
-collect_minute_diffs()
-{
- TEMP_DIR=$1
- TARGET=$(($START + 1))
-
- get_replicate_filename $TARGET
-
- while [[ ( -s $REPLICATE_DIR/$REPLICATE_FILENAME.state.txt ) && ( $(($START + 1440)) -ge $(($TARGET)) ) && ( `du -m $TEMP_DIR | awk '{ print $1; }'` -le 512 ) ]];
- do
- {
- printf -v TARGET_FILE %09u $TARGET
- gunzip <$REPLICATE_DIR/$REPLICATE_FILENAME.osc.gz >$TEMP_DIR/$TARGET_FILE.osc
- TARGET=$(($TARGET + 1))
- get_replicate_filename $TARGET
- };
- done
- TARGET=$(($TARGET - 1))
-};
-
-
-apply_minute_diffs()
-{
- ./update_from_dir --osc-dir=$1 --version=$DATA_VERSION $META --flush-size=0
- EXITCODE=$?
- while [[ $EXITCODE -ne 0 ]];
- do
- {
- sleep 60
- ./update_from_dir --osc-dir=$1 --version=$DATA_VERSION $META --flush-size=0
- EXITCODE=$?
- };
- done
- DIFF_COUNT=$(($DIFF_COUNT + 1))
-};
-
-
-update_state()
-{
- get_replicate_filename $TARGET
- TIMESTAMP_LINE=`grep "^timestamp" <$REPLICATE_DIR/$REPLICATE_FILENAME.state.txt`
- while [[ -z $TIMESTAMP_LINE ]]; do
- {
- sleep 5
- TIMESTAMP_LINE=`grep "^timestamp" <$REPLICATE_DIR/$REPLICATE_FILENAME.state.txt`
- }; done
- DATA_VERSION=${TIMESTAMP_LINE:10}
-};
-
-
-echo >>$DB_DIR/apply_osc_to_db.log
-
-mkdir -p $DB_DIR/augmented_diffs/
-DIFF_COUNT=0
-
-# update_state
-
-pushd "$EXEC_DIR"
-
-while [[ true ]]; do
-{
- if [[ $START == "auto" ]]; then
- {
- START=`cat $DB_DIR/replicate_id`
- }; fi
-
- echo "`date -u '+%F %T'`: updating from $START" >>$DB_DIR/apply_osc_to_db.log
-
- TEMP_DIR=`mktemp -d /tmp/osm-3s_update_XXXXXX`
- collect_minute_diffs $TEMP_DIR
-
- if [[ $TARGET -gt $START ]]; then
- {
- echo "`date -u '+%F %T'`: updating to $TARGET" >>$DB_DIR/apply_osc_to_db.log
-
- update_state
- apply_minute_diffs $TEMP_DIR
- echo "$TARGET" >$DB_DIR/replicate_id
-
- echo "`date -u '+%F %T'`: update complete" $TARGET >>$DB_DIR/apply_osc_to_db.log
- };
- else
- {
- sleep 5
- }; fi
-
- rm -f $TEMP_DIR/*
- rmdir $TEMP_DIR
-
- START=$TARGET
-}; done
diff --git a/0.7.54.13/bin/fetch_osc.sh b/0.7.54.13/bin/fetch_osc.sh
deleted file mode 100755
index a9d163f..0000000
--- a/0.7.54.13/bin/fetch_osc.sh
+++ /dev/null
@@ -1,163 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht
-#
-# This file is part of Overpass_API.
-#
-# Overpass_API is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# Overpass_API is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with Overpass_API. If not, see .
-
-if [[ -z $1 ]]; then
-{
- echo Usage: $0 Replicate_id Source_dir Local_dir [Sleep]
- exit 0
-};
-fi
-
-REPLICATE_ID=$1
-SOURCE_DIR=$2
-LOCAL_DIR=$3
-SLEEP_BETWEEN_DLS=15 # How long to sleep between download attempts (sec). Default: 15. See also FAIL_COUNTER_ALERT
-FAIL_COUNTER_ALERT=20 # After how many sleep cycles do we get nervous
-FAIL_COUNTER=0
-FILE_PANIC=
-
-if [[ ! -d $LOCAL_DIR ]];
- then {
- mkdir $LOCAL_DIR
-};
-fi
-
-if [[ $REPLICATE_ID == "auto" ]] ; then
- REPLICATE_ID=`find $LOCAL_DIR -type f -name '*state.txt' -not -size 0 | sort | tail -n 1 | sed -e "s#^$LOCAL_DIR##" -e 's#[./]##g' -e 's#statetxt##' -e 's/^0\+//g'`
-
- EXEC_DIR="`dirname $0`/"
- DB_DIR=`$EXEC_DIR/dispatcher --show-dir`
-
- if [[ ! -d $DB_DIR ]] ; then
- echo "Can't find DB_DIR. Returned value: $DB_DIR"
- exit 1;
- fi
-
- if [[ "x$REPLICATE_ID" == "x" ]] ; then
- REPLICATE_ID=`cat $DB_DIR/replicate_id`
- fi
- if [[ "x$REPLICATE_ID" == "x" ]] ; then
- echo "Could not determine REPLICATE_ID. Exiting"
- exit 1
- fi
-fi
-
-# $1 - remote source
-# $2 - local destination
-fetch_file()
-{
- wget -nv -O "$2" "$1"
-};
-
-retry_fetch_file()
-{
- FILE_PANIC=
- FAIL_COUNTER=0
- if [[ ! -s "$2" ]]; then {
- fetch_file "$1" "$2"
- if [[ "$3" == "gzip" ]]; then {
- gunzip -t <"$2"
- if [[ $? -ne 0 ]]; then {
- rm "$2"
- }; fi
- }; fi
- }; fi
- until [[ -s "$2" || $FAIL_COUNTER -ge $FAIL_COUNTER_ALERT ]]; do {
- FAIL_COUNTER=$(($FAIL_COUNTER+1))
- sleep $SLEEP_BETWEEN_DLS
- fetch_file "$1" "$2"
- if [[ "$3" == "gzip" ]]; then {
- gunzip -t <"$2"
- if [[ $? -ne 0 ]]; then {
- rm "$2"
- }; fi
- }; fi
- }; done
- if [[ $FAIL_COUNTER -ge $FAIL_COUNTER_ALERT ]]; then
- FILE_PANIC="true"
- fi
-};
-
-file_panic()
-{
- echo "fetch_osc()@"`date -u "+%F %T"`": upstream_delay $REPLICATE_ID" >>$LOCAL_DIR/fetch_osc.log
- REPLICATE_ID=$(($REPLICATE_ID - 1))
-
- printf -v TDIGIT3 %03u $(($REPLICATE_ID % 1000))
- ARG=$(($REPLICATE_ID / 1000))
- printf -v TDIGIT2 %03u $(($ARG % 1000))
- ARG=$(($ARG / 1000))
- printf -v TDIGIT1 %03u $ARG
-
- FILE_PANIC="true"
- until [[ ! -n $FILE_PANIC ]]; do {
- retry_fetch_file "$REMOTE_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.new.osc.gz" "gzip"
- }; done
- FILE_PANIC="true"
- until [[ ! -n $FILE_PANIC ]]; do {
- retry_fetch_file "$REMOTE_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.new.state.txt" "text"
- }; done
-
- RES_GZIP=`diff -q "$LOCAL_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.new.osc.gz"`
- RES_TEXT=`diff -q "$LOCAL_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.new.state.txt"`
- if [[ -n $RES_GZIP || -n $RES_TEXT ]]; then
- echo "fetch_osc()@"`date -u "+%F %T"`": file_panic $REPLICATE_ID" >>$LOCAL_DIR/fetch_osc.log
- echo "fetch_osc()@"`date -u "+%F %T"`": $RES_GZIP" >>$LOCAL_DIR/fetch_osc.log
- echo "fetch_osc()@"`date -u "+%F %T"`": $RES_TEXT" >>$LOCAL_DIR/fetch_osc.log
- exit 1
- fi
-
- rm "$LOCAL_PATH/$TDIGIT3.new.osc.gz"
- rm "$LOCAL_PATH/$TDIGIT3.new.state.txt"
-};
-
-fetch_minute_diff()
-{
- printf -v TDIGIT3 %03u $(($REPLICATE_ID % 1000))
- ARG=$(($REPLICATE_ID / 1000))
- printf -v TDIGIT2 %03u $(($ARG % 1000))
- ARG=$(($ARG / 1000))
- printf -v TDIGIT1 %03u $ARG
-
- LOCAL_PATH="$LOCAL_DIR/$TDIGIT1/$TDIGIT2"
- REMOTE_PATH="$SOURCE_DIR/$TDIGIT1/$TDIGIT2"
- mkdir -p "$LOCAL_DIR/$TDIGIT1/$TDIGIT2"
-
- retry_fetch_file "$REMOTE_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.osc.gz" "gzip"
- if [[ -n $FILE_PANIC ]]; then
- file_panic
- fi
- retry_fetch_file "$REMOTE_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.state.txt" "text"
- if [[ -n $FILE_PANIC ]]; then
- file_panic
- fi
-
- TIMESTAMP_LINE=`grep timestamp $LOCAL_DIR/$TDIGIT1/$TDIGIT2/$TDIGIT3.state.txt`
- TIMESTAMP=${TIMESTAMP_LINE:10}
-};
-
-while [[ true ]];
-do
-{
- REPLICATE_ID=$(($REPLICATE_ID + 1))
- fetch_minute_diff
- echo "fetch_osc()@"`date -u "+%F %T"`": new_replicate_diff $REPLICATE_ID $TIMESTAMP" >>$LOCAL_DIR/fetch_osc.log
- sleep 1
-};
-done
diff --git a/0.7.54.13/bin/update_overpass.sh b/0.7.54.13/bin/update_overpass.sh
new file mode 100644
index 0000000..328e375
--- /dev/null
+++ b/0.7.54.13/bin/update_overpass.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env sh
+
+(
+ set -e
+
+ if [ "x$OVERPASS_META" = "xattic" ] ; then
+ META="--keep-attic"
+ elif [ "x$OVERPASS_META" = "xyes" ] ; then
+ META="--meta"
+ else
+ META=""
+ fi
+
+ if [ ! -d /db/diffs ] ; then
+ mkdir /db/diffs
+ fi
+
+ while `true` ; do
+ (
+ if [ ! -e /db/diffs/changes.osm ] ; then
+ ! /app/venv/bin/pyosmium-get-changes --server $OVERPASS_DIFF_URL -o /db/diffs/changes.osm -f /db/replicate_id
+ OSMIUM_STATUS=$?
+ if [ $OSMIUM_STATUS -eq 1 ]; then
+ echo "There are still some updates remainging"
+ fi
+ if [ $OSMIUM_STATUS -eq 2 ]; then
+ echo "Failure downloading updates"
+ exit 0
+ fi
+ else
+ echo "/db/diffs/changes.osm exists. Trying to apply again."
+ fi
+ /app/bin/update_from_dir --osc-dir=/db/diffs/ $META
+ rm /db/diffs/changes.osm
+ )
+ sleep 60
+ done
+) 2>&1 | tee -a /db/changes.log
\ No newline at end of file
diff --git a/0.7.54.13/docker-entrypoint.sh b/0.7.54.13/docker-entrypoint.sh
index 5a43603..cb69627 100644
--- a/0.7.54.13/docker-entrypoint.sh
+++ b/0.7.54.13/docker-entrypoint.sh
@@ -6,13 +6,15 @@ OVERPASS_META=${OVERPASS_META:-no}
OVERPASS_MODE=${OVERPASS_MODE:-clone}
OVERPASS_COMPRESSION=${OVERPASS_COMPRESSION:-gz}
+if [[ "$OVERPASS_META" == "attic" ]] ; then
+ META="--keep-attic"
+else
+ META="--meta"
+fi
+
+
if [ ! -d /db/db ] ; then
if [ "$OVERPASS_MODE" = "clone" ]; then
- if [[ "$OVERPASS_META" == "attic" ]] ; then
- META="--keep-attic"
- else
- META="--meta=$OVERPASS_META"
- fi
mkdir -p /db/db \
&& /app/bin/download_clone.sh --db-dir=/db/db --source=http://dev.overpass-api.de/api_drolbr/ $META \
&& cp -r /app/etc/rules /db/db \
@@ -22,10 +24,21 @@ if [ ! -d /db/db ] ; then
fi
if [ "$OVERPASS_MODE" = "init" ]; then
- lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet; exit" \
- && /app/bin/init_osm3s.sh /db/planet /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION" \
- && echo $OVERPASS_PLANET_SEQUENCE_ID > /db/db/replicate_id \
- && rm /db/planet \
+ lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet.osm.bz2; exit" \
+ && /app/bin/init_osm3s.sh /db/planet.osm.bz2 /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION --map-compression-method=$OVERPASS_COMPRESSION"\
+ && echo "Database created. Now updating it." && (
+ ! /app/venv/bin/pyosmium-get-changes -O /db/planet.osm.bz2 --server $OVERPASS_DIFF_URL -o /db/changes.osm -f /db/replicate_id
+ OSMIUM_STATUS=$?
+ if [ $OSMIUM_STATUS -eq 1 ]; then
+ echo "There are still some updates remainging"
+ fi
+ if [ $OSMIUM_STATUS -eq 2 ]; then
+ echo "Failure downloading updates"
+ exit 0
+ fi
+ (cat /db/changes.osm | /app/bin/update_database --db-dir=/db/db $META --compression-method=$OVERPASS_COMPRESSION) 2>&1 | tee -a /db/changes.log
+ ) \
+ && rm /db/planet.osm.bz2 /db/changes.osm \
&& cp -r /app/etc/rules /db/db \
&& chown -R overpass:overpass /db \
&& echo "Overpass ready, you can start your container with docker start"
diff --git a/0.7.54.13/etc/nginx-overpass.conf b/0.7.54.13/etc/nginx-overpass.conf
index c019615..a19dde4 100644
--- a/0.7.54.13/etc/nginx-overpass.conf
+++ b/0.7.54.13/etc/nginx-overpass.conf
@@ -25,7 +25,7 @@ http {
sendfile on;
#tcp_nopush on;
- keepalive_timeout 65;
+ keepalive_timeout 1000s;
#gzip on;
@@ -55,6 +55,7 @@ http {
fastcgi_param SCRIPT_FILENAME /app/$fastcgi_script_name;
fastcgi_param PATH_INFO $fastcgi_path_info;
fastcgi_pass unix:/nginx/fcgiwrap.socket;
+ fastcgi_read_timeout 1000s;
}
location /api/ {
diff --git a/0.7.54.13/etc/supervisord.conf b/0.7.54.13/etc/supervisord.conf
index 6c4623b..fc67b74 100644
--- a/0.7.54.13/etc/supervisord.conf
+++ b/0.7.54.13/etc/supervisord.conf
@@ -17,18 +17,12 @@ redirect_stderr=true
user=nginx
priority=3
-[program:fetch_diff]
-command=/app/bin/fetch_osc.sh auto %(ENV_OVERPASS_DIFF_URL)s /db/diffs
+[program:update_overpass]
+command=/app/bin/update_overpass.sh
user=overpass
redirect_stderr=true
priority=5
-[program:apply_diff]
-command=/app/bin/apply_osc_to_db.sh /db/diffs/ auto --meta=%(ENV_OVERPASS_META)s
-redirect_stderr=true
-user=overpass
-priority=4
-
[program:dispatcher_areas]
command=nice /app/bin/dispatcher --areas --db-dir="/db/db"
user=overpass
diff --git a/0.7.54.13/requirements.txt b/0.7.54.13/requirements.txt
new file mode 100644
index 0000000..8314012
--- /dev/null
+++ b/0.7.54.13/requirements.txt
@@ -0,0 +1 @@
+osmium
diff --git a/0.7.54.9/Dockerfile b/0.7.54.9/Dockerfile
index b557fc2..f680449 100644
--- a/0.7.54.9/Dockerfile
+++ b/0.7.54.9/Dockerfile
@@ -1,27 +1,34 @@
-FROM nginx:stable
+FROM nginx:1.14
RUN addgroup overpass && adduser --home /db --disabled-password --gecos overpass --ingroup overpass overpass
+ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.54.9.tar.gz /app/src.tar.gz
+
RUN apt-get update \
&& apt-get install --no-install-recommends --no-install-suggests -y \
autoconf \
automake \
+ bash \
+ bzip2 \
ca-certificates \
expat \
- libexpat1-dev \
+ fcgiwrap \
g++ \
+ lftp \
+ libexpat1-dev \
liblz4-1 \
liblz4-dev \
libtool \
m4 \
make \
+ osmium-tool \
+ python3 \
+ python3-venv \
+ supervisor \
+ wget \
zlib1g \
- zlib1g-dev
-
-
-ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.54.9.tar.gz /app/src.tar.gz
-
-RUN mkdir -p /app/src \
+ zlib1g-dev \
+ && mkdir -p /app/src \
&& cd /app/src \
&& tar -x -z --strip-components 1 -f ../src.tar.gz \
&& autoscan \
@@ -45,26 +52,28 @@ RUN mkdir -p /app/src \
&& apt-get autoremove -y \
&& mkdir -p /db/diffs /app/etc \
&& cp -r /app/src/rules /app/etc/rules \
- && rm -rf /app/src /app/src.tar.gz
+ && rm -rf /app/src /app/src.tar.gz \
+ && rm -rf /var/lib/apt/lists/*
-RUN apt-get install --no-install-recommends --no-install-suggests -y \
- supervisor \
- bash \
- lftp \
- wget \
- fcgiwrap \
- bzip2
+COPY requirements.txt /app/
+RUN python3 -m venv /app/venv \
+ && /app/venv/bin/pip install -r /app/requirements.txt
RUN mkdir /nginx && chown nginx:nginx /nginx && chown -R overpass:overpass /db
+
COPY etc/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+
COPY etc/nginx-overpass.conf /etc/nginx/nginx.conf
-COPY bin/apply_osc_to_db.sh bin/fetch_osc.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/
+
+COPY bin/update_overpass.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/
+
COPY docker-entrypoint.sh /app/
-RUN chmod a+x /app/docker-entrypoint.sh /app/bin/apply_osc_to_db.sh /app/bin/fetch_osc.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh
-VOLUME /db
+
+RUN chmod a+x /app/docker-entrypoint.sh /app/bin/update_overpass.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh
ENV OVERPASS_RULES_LOAD 1
+
EXPOSE 80
# CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"]
CMD ["/app/docker-entrypoint.sh"]
diff --git a/0.7.54.9/bin/apply_osc_to_db.sh b/0.7.54.9/bin/apply_osc_to_db.sh
deleted file mode 100755
index 94d1879..0000000
--- a/0.7.54.9/bin/apply_osc_to_db.sh
+++ /dev/null
@@ -1,163 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht
-#
-# This file is part of Overpass_API.
-#
-# Overpass_API is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# Overpass_API is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with Overpass_API. If not, see .
-
-if [[ -z $3 ]]; then
-{
- echo "Usage: $0 replicate_dir start_id --meta=(attic|yes|no)"
- exit 0
-}; fi
-
-EXEC_DIR="`dirname $0`/"
-if [[ ! ${EXEC_DIR:0:1} == "/" ]]; then
-{
- EXEC_DIR="`pwd`/$EXEC_DIR"
-}; fi
-
-DB_DIR=`$EXEC_DIR/dispatcher --show-dir`
-
-if [[ ! -d $DB_DIR ]] ; then
- echo "Can't find DB_DIR. Returned value: $DB_DIR"
- exit 1;
-fi
-
-REPLICATE_DIR="$1"
-if [[ ! ${REPLICATE_DIR:0:1} == "/" ]]; then
-{
- REPLICATE_DIR="`pwd`/$REPLICATE_DIR"
-}; fi
-
-START=$2
-META=
-
-if [[ $3 == "--meta=attic" ]]; then
- META="--keep-attic"
-elif [[ $3 == "--meta=yes" || $3 == "--meta" ]]; then
- META="--meta"
-elif [[ $3 == "--meta=no" ]]; then
- META=
-else
-{
- echo "You must specify --meta=yes or --meta=no"
- exit 0
-}; fi
-
-PRODUCE_DIFF=
-
-get_replicate_filename()
-{
- printf -v TDIGIT3 %03u $(($1 % 1000))
- ARG=$(($1 / 1000))
- printf -v TDIGIT2 %03u $(($ARG % 1000))
- ARG=$(($ARG / 1000))
- printf -v TDIGIT1 %03u $ARG
- REPLICATE_TRUNK_DIR=$TDIGIT1/$TDIGIT2/
- REPLICATE_FILENAME=$TDIGIT1/$TDIGIT2/$TDIGIT3
-};
-
-
-collect_minute_diffs()
-{
- TEMP_DIR=$1
- TARGET=$(($START + 1))
-
- get_replicate_filename $TARGET
-
- while [[ ( -s $REPLICATE_DIR/$REPLICATE_FILENAME.state.txt ) && ( $(($START + 1440)) -ge $(($TARGET)) ) && ( `du -m $TEMP_DIR | awk '{ print $1; }'` -le 512 ) ]];
- do
- {
- printf -v TARGET_FILE %09u $TARGET
- gunzip <$REPLICATE_DIR/$REPLICATE_FILENAME.osc.gz >$TEMP_DIR/$TARGET_FILE.osc
- TARGET=$(($TARGET + 1))
- get_replicate_filename $TARGET
- };
- done
- TARGET=$(($TARGET - 1))
-};
-
-
-apply_minute_diffs()
-{
- ./update_from_dir --osc-dir=$1 --version=$DATA_VERSION $META --flush-size=0
- EXITCODE=$?
- while [[ $EXITCODE -ne 0 ]];
- do
- {
- sleep 60
- ./update_from_dir --osc-dir=$1 --version=$DATA_VERSION $META --flush-size=0
- EXITCODE=$?
- };
- done
- DIFF_COUNT=$(($DIFF_COUNT + 1))
-};
-
-
-update_state()
-{
- get_replicate_filename $TARGET
- TIMESTAMP_LINE=`grep "^timestamp" <$REPLICATE_DIR/$REPLICATE_FILENAME.state.txt`
- while [[ -z $TIMESTAMP_LINE ]]; do
- {
- sleep 5
- TIMESTAMP_LINE=`grep "^timestamp" <$REPLICATE_DIR/$REPLICATE_FILENAME.state.txt`
- }; done
- DATA_VERSION=${TIMESTAMP_LINE:10}
-};
-
-
-echo >>$DB_DIR/apply_osc_to_db.log
-
-mkdir -p $DB_DIR/augmented_diffs/
-DIFF_COUNT=0
-
-# update_state
-
-pushd "$EXEC_DIR"
-
-while [[ true ]]; do
-{
- if [[ $START == "auto" ]]; then
- {
- START=`cat $DB_DIR/replicate_id`
- }; fi
-
- echo "`date -u '+%F %T'`: updating from $START" >>$DB_DIR/apply_osc_to_db.log
-
- TEMP_DIR=`mktemp -d /tmp/osm-3s_update_XXXXXX`
- collect_minute_diffs $TEMP_DIR
-
- if [[ $TARGET -gt $START ]]; then
- {
- echo "`date -u '+%F %T'`: updating to $TARGET" >>$DB_DIR/apply_osc_to_db.log
-
- update_state
- apply_minute_diffs $TEMP_DIR
- echo "$TARGET" >$DB_DIR/replicate_id
-
- echo "`date -u '+%F %T'`: update complete" $TARGET >>$DB_DIR/apply_osc_to_db.log
- };
- else
- {
- sleep 5
- }; fi
-
- rm -f $TEMP_DIR/*
- rmdir $TEMP_DIR
-
- START=$TARGET
-}; done
diff --git a/0.7.54.9/bin/fetch_osc.sh b/0.7.54.9/bin/fetch_osc.sh
deleted file mode 100755
index a9d163f..0000000
--- a/0.7.54.9/bin/fetch_osc.sh
+++ /dev/null
@@ -1,163 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht
-#
-# This file is part of Overpass_API.
-#
-# Overpass_API is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# Overpass_API is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with Overpass_API. If not, see .
-
-if [[ -z $1 ]]; then
-{
- echo Usage: $0 Replicate_id Source_dir Local_dir [Sleep]
- exit 0
-};
-fi
-
-REPLICATE_ID=$1
-SOURCE_DIR=$2
-LOCAL_DIR=$3
-SLEEP_BETWEEN_DLS=15 # How long to sleep between download attempts (sec). Default: 15. See also FAIL_COUNTER_ALERT
-FAIL_COUNTER_ALERT=20 # After how many sleep cycles do we get nervous
-FAIL_COUNTER=0
-FILE_PANIC=
-
-if [[ ! -d $LOCAL_DIR ]];
- then {
- mkdir $LOCAL_DIR
-};
-fi
-
-if [[ $REPLICATE_ID == "auto" ]] ; then
- REPLICATE_ID=`find $LOCAL_DIR -type f -name '*state.txt' -not -size 0 | sort | tail -n 1 | sed -e "s#^$LOCAL_DIR##" -e 's#[./]##g' -e 's#statetxt##' -e 's/^0\+//g'`
-
- EXEC_DIR="`dirname $0`/"
- DB_DIR=`$EXEC_DIR/dispatcher --show-dir`
-
- if [[ ! -d $DB_DIR ]] ; then
- echo "Can't find DB_DIR. Returned value: $DB_DIR"
- exit 1;
- fi
-
- if [[ "x$REPLICATE_ID" == "x" ]] ; then
- REPLICATE_ID=`cat $DB_DIR/replicate_id`
- fi
- if [[ "x$REPLICATE_ID" == "x" ]] ; then
- echo "Could not determine REPLICATE_ID. Exiting"
- exit 1
- fi
-fi
-
-# $1 - remote source
-# $2 - local destination
-fetch_file()
-{
- wget -nv -O "$2" "$1"
-};
-
-retry_fetch_file()
-{
- FILE_PANIC=
- FAIL_COUNTER=0
- if [[ ! -s "$2" ]]; then {
- fetch_file "$1" "$2"
- if [[ "$3" == "gzip" ]]; then {
- gunzip -t <"$2"
- if [[ $? -ne 0 ]]; then {
- rm "$2"
- }; fi
- }; fi
- }; fi
- until [[ -s "$2" || $FAIL_COUNTER -ge $FAIL_COUNTER_ALERT ]]; do {
- FAIL_COUNTER=$(($FAIL_COUNTER+1))
- sleep $SLEEP_BETWEEN_DLS
- fetch_file "$1" "$2"
- if [[ "$3" == "gzip" ]]; then {
- gunzip -t <"$2"
- if [[ $? -ne 0 ]]; then {
- rm "$2"
- }; fi
- }; fi
- }; done
- if [[ $FAIL_COUNTER -ge $FAIL_COUNTER_ALERT ]]; then
- FILE_PANIC="true"
- fi
-};
-
-file_panic()
-{
- echo "fetch_osc()@"`date -u "+%F %T"`": upstream_delay $REPLICATE_ID" >>$LOCAL_DIR/fetch_osc.log
- REPLICATE_ID=$(($REPLICATE_ID - 1))
-
- printf -v TDIGIT3 %03u $(($REPLICATE_ID % 1000))
- ARG=$(($REPLICATE_ID / 1000))
- printf -v TDIGIT2 %03u $(($ARG % 1000))
- ARG=$(($ARG / 1000))
- printf -v TDIGIT1 %03u $ARG
-
- FILE_PANIC="true"
- until [[ ! -n $FILE_PANIC ]]; do {
- retry_fetch_file "$REMOTE_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.new.osc.gz" "gzip"
- }; done
- FILE_PANIC="true"
- until [[ ! -n $FILE_PANIC ]]; do {
- retry_fetch_file "$REMOTE_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.new.state.txt" "text"
- }; done
-
- RES_GZIP=`diff -q "$LOCAL_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.new.osc.gz"`
- RES_TEXT=`diff -q "$LOCAL_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.new.state.txt"`
- if [[ -n $RES_GZIP || -n $RES_TEXT ]]; then
- echo "fetch_osc()@"`date -u "+%F %T"`": file_panic $REPLICATE_ID" >>$LOCAL_DIR/fetch_osc.log
- echo "fetch_osc()@"`date -u "+%F %T"`": $RES_GZIP" >>$LOCAL_DIR/fetch_osc.log
- echo "fetch_osc()@"`date -u "+%F %T"`": $RES_TEXT" >>$LOCAL_DIR/fetch_osc.log
- exit 1
- fi
-
- rm "$LOCAL_PATH/$TDIGIT3.new.osc.gz"
- rm "$LOCAL_PATH/$TDIGIT3.new.state.txt"
-};
-
-fetch_minute_diff()
-{
- printf -v TDIGIT3 %03u $(($REPLICATE_ID % 1000))
- ARG=$(($REPLICATE_ID / 1000))
- printf -v TDIGIT2 %03u $(($ARG % 1000))
- ARG=$(($ARG / 1000))
- printf -v TDIGIT1 %03u $ARG
-
- LOCAL_PATH="$LOCAL_DIR/$TDIGIT1/$TDIGIT2"
- REMOTE_PATH="$SOURCE_DIR/$TDIGIT1/$TDIGIT2"
- mkdir -p "$LOCAL_DIR/$TDIGIT1/$TDIGIT2"
-
- retry_fetch_file "$REMOTE_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.osc.gz" "gzip"
- if [[ -n $FILE_PANIC ]]; then
- file_panic
- fi
- retry_fetch_file "$REMOTE_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.state.txt" "text"
- if [[ -n $FILE_PANIC ]]; then
- file_panic
- fi
-
- TIMESTAMP_LINE=`grep timestamp $LOCAL_DIR/$TDIGIT1/$TDIGIT2/$TDIGIT3.state.txt`
- TIMESTAMP=${TIMESTAMP_LINE:10}
-};
-
-while [[ true ]];
-do
-{
- REPLICATE_ID=$(($REPLICATE_ID + 1))
- fetch_minute_diff
- echo "fetch_osc()@"`date -u "+%F %T"`": new_replicate_diff $REPLICATE_ID $TIMESTAMP" >>$LOCAL_DIR/fetch_osc.log
- sleep 1
-};
-done
diff --git a/0.7.54.9/bin/update_overpass.sh b/0.7.54.9/bin/update_overpass.sh
new file mode 100644
index 0000000..328e375
--- /dev/null
+++ b/0.7.54.9/bin/update_overpass.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env sh
+
+(
+ set -e
+
+ if [ "x$OVERPASS_META" = "xattic" ] ; then
+ META="--keep-attic"
+ elif [ "x$OVERPASS_META" = "xyes" ] ; then
+ META="--meta"
+ else
+ META=""
+ fi
+
+ if [ ! -d /db/diffs ] ; then
+ mkdir /db/diffs
+ fi
+
+ while `true` ; do
+ (
+ if [ ! -e /db/diffs/changes.osm ] ; then
+ ! /app/venv/bin/pyosmium-get-changes --server $OVERPASS_DIFF_URL -o /db/diffs/changes.osm -f /db/replicate_id
+ OSMIUM_STATUS=$?
+ if [ $OSMIUM_STATUS -eq 1 ]; then
+ echo "There are still some updates remainging"
+ fi
+ if [ $OSMIUM_STATUS -eq 2 ]; then
+ echo "Failure downloading updates"
+ exit 0
+ fi
+ else
+ echo "/db/diffs/changes.osm exists. Trying to apply again."
+ fi
+ /app/bin/update_from_dir --osc-dir=/db/diffs/ $META
+ rm /db/diffs/changes.osm
+ )
+ sleep 60
+ done
+) 2>&1 | tee -a /db/changes.log
\ No newline at end of file
diff --git a/0.7.54.9/docker-entrypoint.sh b/0.7.54.9/docker-entrypoint.sh
index 5a43603..cb69627 100644
--- a/0.7.54.9/docker-entrypoint.sh
+++ b/0.7.54.9/docker-entrypoint.sh
@@ -6,13 +6,15 @@ OVERPASS_META=${OVERPASS_META:-no}
OVERPASS_MODE=${OVERPASS_MODE:-clone}
OVERPASS_COMPRESSION=${OVERPASS_COMPRESSION:-gz}
+if [[ "$OVERPASS_META" == "attic" ]] ; then
+ META="--keep-attic"
+else
+ META="--meta"
+fi
+
+
if [ ! -d /db/db ] ; then
if [ "$OVERPASS_MODE" = "clone" ]; then
- if [[ "$OVERPASS_META" == "attic" ]] ; then
- META="--keep-attic"
- else
- META="--meta=$OVERPASS_META"
- fi
mkdir -p /db/db \
&& /app/bin/download_clone.sh --db-dir=/db/db --source=http://dev.overpass-api.de/api_drolbr/ $META \
&& cp -r /app/etc/rules /db/db \
@@ -22,10 +24,21 @@ if [ ! -d /db/db ] ; then
fi
if [ "$OVERPASS_MODE" = "init" ]; then
- lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet; exit" \
- && /app/bin/init_osm3s.sh /db/planet /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION" \
- && echo $OVERPASS_PLANET_SEQUENCE_ID > /db/db/replicate_id \
- && rm /db/planet \
+ lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet.osm.bz2; exit" \
+ && /app/bin/init_osm3s.sh /db/planet.osm.bz2 /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION --map-compression-method=$OVERPASS_COMPRESSION"\
+ && echo "Database created. Now updating it." && (
+ ! /app/venv/bin/pyosmium-get-changes -O /db/planet.osm.bz2 --server $OVERPASS_DIFF_URL -o /db/changes.osm -f /db/replicate_id
+ OSMIUM_STATUS=$?
+ if [ $OSMIUM_STATUS -eq 1 ]; then
+ echo "There are still some updates remainging"
+ fi
+ if [ $OSMIUM_STATUS -eq 2 ]; then
+ echo "Failure downloading updates"
+ exit 0
+ fi
+ (cat /db/changes.osm | /app/bin/update_database --db-dir=/db/db $META --compression-method=$OVERPASS_COMPRESSION) 2>&1 | tee -a /db/changes.log
+ ) \
+ && rm /db/planet.osm.bz2 /db/changes.osm \
&& cp -r /app/etc/rules /db/db \
&& chown -R overpass:overpass /db \
&& echo "Overpass ready, you can start your container with docker start"
diff --git a/0.7.54.9/etc/nginx-overpass.conf b/0.7.54.9/etc/nginx-overpass.conf
index c019615..a19dde4 100644
--- a/0.7.54.9/etc/nginx-overpass.conf
+++ b/0.7.54.9/etc/nginx-overpass.conf
@@ -25,7 +25,7 @@ http {
sendfile on;
#tcp_nopush on;
- keepalive_timeout 65;
+ keepalive_timeout 1000s;
#gzip on;
@@ -55,6 +55,7 @@ http {
fastcgi_param SCRIPT_FILENAME /app/$fastcgi_script_name;
fastcgi_param PATH_INFO $fastcgi_path_info;
fastcgi_pass unix:/nginx/fcgiwrap.socket;
+ fastcgi_read_timeout 1000s;
}
location /api/ {
diff --git a/0.7.54.9/etc/supervisord.conf b/0.7.54.9/etc/supervisord.conf
index 6c4623b..fc67b74 100644
--- a/0.7.54.9/etc/supervisord.conf
+++ b/0.7.54.9/etc/supervisord.conf
@@ -17,18 +17,12 @@ redirect_stderr=true
user=nginx
priority=3
-[program:fetch_diff]
-command=/app/bin/fetch_osc.sh auto %(ENV_OVERPASS_DIFF_URL)s /db/diffs
+[program:update_overpass]
+command=/app/bin/update_overpass.sh
user=overpass
redirect_stderr=true
priority=5
-[program:apply_diff]
-command=/app/bin/apply_osc_to_db.sh /db/diffs/ auto --meta=%(ENV_OVERPASS_META)s
-redirect_stderr=true
-user=overpass
-priority=4
-
[program:dispatcher_areas]
command=nice /app/bin/dispatcher --areas --db-dir="/db/db"
user=overpass
diff --git a/0.7.54.9/requirements.txt b/0.7.54.9/requirements.txt
new file mode 100644
index 0000000..8314012
--- /dev/null
+++ b/0.7.54.9/requirements.txt
@@ -0,0 +1 @@
+osmium
diff --git a/0.7.54/Dockerfile b/0.7.54/Dockerfile
index b6f8025..85cca11 100644
--- a/0.7.54/Dockerfile
+++ b/0.7.54/Dockerfile
@@ -1,27 +1,34 @@
-FROM nginx:stable
+FROM nginx:1.14
RUN addgroup overpass && adduser --home /db --disabled-password --gecos overpass --ingroup overpass overpass
+ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.54.tar.gz /app/src.tar.gz
+
RUN apt-get update \
&& apt-get install --no-install-recommends --no-install-suggests -y \
autoconf \
automake \
+ bash \
+ bzip2 \
ca-certificates \
expat \
- libexpat1-dev \
+ fcgiwrap \
g++ \
+ lftp \
+ libexpat1-dev \
liblz4-1 \
liblz4-dev \
libtool \
m4 \
make \
+ osmium-tool \
+ python3 \
+ python3-venv \
+ supervisor \
+ wget \
zlib1g \
- zlib1g-dev
-
-
-ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.54.tar.gz /app/src.tar.gz
-
-RUN mkdir -p /app/src \
+ zlib1g-dev \
+ && mkdir -p /app/src \
&& cd /app/src \
&& tar -x -z --strip-components 1 -f ../src.tar.gz \
&& autoscan \
@@ -45,26 +52,28 @@ RUN mkdir -p /app/src \
&& apt-get autoremove -y \
&& mkdir -p /db/diffs /app/etc \
&& cp -r /app/src/rules /app/etc/rules \
- && rm -rf /app/src /app/src.tar.gz
+ && rm -rf /app/src /app/src.tar.gz \
+ && rm -rf /var/lib/apt/lists/*
-RUN apt-get install --no-install-recommends --no-install-suggests -y \
- supervisor \
- bash \
- lftp \
- wget \
- fcgiwrap \
- bzip2
+COPY requirements.txt /app/
+RUN python3 -m venv /app/venv \
+ && /app/venv/bin/pip install -r /app/requirements.txt
RUN mkdir /nginx && chown nginx:nginx /nginx && chown -R overpass:overpass /db
+
COPY etc/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+
COPY etc/nginx-overpass.conf /etc/nginx/nginx.conf
-COPY bin/apply_osc_to_db.sh bin/fetch_osc.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/
+
+COPY bin/update_overpass.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/
+
COPY docker-entrypoint.sh /app/
-RUN chmod a+x /app/docker-entrypoint.sh /app/bin/apply_osc_to_db.sh /app/bin/fetch_osc.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh
-VOLUME /db
+
+RUN chmod a+x /app/docker-entrypoint.sh /app/bin/update_overpass.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh
ENV OVERPASS_RULES_LOAD 1
+
EXPOSE 80
# CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"]
CMD ["/app/docker-entrypoint.sh"]
diff --git a/0.7.54/bin/apply_osc_to_db.sh b/0.7.54/bin/apply_osc_to_db.sh
deleted file mode 100755
index 94d1879..0000000
--- a/0.7.54/bin/apply_osc_to_db.sh
+++ /dev/null
@@ -1,163 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht
-#
-# This file is part of Overpass_API.
-#
-# Overpass_API is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# Overpass_API is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with Overpass_API. If not, see .
-
-if [[ -z $3 ]]; then
-{
- echo "Usage: $0 replicate_dir start_id --meta=(attic|yes|no)"
- exit 0
-}; fi
-
-EXEC_DIR="`dirname $0`/"
-if [[ ! ${EXEC_DIR:0:1} == "/" ]]; then
-{
- EXEC_DIR="`pwd`/$EXEC_DIR"
-}; fi
-
-DB_DIR=`$EXEC_DIR/dispatcher --show-dir`
-
-if [[ ! -d $DB_DIR ]] ; then
- echo "Can't find DB_DIR. Returned value: $DB_DIR"
- exit 1;
-fi
-
-REPLICATE_DIR="$1"
-if [[ ! ${REPLICATE_DIR:0:1} == "/" ]]; then
-{
- REPLICATE_DIR="`pwd`/$REPLICATE_DIR"
-}; fi
-
-START=$2
-META=
-
-if [[ $3 == "--meta=attic" ]]; then
- META="--keep-attic"
-elif [[ $3 == "--meta=yes" || $3 == "--meta" ]]; then
- META="--meta"
-elif [[ $3 == "--meta=no" ]]; then
- META=
-else
-{
- echo "You must specify --meta=yes or --meta=no"
- exit 0
-}; fi
-
-PRODUCE_DIFF=
-
-get_replicate_filename()
-{
- printf -v TDIGIT3 %03u $(($1 % 1000))
- ARG=$(($1 / 1000))
- printf -v TDIGIT2 %03u $(($ARG % 1000))
- ARG=$(($ARG / 1000))
- printf -v TDIGIT1 %03u $ARG
- REPLICATE_TRUNK_DIR=$TDIGIT1/$TDIGIT2/
- REPLICATE_FILENAME=$TDIGIT1/$TDIGIT2/$TDIGIT3
-};
-
-
-collect_minute_diffs()
-{
- TEMP_DIR=$1
- TARGET=$(($START + 1))
-
- get_replicate_filename $TARGET
-
- while [[ ( -s $REPLICATE_DIR/$REPLICATE_FILENAME.state.txt ) && ( $(($START + 1440)) -ge $(($TARGET)) ) && ( `du -m $TEMP_DIR | awk '{ print $1; }'` -le 512 ) ]];
- do
- {
- printf -v TARGET_FILE %09u $TARGET
- gunzip <$REPLICATE_DIR/$REPLICATE_FILENAME.osc.gz >$TEMP_DIR/$TARGET_FILE.osc
- TARGET=$(($TARGET + 1))
- get_replicate_filename $TARGET
- };
- done
- TARGET=$(($TARGET - 1))
-};
-
-
-apply_minute_diffs()
-{
- ./update_from_dir --osc-dir=$1 --version=$DATA_VERSION $META --flush-size=0
- EXITCODE=$?
- while [[ $EXITCODE -ne 0 ]];
- do
- {
- sleep 60
- ./update_from_dir --osc-dir=$1 --version=$DATA_VERSION $META --flush-size=0
- EXITCODE=$?
- };
- done
- DIFF_COUNT=$(($DIFF_COUNT + 1))
-};
-
-
-update_state()
-{
- get_replicate_filename $TARGET
- TIMESTAMP_LINE=`grep "^timestamp" <$REPLICATE_DIR/$REPLICATE_FILENAME.state.txt`
- while [[ -z $TIMESTAMP_LINE ]]; do
- {
- sleep 5
- TIMESTAMP_LINE=`grep "^timestamp" <$REPLICATE_DIR/$REPLICATE_FILENAME.state.txt`
- }; done
- DATA_VERSION=${TIMESTAMP_LINE:10}
-};
-
-
-echo >>$DB_DIR/apply_osc_to_db.log
-
-mkdir -p $DB_DIR/augmented_diffs/
-DIFF_COUNT=0
-
-# update_state
-
-pushd "$EXEC_DIR"
-
-while [[ true ]]; do
-{
- if [[ $START == "auto" ]]; then
- {
- START=`cat $DB_DIR/replicate_id`
- }; fi
-
- echo "`date -u '+%F %T'`: updating from $START" >>$DB_DIR/apply_osc_to_db.log
-
- TEMP_DIR=`mktemp -d /tmp/osm-3s_update_XXXXXX`
- collect_minute_diffs $TEMP_DIR
-
- if [[ $TARGET -gt $START ]]; then
- {
- echo "`date -u '+%F %T'`: updating to $TARGET" >>$DB_DIR/apply_osc_to_db.log
-
- update_state
- apply_minute_diffs $TEMP_DIR
- echo "$TARGET" >$DB_DIR/replicate_id
-
- echo "`date -u '+%F %T'`: update complete" $TARGET >>$DB_DIR/apply_osc_to_db.log
- };
- else
- {
- sleep 5
- }; fi
-
- rm -f $TEMP_DIR/*
- rmdir $TEMP_DIR
-
- START=$TARGET
-}; done
diff --git a/0.7.54/bin/fetch_osc.sh b/0.7.54/bin/fetch_osc.sh
deleted file mode 100755
index a9d163f..0000000
--- a/0.7.54/bin/fetch_osc.sh
+++ /dev/null
@@ -1,163 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht
-#
-# This file is part of Overpass_API.
-#
-# Overpass_API is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# Overpass_API is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with Overpass_API. If not, see .
-
-if [[ -z $1 ]]; then
-{
- echo Usage: $0 Replicate_id Source_dir Local_dir [Sleep]
- exit 0
-};
-fi
-
-REPLICATE_ID=$1
-SOURCE_DIR=$2
-LOCAL_DIR=$3
-SLEEP_BETWEEN_DLS=15 # How long to sleep between download attempts (sec). Default: 15. See also FAIL_COUNTER_ALERT
-FAIL_COUNTER_ALERT=20 # After how many sleep cycles do we get nervous
-FAIL_COUNTER=0
-FILE_PANIC=
-
-if [[ ! -d $LOCAL_DIR ]];
- then {
- mkdir $LOCAL_DIR
-};
-fi
-
-if [[ $REPLICATE_ID == "auto" ]] ; then
- REPLICATE_ID=`find $LOCAL_DIR -type f -name '*state.txt' -not -size 0 | sort | tail -n 1 | sed -e "s#^$LOCAL_DIR##" -e 's#[./]##g' -e 's#statetxt##' -e 's/^0\+//g'`
-
- EXEC_DIR="`dirname $0`/"
- DB_DIR=`$EXEC_DIR/dispatcher --show-dir`
-
- if [[ ! -d $DB_DIR ]] ; then
- echo "Can't find DB_DIR. Returned value: $DB_DIR"
- exit 1;
- fi
-
- if [[ "x$REPLICATE_ID" == "x" ]] ; then
- REPLICATE_ID=`cat $DB_DIR/replicate_id`
- fi
- if [[ "x$REPLICATE_ID" == "x" ]] ; then
- echo "Could not determine REPLICATE_ID. Exiting"
- exit 1
- fi
-fi
-
-# $1 - remote source
-# $2 - local destination
-fetch_file()
-{
- wget -nv -O "$2" "$1"
-};
-
-retry_fetch_file()
-{
- FILE_PANIC=
- FAIL_COUNTER=0
- if [[ ! -s "$2" ]]; then {
- fetch_file "$1" "$2"
- if [[ "$3" == "gzip" ]]; then {
- gunzip -t <"$2"
- if [[ $? -ne 0 ]]; then {
- rm "$2"
- }; fi
- }; fi
- }; fi
- until [[ -s "$2" || $FAIL_COUNTER -ge $FAIL_COUNTER_ALERT ]]; do {
- FAIL_COUNTER=$(($FAIL_COUNTER+1))
- sleep $SLEEP_BETWEEN_DLS
- fetch_file "$1" "$2"
- if [[ "$3" == "gzip" ]]; then {
- gunzip -t <"$2"
- if [[ $? -ne 0 ]]; then {
- rm "$2"
- }; fi
- }; fi
- }; done
- if [[ $FAIL_COUNTER -ge $FAIL_COUNTER_ALERT ]]; then
- FILE_PANIC="true"
- fi
-};
-
-file_panic()
-{
- echo "fetch_osc()@"`date -u "+%F %T"`": upstream_delay $REPLICATE_ID" >>$LOCAL_DIR/fetch_osc.log
- REPLICATE_ID=$(($REPLICATE_ID - 1))
-
- printf -v TDIGIT3 %03u $(($REPLICATE_ID % 1000))
- ARG=$(($REPLICATE_ID / 1000))
- printf -v TDIGIT2 %03u $(($ARG % 1000))
- ARG=$(($ARG / 1000))
- printf -v TDIGIT1 %03u $ARG
-
- FILE_PANIC="true"
- until [[ ! -n $FILE_PANIC ]]; do {
- retry_fetch_file "$REMOTE_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.new.osc.gz" "gzip"
- }; done
- FILE_PANIC="true"
- until [[ ! -n $FILE_PANIC ]]; do {
- retry_fetch_file "$REMOTE_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.new.state.txt" "text"
- }; done
-
- RES_GZIP=`diff -q "$LOCAL_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.new.osc.gz"`
- RES_TEXT=`diff -q "$LOCAL_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.new.state.txt"`
- if [[ -n $RES_GZIP || -n $RES_TEXT ]]; then
- echo "fetch_osc()@"`date -u "+%F %T"`": file_panic $REPLICATE_ID" >>$LOCAL_DIR/fetch_osc.log
- echo "fetch_osc()@"`date -u "+%F %T"`": $RES_GZIP" >>$LOCAL_DIR/fetch_osc.log
- echo "fetch_osc()@"`date -u "+%F %T"`": $RES_TEXT" >>$LOCAL_DIR/fetch_osc.log
- exit 1
- fi
-
- rm "$LOCAL_PATH/$TDIGIT3.new.osc.gz"
- rm "$LOCAL_PATH/$TDIGIT3.new.state.txt"
-};
-
-fetch_minute_diff()
-{
- printf -v TDIGIT3 %03u $(($REPLICATE_ID % 1000))
- ARG=$(($REPLICATE_ID / 1000))
- printf -v TDIGIT2 %03u $(($ARG % 1000))
- ARG=$(($ARG / 1000))
- printf -v TDIGIT1 %03u $ARG
-
- LOCAL_PATH="$LOCAL_DIR/$TDIGIT1/$TDIGIT2"
- REMOTE_PATH="$SOURCE_DIR/$TDIGIT1/$TDIGIT2"
- mkdir -p "$LOCAL_DIR/$TDIGIT1/$TDIGIT2"
-
- retry_fetch_file "$REMOTE_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.osc.gz" "gzip"
- if [[ -n $FILE_PANIC ]]; then
- file_panic
- fi
- retry_fetch_file "$REMOTE_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.state.txt" "text"
- if [[ -n $FILE_PANIC ]]; then
- file_panic
- fi
-
- TIMESTAMP_LINE=`grep timestamp $LOCAL_DIR/$TDIGIT1/$TDIGIT2/$TDIGIT3.state.txt`
- TIMESTAMP=${TIMESTAMP_LINE:10}
-};
-
-while [[ true ]];
-do
-{
- REPLICATE_ID=$(($REPLICATE_ID + 1))
- fetch_minute_diff
- echo "fetch_osc()@"`date -u "+%F %T"`": new_replicate_diff $REPLICATE_ID $TIMESTAMP" >>$LOCAL_DIR/fetch_osc.log
- sleep 1
-};
-done
diff --git a/0.7.54/bin/update_overpass.sh b/0.7.54/bin/update_overpass.sh
new file mode 100644
index 0000000..328e375
--- /dev/null
+++ b/0.7.54/bin/update_overpass.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env sh
+
+(
+ set -e
+
+ if [ "x$OVERPASS_META" = "xattic" ] ; then
+ META="--keep-attic"
+ elif [ "x$OVERPASS_META" = "xyes" ] ; then
+ META="--meta"
+ else
+ META=""
+ fi
+
+ if [ ! -d /db/diffs ] ; then
+ mkdir /db/diffs
+ fi
+
+ while `true` ; do
+ (
+ if [ ! -e /db/diffs/changes.osm ] ; then
+ ! /app/venv/bin/pyosmium-get-changes --server $OVERPASS_DIFF_URL -o /db/diffs/changes.osm -f /db/replicate_id
+ OSMIUM_STATUS=$?
+ if [ $OSMIUM_STATUS -eq 1 ]; then
+ echo "There are still some updates remainging"
+ fi
+ if [ $OSMIUM_STATUS -eq 2 ]; then
+ echo "Failure downloading updates"
+ exit 0
+ fi
+ else
+ echo "/db/diffs/changes.osm exists. Trying to apply again."
+ fi
+ /app/bin/update_from_dir --osc-dir=/db/diffs/ $META
+ rm /db/diffs/changes.osm
+ )
+ sleep 60
+ done
+) 2>&1 | tee -a /db/changes.log
\ No newline at end of file
diff --git a/0.7.54/docker-entrypoint.sh b/0.7.54/docker-entrypoint.sh
index 5a43603..cb69627 100644
--- a/0.7.54/docker-entrypoint.sh
+++ b/0.7.54/docker-entrypoint.sh
@@ -6,13 +6,15 @@ OVERPASS_META=${OVERPASS_META:-no}
OVERPASS_MODE=${OVERPASS_MODE:-clone}
OVERPASS_COMPRESSION=${OVERPASS_COMPRESSION:-gz}
+if [[ "$OVERPASS_META" == "attic" ]] ; then
+ META="--keep-attic"
+else
+ META="--meta"
+fi
+
+
if [ ! -d /db/db ] ; then
if [ "$OVERPASS_MODE" = "clone" ]; then
- if [[ "$OVERPASS_META" == "attic" ]] ; then
- META="--keep-attic"
- else
- META="--meta=$OVERPASS_META"
- fi
mkdir -p /db/db \
&& /app/bin/download_clone.sh --db-dir=/db/db --source=http://dev.overpass-api.de/api_drolbr/ $META \
&& cp -r /app/etc/rules /db/db \
@@ -22,10 +24,21 @@ if [ ! -d /db/db ] ; then
fi
if [ "$OVERPASS_MODE" = "init" ]; then
- lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet; exit" \
- && /app/bin/init_osm3s.sh /db/planet /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION" \
- && echo $OVERPASS_PLANET_SEQUENCE_ID > /db/db/replicate_id \
- && rm /db/planet \
+ lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet.osm.bz2; exit" \
+ && /app/bin/init_osm3s.sh /db/planet.osm.bz2 /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION --map-compression-method=$OVERPASS_COMPRESSION"\
+ && echo "Database created. Now updating it." && (
+ ! /app/venv/bin/pyosmium-get-changes -O /db/planet.osm.bz2 --server $OVERPASS_DIFF_URL -o /db/changes.osm -f /db/replicate_id
+ OSMIUM_STATUS=$?
+ if [ $OSMIUM_STATUS -eq 1 ]; then
+ echo "There are still some updates remainging"
+ fi
+ if [ $OSMIUM_STATUS -eq 2 ]; then
+ echo "Failure downloading updates"
+ exit 0
+ fi
+ (cat /db/changes.osm | /app/bin/update_database --db-dir=/db/db $META --compression-method=$OVERPASS_COMPRESSION) 2>&1 | tee -a /db/changes.log
+ ) \
+ && rm /db/planet.osm.bz2 /db/changes.osm \
&& cp -r /app/etc/rules /db/db \
&& chown -R overpass:overpass /db \
&& echo "Overpass ready, you can start your container with docker start"
diff --git a/0.7.54/etc/nginx-overpass.conf b/0.7.54/etc/nginx-overpass.conf
index c019615..a19dde4 100644
--- a/0.7.54/etc/nginx-overpass.conf
+++ b/0.7.54/etc/nginx-overpass.conf
@@ -25,7 +25,7 @@ http {
sendfile on;
#tcp_nopush on;
- keepalive_timeout 65;
+ keepalive_timeout 1000s;
#gzip on;
@@ -55,6 +55,7 @@ http {
fastcgi_param SCRIPT_FILENAME /app/$fastcgi_script_name;
fastcgi_param PATH_INFO $fastcgi_path_info;
fastcgi_pass unix:/nginx/fcgiwrap.socket;
+ fastcgi_read_timeout 1000s;
}
location /api/ {
diff --git a/0.7.54/etc/supervisord.conf b/0.7.54/etc/supervisord.conf
index 6c4623b..fc67b74 100644
--- a/0.7.54/etc/supervisord.conf
+++ b/0.7.54/etc/supervisord.conf
@@ -17,18 +17,12 @@ redirect_stderr=true
user=nginx
priority=3
-[program:fetch_diff]
-command=/app/bin/fetch_osc.sh auto %(ENV_OVERPASS_DIFF_URL)s /db/diffs
+[program:update_overpass]
+command=/app/bin/update_overpass.sh
user=overpass
redirect_stderr=true
priority=5
-[program:apply_diff]
-command=/app/bin/apply_osc_to_db.sh /db/diffs/ auto --meta=%(ENV_OVERPASS_META)s
-redirect_stderr=true
-user=overpass
-priority=4
-
[program:dispatcher_areas]
command=nice /app/bin/dispatcher --areas --db-dir="/db/db"
user=overpass
diff --git a/0.7.54/requirements.txt b/0.7.54/requirements.txt
new file mode 100644
index 0000000..8314012
--- /dev/null
+++ b/0.7.54/requirements.txt
@@ -0,0 +1 @@
+osmium
diff --git a/0.7.51/Dockerfile b/0.7.55.1/Dockerfile
similarity index 71%
rename from 0.7.51/Dockerfile
rename to 0.7.55.1/Dockerfile
index bc4f5b5..3d01769 100644
--- a/0.7.51/Dockerfile
+++ b/0.7.55.1/Dockerfile
@@ -1,30 +1,36 @@
-FROM nginx:stable
+FROM nginx:1.14
RUN addgroup overpass && adduser --home /db --disabled-password --gecos overpass --ingroup overpass overpass
+ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.55.1.tar.gz /app/src.tar.gz
+
RUN apt-get update \
&& apt-get install --no-install-recommends --no-install-suggests -y \
autoconf \
automake \
+ bash \
+ bzip2 \
ca-certificates \
expat \
- libexpat1-dev \
+ fcgiwrap \
g++ \
+ lftp \
+ libexpat1-dev \
liblz4-1 \
liblz4-dev \
libtool \
m4 \
make \
+ osmium-tool \
+ python3 \
+ python3-venv \
+ supervisor \
+ wget \
zlib1g \
- zlib1g-dev
-
-
-ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.51.tar.gz /app/src.tar.gz
-
-RUN mkdir -p /app/src \
+ zlib1g-dev \
+ && mkdir -p /app/src \
&& cd /app/src \
&& tar -x -z --strip-components 1 -f ../src.tar.gz \
- && cd src \
&& autoscan \
&& aclocal \
&& autoheader \
@@ -46,26 +52,28 @@ RUN mkdir -p /app/src \
&& apt-get autoremove -y \
&& mkdir -p /db/diffs /app/etc \
&& cp -r /app/src/rules /app/etc/rules \
- && rm -rf /app/src /app/src.tar.gz
+ && rm -rf /app/src /app/src.tar.gz \
+ && rm -rf /var/lib/apt/lists/*
-RUN apt-get install --no-install-recommends --no-install-suggests -y \
- supervisor \
- bash \
- lftp \
- wget \
- fcgiwrap \
- bzip2
+COPY requirements.txt /app/
+RUN python3 -m venv /app/venv \
+ && /app/venv/bin/pip install -r /app/requirements.txt
RUN mkdir /nginx && chown nginx:nginx /nginx && chown -R overpass:overpass /db
+
COPY etc/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+
COPY etc/nginx-overpass.conf /etc/nginx/nginx.conf
-COPY bin/apply_osc_to_db.sh bin/fetch_osc.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/
+
+COPY bin/update_overpass.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/
+
COPY docker-entrypoint.sh /app/
-RUN chmod a+x /app/docker-entrypoint.sh /app/bin/apply_osc_to_db.sh /app/bin/fetch_osc.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh
-VOLUME /db
+
+RUN chmod a+x /app/docker-entrypoint.sh /app/bin/update_overpass.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh
ENV OVERPASS_RULES_LOAD 1
+
EXPOSE 80
# CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"]
CMD ["/app/docker-entrypoint.sh"]
diff --git a/0.7.51/bin/dispatcher_start.sh b/0.7.55.1/bin/dispatcher_start.sh
old mode 100755
new mode 100644
similarity index 100%
rename from 0.7.51/bin/dispatcher_start.sh
rename to 0.7.55.1/bin/dispatcher_start.sh
diff --git a/0.7.51/bin/rules_loop.sh b/0.7.55.1/bin/rules_loop.sh
old mode 100755
new mode 100644
similarity index 100%
rename from 0.7.51/bin/rules_loop.sh
rename to 0.7.55.1/bin/rules_loop.sh
diff --git a/0.7.55.1/bin/update_overpass.sh b/0.7.55.1/bin/update_overpass.sh
new file mode 100644
index 0000000..328e375
--- /dev/null
+++ b/0.7.55.1/bin/update_overpass.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env sh
+
+(
+ set -e
+
+ if [ "x$OVERPASS_META" = "xattic" ] ; then
+ META="--keep-attic"
+ elif [ "x$OVERPASS_META" = "xyes" ] ; then
+ META="--meta"
+ else
+ META=""
+ fi
+
+ if [ ! -d /db/diffs ] ; then
+ mkdir /db/diffs
+ fi
+
+ while `true` ; do
+ (
+ if [ ! -e /db/diffs/changes.osm ] ; then
+ ! /app/venv/bin/pyosmium-get-changes --server $OVERPASS_DIFF_URL -o /db/diffs/changes.osm -f /db/replicate_id
+ OSMIUM_STATUS=$?
+ if [ $OSMIUM_STATUS -eq 1 ]; then
+ echo "There are still some updates remainging"
+ fi
+ if [ $OSMIUM_STATUS -eq 2 ]; then
+ echo "Failure downloading updates"
+ exit 0
+ fi
+ else
+ echo "/db/diffs/changes.osm exists. Trying to apply again."
+ fi
+ /app/bin/update_from_dir --osc-dir=/db/diffs/ $META
+ rm /db/diffs/changes.osm
+ )
+ sleep 60
+ done
+) 2>&1 | tee -a /db/changes.log
\ No newline at end of file
diff --git a/0.7.55.1/docker-entrypoint.sh b/0.7.55.1/docker-entrypoint.sh
new file mode 100644
index 0000000..cb69627
--- /dev/null
+++ b/0.7.55.1/docker-entrypoint.sh
@@ -0,0 +1,49 @@
+#!/bin/bash
+
+set -eo pipefail
+shopt -s nullglob
+OVERPASS_META=${OVERPASS_META:-no}
+OVERPASS_MODE=${OVERPASS_MODE:-clone}
+OVERPASS_COMPRESSION=${OVERPASS_COMPRESSION:-gz}
+
+if [[ "$OVERPASS_META" == "attic" ]] ; then
+ META="--keep-attic"
+else
+ META="--meta"
+fi
+
+
+if [ ! -d /db/db ] ; then
+ if [ "$OVERPASS_MODE" = "clone" ]; then
+ mkdir -p /db/db \
+ && /app/bin/download_clone.sh --db-dir=/db/db --source=http://dev.overpass-api.de/api_drolbr/ $META \
+ && cp -r /app/etc/rules /db/db \
+ && chown -R overpass:overpass /db \
+ && echo "Overpass ready, you can start your container with docker start"
+ exit
+ fi
+
+ if [ "$OVERPASS_MODE" = "init" ]; then
+ lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet.osm.bz2; exit" \
+ && /app/bin/init_osm3s.sh /db/planet.osm.bz2 /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION --map-compression-method=$OVERPASS_COMPRESSION"\
+ && echo "Database created. Now updating it." && (
+ ! /app/venv/bin/pyosmium-get-changes -O /db/planet.osm.bz2 --server $OVERPASS_DIFF_URL -o /db/changes.osm -f /db/replicate_id
+ OSMIUM_STATUS=$?
+ if [ $OSMIUM_STATUS -eq 1 ]; then
+ echo "There are still some updates remainging"
+ fi
+ if [ $OSMIUM_STATUS -eq 2 ]; then
+ echo "Failure downloading updates"
+ exit 0
+ fi
+ (cat /db/changes.osm | /app/bin/update_database --db-dir=/db/db $META --compression-method=$OVERPASS_COMPRESSION) 2>&1 | tee -a /db/changes.log
+ ) \
+ && rm /db/planet.osm.bz2 /db/changes.osm \
+ && cp -r /app/etc/rules /db/db \
+ && chown -R overpass:overpass /db \
+ && echo "Overpass ready, you can start your container with docker start"
+ exit
+ fi
+fi
+
+exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf
diff --git a/0.7.54.11/etc/nginx-overpass.conf b/0.7.55.1/etc/nginx-overpass.conf
similarity index 95%
rename from 0.7.54.11/etc/nginx-overpass.conf
rename to 0.7.55.1/etc/nginx-overpass.conf
index c019615..a19dde4 100644
--- a/0.7.54.11/etc/nginx-overpass.conf
+++ b/0.7.55.1/etc/nginx-overpass.conf
@@ -25,7 +25,7 @@ http {
sendfile on;
#tcp_nopush on;
- keepalive_timeout 65;
+ keepalive_timeout 1000s;
#gzip on;
@@ -55,6 +55,7 @@ http {
fastcgi_param SCRIPT_FILENAME /app/$fastcgi_script_name;
fastcgi_param PATH_INFO $fastcgi_path_info;
fastcgi_pass unix:/nginx/fcgiwrap.socket;
+ fastcgi_read_timeout 1000s;
}
location /api/ {
diff --git a/0.7.51/etc/supervisord.conf b/0.7.55.1/etc/supervisord.conf
similarity index 74%
rename from 0.7.51/etc/supervisord.conf
rename to 0.7.55.1/etc/supervisord.conf
index 6c4623b..fc67b74 100644
--- a/0.7.51/etc/supervisord.conf
+++ b/0.7.55.1/etc/supervisord.conf
@@ -17,18 +17,12 @@ redirect_stderr=true
user=nginx
priority=3
-[program:fetch_diff]
-command=/app/bin/fetch_osc.sh auto %(ENV_OVERPASS_DIFF_URL)s /db/diffs
+[program:update_overpass]
+command=/app/bin/update_overpass.sh
user=overpass
redirect_stderr=true
priority=5
-[program:apply_diff]
-command=/app/bin/apply_osc_to_db.sh /db/diffs/ auto --meta=%(ENV_OVERPASS_META)s
-redirect_stderr=true
-user=overpass
-priority=4
-
[program:dispatcher_areas]
command=nice /app/bin/dispatcher --areas --db-dir="/db/db"
user=overpass
diff --git a/0.7.55.1/requirements.txt b/0.7.55.1/requirements.txt
new file mode 100644
index 0000000..8314012
--- /dev/null
+++ b/0.7.55.1/requirements.txt
@@ -0,0 +1 @@
+osmium
diff --git a/0.7.54.11/Dockerfile b/0.7.55.2/Dockerfile
similarity index 71%
rename from 0.7.54.11/Dockerfile
rename to 0.7.55.2/Dockerfile
index 39ad38e..3da8118 100644
--- a/0.7.54.11/Dockerfile
+++ b/0.7.55.2/Dockerfile
@@ -1,27 +1,34 @@
-FROM nginx:stable
+FROM nginx:1.14
RUN addgroup overpass && adduser --home /db --disabled-password --gecos overpass --ingroup overpass overpass
+ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.55.2.tar.gz /app/src.tar.gz
+
RUN apt-get update \
&& apt-get install --no-install-recommends --no-install-suggests -y \
autoconf \
automake \
+ bash \
+ bzip2 \
ca-certificates \
expat \
- libexpat1-dev \
+ fcgiwrap \
g++ \
+ lftp \
+ libexpat1-dev \
liblz4-1 \
liblz4-dev \
libtool \
m4 \
make \
+ osmium-tool \
+ python3 \
+ python3-venv \
+ supervisor \
+ wget \
zlib1g \
- zlib1g-dev
-
-
-ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.54.11.tar.gz /app/src.tar.gz
-
-RUN mkdir -p /app/src \
+ zlib1g-dev \
+ && mkdir -p /app/src \
&& cd /app/src \
&& tar -x -z --strip-components 1 -f ../src.tar.gz \
&& autoscan \
@@ -45,26 +52,28 @@ RUN mkdir -p /app/src \
&& apt-get autoremove -y \
&& mkdir -p /db/diffs /app/etc \
&& cp -r /app/src/rules /app/etc/rules \
- && rm -rf /app/src /app/src.tar.gz
+ && rm -rf /app/src /app/src.tar.gz \
+ && rm -rf /var/lib/apt/lists/*
-RUN apt-get install --no-install-recommends --no-install-suggests -y \
- supervisor \
- bash \
- lftp \
- wget \
- fcgiwrap \
- bzip2
+COPY requirements.txt /app/
+RUN python3 -m venv /app/venv \
+ && /app/venv/bin/pip install -r /app/requirements.txt
RUN mkdir /nginx && chown nginx:nginx /nginx && chown -R overpass:overpass /db
+
COPY etc/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+
COPY etc/nginx-overpass.conf /etc/nginx/nginx.conf
-COPY bin/apply_osc_to_db.sh bin/fetch_osc.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/
+
+COPY bin/update_overpass.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/
+
COPY docker-entrypoint.sh /app/
-RUN chmod a+x /app/docker-entrypoint.sh /app/bin/apply_osc_to_db.sh /app/bin/fetch_osc.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh
-VOLUME /db
+
+RUN chmod a+x /app/docker-entrypoint.sh /app/bin/update_overpass.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh
ENV OVERPASS_RULES_LOAD 1
+
EXPOSE 80
# CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"]
CMD ["/app/docker-entrypoint.sh"]
diff --git a/0.7.54.11/bin/dispatcher_start.sh b/0.7.55.2/bin/dispatcher_start.sh
old mode 100755
new mode 100644
similarity index 100%
rename from 0.7.54.11/bin/dispatcher_start.sh
rename to 0.7.55.2/bin/dispatcher_start.sh
diff --git a/0.7.54.11/bin/rules_loop.sh b/0.7.55.2/bin/rules_loop.sh
old mode 100755
new mode 100644
similarity index 100%
rename from 0.7.54.11/bin/rules_loop.sh
rename to 0.7.55.2/bin/rules_loop.sh
diff --git a/0.7.55.2/bin/update_overpass.sh b/0.7.55.2/bin/update_overpass.sh
new file mode 100644
index 0000000..328e375
--- /dev/null
+++ b/0.7.55.2/bin/update_overpass.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env sh
+
+(
+ set -e
+
+ if [ "x$OVERPASS_META" = "xattic" ] ; then
+ META="--keep-attic"
+ elif [ "x$OVERPASS_META" = "xyes" ] ; then
+ META="--meta"
+ else
+ META=""
+ fi
+
+ if [ ! -d /db/diffs ] ; then
+ mkdir /db/diffs
+ fi
+
+ while `true` ; do
+ (
+ if [ ! -e /db/diffs/changes.osm ] ; then
+ ! /app/venv/bin/pyosmium-get-changes --server $OVERPASS_DIFF_URL -o /db/diffs/changes.osm -f /db/replicate_id
+ OSMIUM_STATUS=$?
+ if [ $OSMIUM_STATUS -eq 1 ]; then
+ echo "There are still some updates remainging"
+ fi
+ if [ $OSMIUM_STATUS -eq 2 ]; then
+ echo "Failure downloading updates"
+ exit 0
+ fi
+ else
+ echo "/db/diffs/changes.osm exists. Trying to apply again."
+ fi
+ /app/bin/update_from_dir --osc-dir=/db/diffs/ $META
+ rm /db/diffs/changes.osm
+ )
+ sleep 60
+ done
+) 2>&1 | tee -a /db/changes.log
\ No newline at end of file
diff --git a/0.7.55.2/docker-entrypoint.sh b/0.7.55.2/docker-entrypoint.sh
new file mode 100644
index 0000000..cb69627
--- /dev/null
+++ b/0.7.55.2/docker-entrypoint.sh
@@ -0,0 +1,49 @@
+#!/bin/bash
+
+set -eo pipefail
+shopt -s nullglob
+OVERPASS_META=${OVERPASS_META:-no}
+OVERPASS_MODE=${OVERPASS_MODE:-clone}
+OVERPASS_COMPRESSION=${OVERPASS_COMPRESSION:-gz}
+
+if [[ "$OVERPASS_META" == "attic" ]] ; then
+ META="--keep-attic"
+else
+ META="--meta"
+fi
+
+
+if [ ! -d /db/db ] ; then
+ if [ "$OVERPASS_MODE" = "clone" ]; then
+ mkdir -p /db/db \
+ && /app/bin/download_clone.sh --db-dir=/db/db --source=http://dev.overpass-api.de/api_drolbr/ $META \
+ && cp -r /app/etc/rules /db/db \
+ && chown -R overpass:overpass /db \
+ && echo "Overpass ready, you can start your container with docker start"
+ exit
+ fi
+
+ if [ "$OVERPASS_MODE" = "init" ]; then
+ lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet.osm.bz2; exit" \
+ && /app/bin/init_osm3s.sh /db/planet.osm.bz2 /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION --map-compression-method=$OVERPASS_COMPRESSION"\
+ && echo "Database created. Now updating it." && (
+ ! /app/venv/bin/pyosmium-get-changes -O /db/planet.osm.bz2 --server $OVERPASS_DIFF_URL -o /db/changes.osm -f /db/replicate_id
+ OSMIUM_STATUS=$?
+ if [ $OSMIUM_STATUS -eq 1 ]; then
+ echo "There are still some updates remainging"
+ fi
+ if [ $OSMIUM_STATUS -eq 2 ]; then
+ echo "Failure downloading updates"
+ exit 0
+ fi
+ (cat /db/changes.osm | /app/bin/update_database --db-dir=/db/db $META --compression-method=$OVERPASS_COMPRESSION) 2>&1 | tee -a /db/changes.log
+ ) \
+ && rm /db/planet.osm.bz2 /db/changes.osm \
+ && cp -r /app/etc/rules /db/db \
+ && chown -R overpass:overpass /db \
+ && echo "Overpass ready, you can start your container with docker start"
+ exit
+ fi
+fi
+
+exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf
diff --git a/0.7.51/etc/nginx-overpass.conf b/0.7.55.2/etc/nginx-overpass.conf
similarity index 95%
rename from 0.7.51/etc/nginx-overpass.conf
rename to 0.7.55.2/etc/nginx-overpass.conf
index c019615..a19dde4 100644
--- a/0.7.51/etc/nginx-overpass.conf
+++ b/0.7.55.2/etc/nginx-overpass.conf
@@ -25,7 +25,7 @@ http {
sendfile on;
#tcp_nopush on;
- keepalive_timeout 65;
+ keepalive_timeout 1000s;
#gzip on;
@@ -55,6 +55,7 @@ http {
fastcgi_param SCRIPT_FILENAME /app/$fastcgi_script_name;
fastcgi_param PATH_INFO $fastcgi_path_info;
fastcgi_pass unix:/nginx/fcgiwrap.socket;
+ fastcgi_read_timeout 1000s;
}
location /api/ {
diff --git a/0.7.54.11/etc/supervisord.conf b/0.7.55.2/etc/supervisord.conf
similarity index 74%
rename from 0.7.54.11/etc/supervisord.conf
rename to 0.7.55.2/etc/supervisord.conf
index 6c4623b..fc67b74 100644
--- a/0.7.54.11/etc/supervisord.conf
+++ b/0.7.55.2/etc/supervisord.conf
@@ -17,18 +17,12 @@ redirect_stderr=true
user=nginx
priority=3
-[program:fetch_diff]
-command=/app/bin/fetch_osc.sh auto %(ENV_OVERPASS_DIFF_URL)s /db/diffs
+[program:update_overpass]
+command=/app/bin/update_overpass.sh
user=overpass
redirect_stderr=true
priority=5
-[program:apply_diff]
-command=/app/bin/apply_osc_to_db.sh /db/diffs/ auto --meta=%(ENV_OVERPASS_META)s
-redirect_stderr=true
-user=overpass
-priority=4
-
[program:dispatcher_areas]
command=nice /app/bin/dispatcher --areas --db-dir="/db/db"
user=overpass
diff --git a/0.7.55.2/requirements.txt b/0.7.55.2/requirements.txt
new file mode 100644
index 0000000..8314012
--- /dev/null
+++ b/0.7.55.2/requirements.txt
@@ -0,0 +1 @@
+osmium
diff --git a/0.7.55.3/Dockerfile b/0.7.55.3/Dockerfile
new file mode 100644
index 0000000..7a08746
--- /dev/null
+++ b/0.7.55.3/Dockerfile
@@ -0,0 +1,79 @@
+FROM nginx:1.14
+
+RUN addgroup overpass && adduser --home /db --disabled-password --gecos overpass --ingroup overpass overpass
+
+ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.55.3.tar.gz /app/src.tar.gz
+
+RUN apt-get update \
+ && apt-get install --no-install-recommends --no-install-suggests -y \
+ autoconf \
+ automake \
+ bash \
+ bzip2 \
+ ca-certificates \
+ expat \
+ fcgiwrap \
+ g++ \
+ lftp \
+ libexpat1-dev \
+ liblz4-1 \
+ liblz4-dev \
+ libtool \
+ m4 \
+ make \
+ osmium-tool \
+ python3 \
+ python3-venv \
+ supervisor \
+ wget \
+ zlib1g \
+ zlib1g-dev \
+ && mkdir -p /app/src \
+ && cd /app/src \
+ && tar -x -z --strip-components 1 -f ../src.tar.gz \
+ && autoscan \
+ && aclocal \
+ && autoheader \
+ && libtoolize \
+ && automake --add-missing \
+ && autoconf \
+ && CXXFLAGS='-O2' CFLAGS='-O2' ./configure --prefix=/app --enable-lz4 \
+ && make -j $(grep -c ^processor /proc/cpuinfo) dist install clean \
+ && apt-get remove -y \
+ autoconf \
+ automake \
+ libexpat1-dev \
+ g++ \
+ libtool \
+ liblz4-dev \
+ m4 \
+ make \
+ zlib1g-dev \
+ && apt-get autoremove -y \
+ && mkdir -p /db/diffs /app/etc \
+ && cp -r /app/src/rules /app/etc/rules \
+ && rm -rf /app/src /app/src.tar.gz \
+ && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt /app/
+
+RUN python3 -m venv /app/venv \
+ && /app/venv/bin/pip install -r /app/requirements.txt
+
+RUN mkdir /nginx && chown nginx:nginx /nginx && chown -R overpass:overpass /db
+
+COPY etc/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+
+COPY etc/nginx-overpass.conf /etc/nginx/nginx.conf
+
+COPY bin/update_overpass.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/
+
+COPY docker-entrypoint.sh /app/
+
+RUN chmod a+x /app/docker-entrypoint.sh /app/bin/update_overpass.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh
+
+ENV OVERPASS_RULES_LOAD 1
+
+EXPOSE 80
+# CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"]
+CMD ["/app/docker-entrypoint.sh"]
diff --git a/0.7.55.3/bin/dispatcher_start.sh b/0.7.55.3/bin/dispatcher_start.sh
new file mode 100644
index 0000000..19a3c55
--- /dev/null
+++ b/0.7.55.3/bin/dispatcher_start.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e -o pipefail
+
+if [[ $OVERPASS_META == 'yes' ]] ; then
+ META_ARG='--meta'
+elif [[ $OVERPASS_META == 'attic' ]] ; then
+ META_ARG='--attic'
+else
+ META_ARG_=''
+fi
+
+find /db/db -type s -print0 | xargs -0 --no-run-if-empty rm && /app/bin/dispatcher --osm-base "${META_ARG}" --db-dir=/db/db
+
diff --git a/0.7.55.3/bin/rules_loop.sh b/0.7.55.3/bin/rules_loop.sh
new file mode 100644
index 0000000..a50f94f
--- /dev/null
+++ b/0.7.55.3/bin/rules_loop.sh
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht
+#
+# This file is part of Overpass_API.
+#
+# Overpass_API is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# Overpass_API is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Overpass_API. If not, see .
+
+if [[ -z $1 ]]; then
+{
+ echo "Usage: $0 database_dir [desired_cpu_load]"
+ exit 0
+};
+fi
+
+CPU_LOAD=${2:-100}
+DB_DIR="`pwd`/$1"
+
+EXEC_DIR="`dirname $0`/"
+if [[ ! ${EXEC_DIR:0:1} == "/" ]]; then
+{
+ EXEC_DIR="`pwd`/$EXEC_DIR"
+};
+fi
+
+pushd "$EXEC_DIR"
+
+while [[ true ]]; do
+{
+ START=$(date +%s)
+ echo "`date '+%F %T'`: update started" >>$DB_DIR/rules_loop.log
+ ./osm3s_query --progress --rules <$DB_DIR/rules/areas.osm3s
+ echo "`date '+%F %T'`: update finished" >>$DB_DIR/rules_loop.log
+ WORK_TIME=$(( $(date +%s) - START ))
+ SLEEP_TIME=$(( WORK_TIME * 100 / CPU_LOAD - WORK_TIME))
+ # let SLEEP_TIME be at least 3 seconds
+ SLEEP_TIME=$(( SLEEP_TIME < 3 ? 3 : SLEEP_TIME))
+ echo "It took $WORK_TIME to run the loop. Desired load is: ${CPU_LOAD}%. Sleeping: $SLEEP_TIME"
+ sleep $SLEEP_TIME
+}; done
diff --git a/0.7.55.3/bin/update_overpass.sh b/0.7.55.3/bin/update_overpass.sh
new file mode 100644
index 0000000..328e375
--- /dev/null
+++ b/0.7.55.3/bin/update_overpass.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env sh
+
+(
+ set -e
+
+ if [ "x$OVERPASS_META" = "xattic" ] ; then
+ META="--keep-attic"
+ elif [ "x$OVERPASS_META" = "xyes" ] ; then
+ META="--meta"
+ else
+ META=""
+ fi
+
+ if [ ! -d /db/diffs ] ; then
+ mkdir /db/diffs
+ fi
+
+ while `true` ; do
+ (
+ if [ ! -e /db/diffs/changes.osm ] ; then
+ ! /app/venv/bin/pyosmium-get-changes --server $OVERPASS_DIFF_URL -o /db/diffs/changes.osm -f /db/replicate_id
+ OSMIUM_STATUS=$?
+ if [ $OSMIUM_STATUS -eq 1 ]; then
+ echo "There are still some updates remainging"
+ fi
+ if [ $OSMIUM_STATUS -eq 2 ]; then
+ echo "Failure downloading updates"
+ exit 0
+ fi
+ else
+ echo "/db/diffs/changes.osm exists. Trying to apply again."
+ fi
+ /app/bin/update_from_dir --osc-dir=/db/diffs/ $META
+ rm /db/diffs/changes.osm
+ )
+ sleep 60
+ done
+) 2>&1 | tee -a /db/changes.log
\ No newline at end of file
diff --git a/0.7.55.3/docker-entrypoint.sh b/0.7.55.3/docker-entrypoint.sh
new file mode 100644
index 0000000..cb69627
--- /dev/null
+++ b/0.7.55.3/docker-entrypoint.sh
@@ -0,0 +1,49 @@
+#!/bin/bash
+
+set -eo pipefail
+shopt -s nullglob
+OVERPASS_META=${OVERPASS_META:-no}
+OVERPASS_MODE=${OVERPASS_MODE:-clone}
+OVERPASS_COMPRESSION=${OVERPASS_COMPRESSION:-gz}
+
+if [[ "$OVERPASS_META" == "attic" ]] ; then
+ META="--keep-attic"
+else
+ META="--meta"
+fi
+
+
+if [ ! -d /db/db ] ; then
+ if [ "$OVERPASS_MODE" = "clone" ]; then
+ mkdir -p /db/db \
+ && /app/bin/download_clone.sh --db-dir=/db/db --source=http://dev.overpass-api.de/api_drolbr/ $META \
+ && cp -r /app/etc/rules /db/db \
+ && chown -R overpass:overpass /db \
+ && echo "Overpass ready, you can start your container with docker start"
+ exit
+ fi
+
+ if [ "$OVERPASS_MODE" = "init" ]; then
+ lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet.osm.bz2; exit" \
+ && /app/bin/init_osm3s.sh /db/planet.osm.bz2 /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION --map-compression-method=$OVERPASS_COMPRESSION"\
+ && echo "Database created. Now updating it." && (
+ ! /app/venv/bin/pyosmium-get-changes -O /db/planet.osm.bz2 --server $OVERPASS_DIFF_URL -o /db/changes.osm -f /db/replicate_id
+ OSMIUM_STATUS=$?
+ if [ $OSMIUM_STATUS -eq 1 ]; then
+ echo "There are still some updates remainging"
+ fi
+ if [ $OSMIUM_STATUS -eq 2 ]; then
+ echo "Failure downloading updates"
+ exit 0
+ fi
+ (cat /db/changes.osm | /app/bin/update_database --db-dir=/db/db $META --compression-method=$OVERPASS_COMPRESSION) 2>&1 | tee -a /db/changes.log
+ ) \
+ && rm /db/planet.osm.bz2 /db/changes.osm \
+ && cp -r /app/etc/rules /db/db \
+ && chown -R overpass:overpass /db \
+ && echo "Overpass ready, you can start your container with docker start"
+ exit
+ fi
+fi
+
+exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf
diff --git a/0.7.55.3/etc/nginx-overpass.conf b/0.7.55.3/etc/nginx-overpass.conf
new file mode 100644
index 0000000..a19dde4
--- /dev/null
+++ b/0.7.55.3/etc/nginx-overpass.conf
@@ -0,0 +1,67 @@
+daemon off;
+
+user nginx;
+worker_processes 4;
+
+error_log /var/log/nginx/error.log warn;
+pid /var/run/nginx.pid;
+
+
+events {
+ worker_connections 1024;
+}
+
+
+http {
+ include /etc/nginx/mime.types;
+ default_type application/octet-stream;
+
+ log_format main '$remote_addr - $remote_user [$time_local] "$request" '
+ '$status $body_bytes_sent "$http_referer" '
+ '"$http_user_agent" "$http_x_forwarded_for"';
+
+ access_log /var/log/nginx/access.log main;
+
+ sendfile on;
+ #tcp_nopush on;
+
+ keepalive_timeout 1000s;
+
+ #gzip on;
+
+ server {
+ listen 80;
+ server_name localhost;
+
+ #charset koi8-r;
+ #access_log /var/log/nginx/log/host.access.log main;
+
+ location / {
+ root /usr/share/nginx/html;
+ index index.html index.htm;
+ }
+
+ #error_page 404 /404.html;
+
+ # redirect server error pages to the static page /50x.html
+ #
+ error_page 500 502 503 504 /50x.html;
+ location = /50x.html {
+ root /usr/share/nginx/html;
+ }
+
+ location /cgi-bin/ {
+ include fastcgi_params;
+ fastcgi_param SCRIPT_FILENAME /app/$fastcgi_script_name;
+ fastcgi_param PATH_INFO $fastcgi_path_info;
+ fastcgi_pass unix:/nginx/fcgiwrap.socket;
+ fastcgi_read_timeout 1000s;
+ }
+
+ location /api/ {
+ rewrite ^/api/(.+)$ /cgi-bin/$1 last;
+ }
+ }
+
+
+}
diff --git a/0.7.55.3/etc/supervisord.conf b/0.7.55.3/etc/supervisord.conf
new file mode 100644
index 0000000..fc67b74
--- /dev/null
+++ b/0.7.55.3/etc/supervisord.conf
@@ -0,0 +1,36 @@
+[supervisord]
+nodaemon=true
+
+[program:overpass_dispatch]
+command=/app/bin/dispatcher_start.sh
+user=overpass
+redirect_stderr=true
+priority=1
+
+[program:nginx]
+command=nginx
+priority=2
+
+[program:fcgiwrap]
+command=/bin/bash -c "find /nginx -type s -print0 | xargs -0 --no-run-if-empty rm && fcgiwrap -s unix:/nginx/fcgiwrap.socket"
+redirect_stderr=true
+user=nginx
+priority=3
+
+[program:update_overpass]
+command=/app/bin/update_overpass.sh
+user=overpass
+redirect_stderr=true
+priority=5
+
+[program:dispatcher_areas]
+command=nice /app/bin/dispatcher --areas --db-dir="/db/db"
+user=overpass
+redirect_stderr=true
+priority=6
+
+[program:areas_rules]
+command=nice /app/bin/rules_loop.sh /db/db %(ENV_OVERPASS_RULES_LOAD)s
+user=overpass
+redirect_stderr=true
+priority=7
diff --git a/0.7.55.3/requirements.txt b/0.7.55.3/requirements.txt
new file mode 100644
index 0000000..8314012
--- /dev/null
+++ b/0.7.55.3/requirements.txt
@@ -0,0 +1 @@
+osmium
diff --git a/0.7.55.4/Dockerfile b/0.7.55.4/Dockerfile
new file mode 100644
index 0000000..4662687
--- /dev/null
+++ b/0.7.55.4/Dockerfile
@@ -0,0 +1,79 @@
+FROM nginx:1.14
+
+RUN addgroup overpass && adduser --home /db --disabled-password --gecos overpass --ingroup overpass overpass
+
+ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.55.4.tar.gz /app/src.tar.gz
+
+RUN apt-get update \
+ && apt-get install --no-install-recommends --no-install-suggests -y \
+ autoconf \
+ automake \
+ bash \
+ bzip2 \
+ ca-certificates \
+ expat \
+ fcgiwrap \
+ g++ \
+ lftp \
+ libexpat1-dev \
+ liblz4-1 \
+ liblz4-dev \
+ libtool \
+ m4 \
+ make \
+ osmium-tool \
+ python3 \
+ python3-venv \
+ supervisor \
+ wget \
+ zlib1g \
+ zlib1g-dev \
+ && mkdir -p /app/src \
+ && cd /app/src \
+ && tar -x -z --strip-components 1 -f ../src.tar.gz \
+ && autoscan \
+ && aclocal \
+ && autoheader \
+ && libtoolize \
+ && automake --add-missing \
+ && autoconf \
+ && CXXFLAGS='-O2' CFLAGS='-O2' ./configure --prefix=/app --enable-lz4 \
+ && make -j $(grep -c ^processor /proc/cpuinfo) dist install clean \
+ && apt-get remove -y \
+ autoconf \
+ automake \
+ libexpat1-dev \
+ g++ \
+ libtool \
+ liblz4-dev \
+ m4 \
+ make \
+ zlib1g-dev \
+ && apt-get autoremove -y \
+ && mkdir -p /db/diffs /app/etc \
+ && cp -r /app/src/rules /app/etc/rules \
+ && rm -rf /app/src /app/src.tar.gz \
+ && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt /app/
+
+RUN python3 -m venv /app/venv \
+ && /app/venv/bin/pip install -r /app/requirements.txt
+
+RUN mkdir /nginx && chown nginx:nginx /nginx && chown -R overpass:overpass /db
+
+COPY etc/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+
+COPY etc/nginx-overpass.conf /etc/nginx/nginx.conf
+
+COPY bin/update_overpass.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/
+
+COPY docker-entrypoint.sh /app/
+
+RUN chmod a+x /app/docker-entrypoint.sh /app/bin/update_overpass.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh
+
+ENV OVERPASS_RULES_LOAD 1
+
+EXPOSE 80
+# CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"]
+CMD ["/app/docker-entrypoint.sh"]
diff --git a/0.7.55.4/bin/dispatcher_start.sh b/0.7.55.4/bin/dispatcher_start.sh
new file mode 100644
index 0000000..19a3c55
--- /dev/null
+++ b/0.7.55.4/bin/dispatcher_start.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e -o pipefail
+
+if [[ $OVERPASS_META == 'yes' ]] ; then
+ META_ARG='--meta'
+elif [[ $OVERPASS_META == 'attic' ]] ; then
+ META_ARG='--attic'
+else
+ META_ARG_=''
+fi
+
+find /db/db -type s -print0 | xargs -0 --no-run-if-empty rm && /app/bin/dispatcher --osm-base "${META_ARG}" --db-dir=/db/db
+
diff --git a/0.7.55.4/bin/rules_loop.sh b/0.7.55.4/bin/rules_loop.sh
new file mode 100644
index 0000000..a50f94f
--- /dev/null
+++ b/0.7.55.4/bin/rules_loop.sh
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht
+#
+# This file is part of Overpass_API.
+#
+# Overpass_API is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# Overpass_API is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Overpass_API. If not, see .
+
+if [[ -z $1 ]]; then
+{
+ echo "Usage: $0 database_dir [desired_cpu_load]"
+ exit 0
+};
+fi
+
+CPU_LOAD=${2:-100}
+DB_DIR="`pwd`/$1"
+
+EXEC_DIR="`dirname $0`/"
+if [[ ! ${EXEC_DIR:0:1} == "/" ]]; then
+{
+ EXEC_DIR="`pwd`/$EXEC_DIR"
+};
+fi
+
+pushd "$EXEC_DIR"
+
+while [[ true ]]; do
+{
+ START=$(date +%s)
+ echo "`date '+%F %T'`: update started" >>$DB_DIR/rules_loop.log
+ ./osm3s_query --progress --rules <$DB_DIR/rules/areas.osm3s
+ echo "`date '+%F %T'`: update finished" >>$DB_DIR/rules_loop.log
+ WORK_TIME=$(( $(date +%s) - START ))
+ SLEEP_TIME=$(( WORK_TIME * 100 / CPU_LOAD - WORK_TIME))
+ # let SLEEP_TIME be at least 3 seconds
+ SLEEP_TIME=$(( SLEEP_TIME < 3 ? 3 : SLEEP_TIME))
+ echo "It took $WORK_TIME to run the loop. Desired load is: ${CPU_LOAD}%. Sleeping: $SLEEP_TIME"
+ sleep $SLEEP_TIME
+}; done
diff --git a/0.7.55.4/bin/update_overpass.sh b/0.7.55.4/bin/update_overpass.sh
new file mode 100644
index 0000000..328e375
--- /dev/null
+++ b/0.7.55.4/bin/update_overpass.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env sh
+
+(
+ set -e
+
+ if [ "x$OVERPASS_META" = "xattic" ] ; then
+ META="--keep-attic"
+ elif [ "x$OVERPASS_META" = "xyes" ] ; then
+ META="--meta"
+ else
+ META=""
+ fi
+
+ if [ ! -d /db/diffs ] ; then
+ mkdir /db/diffs
+ fi
+
+ while `true` ; do
+ (
+ if [ ! -e /db/diffs/changes.osm ] ; then
+ ! /app/venv/bin/pyosmium-get-changes --server $OVERPASS_DIFF_URL -o /db/diffs/changes.osm -f /db/replicate_id
+ OSMIUM_STATUS=$?
+ if [ $OSMIUM_STATUS -eq 1 ]; then
+ echo "There are still some updates remainging"
+ fi
+ if [ $OSMIUM_STATUS -eq 2 ]; then
+ echo "Failure downloading updates"
+ exit 0
+ fi
+ else
+ echo "/db/diffs/changes.osm exists. Trying to apply again."
+ fi
+ /app/bin/update_from_dir --osc-dir=/db/diffs/ $META
+ rm /db/diffs/changes.osm
+ )
+ sleep 60
+ done
+) 2>&1 | tee -a /db/changes.log
\ No newline at end of file
diff --git a/0.7.55.4/docker-entrypoint.sh b/0.7.55.4/docker-entrypoint.sh
new file mode 100644
index 0000000..cb69627
--- /dev/null
+++ b/0.7.55.4/docker-entrypoint.sh
@@ -0,0 +1,49 @@
+#!/bin/bash
+
+set -eo pipefail
+shopt -s nullglob
+OVERPASS_META=${OVERPASS_META:-no}
+OVERPASS_MODE=${OVERPASS_MODE:-clone}
+OVERPASS_COMPRESSION=${OVERPASS_COMPRESSION:-gz}
+
+if [[ "$OVERPASS_META" == "attic" ]] ; then
+ META="--keep-attic"
+else
+ META="--meta"
+fi
+
+
+if [ ! -d /db/db ] ; then
+ if [ "$OVERPASS_MODE" = "clone" ]; then
+ mkdir -p /db/db \
+ && /app/bin/download_clone.sh --db-dir=/db/db --source=http://dev.overpass-api.de/api_drolbr/ $META \
+ && cp -r /app/etc/rules /db/db \
+ && chown -R overpass:overpass /db \
+ && echo "Overpass ready, you can start your container with docker start"
+ exit
+ fi
+
+ if [ "$OVERPASS_MODE" = "init" ]; then
+ lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet.osm.bz2; exit" \
+ && /app/bin/init_osm3s.sh /db/planet.osm.bz2 /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION --map-compression-method=$OVERPASS_COMPRESSION"\
+ && echo "Database created. Now updating it." && (
+ ! /app/venv/bin/pyosmium-get-changes -O /db/planet.osm.bz2 --server $OVERPASS_DIFF_URL -o /db/changes.osm -f /db/replicate_id
+ OSMIUM_STATUS=$?
+ if [ $OSMIUM_STATUS -eq 1 ]; then
+ echo "There are still some updates remainging"
+ fi
+ if [ $OSMIUM_STATUS -eq 2 ]; then
+ echo "Failure downloading updates"
+ exit 0
+ fi
+ (cat /db/changes.osm | /app/bin/update_database --db-dir=/db/db $META --compression-method=$OVERPASS_COMPRESSION) 2>&1 | tee -a /db/changes.log
+ ) \
+ && rm /db/planet.osm.bz2 /db/changes.osm \
+ && cp -r /app/etc/rules /db/db \
+ && chown -R overpass:overpass /db \
+ && echo "Overpass ready, you can start your container with docker start"
+ exit
+ fi
+fi
+
+exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf
diff --git a/0.7.55.4/etc/nginx-overpass.conf b/0.7.55.4/etc/nginx-overpass.conf
new file mode 100644
index 0000000..a19dde4
--- /dev/null
+++ b/0.7.55.4/etc/nginx-overpass.conf
@@ -0,0 +1,67 @@
+daemon off;
+
+user nginx;
+worker_processes 4;
+
+error_log /var/log/nginx/error.log warn;
+pid /var/run/nginx.pid;
+
+
+events {
+ worker_connections 1024;
+}
+
+
+http {
+ include /etc/nginx/mime.types;
+ default_type application/octet-stream;
+
+ log_format main '$remote_addr - $remote_user [$time_local] "$request" '
+ '$status $body_bytes_sent "$http_referer" '
+ '"$http_user_agent" "$http_x_forwarded_for"';
+
+ access_log /var/log/nginx/access.log main;
+
+ sendfile on;
+ #tcp_nopush on;
+
+ keepalive_timeout 1000s;
+
+ #gzip on;
+
+ server {
+ listen 80;
+ server_name localhost;
+
+ #charset koi8-r;
+ #access_log /var/log/nginx/log/host.access.log main;
+
+ location / {
+ root /usr/share/nginx/html;
+ index index.html index.htm;
+ }
+
+ #error_page 404 /404.html;
+
+ # redirect server error pages to the static page /50x.html
+ #
+ error_page 500 502 503 504 /50x.html;
+ location = /50x.html {
+ root /usr/share/nginx/html;
+ }
+
+ location /cgi-bin/ {
+ include fastcgi_params;
+ fastcgi_param SCRIPT_FILENAME /app/$fastcgi_script_name;
+ fastcgi_param PATH_INFO $fastcgi_path_info;
+ fastcgi_pass unix:/nginx/fcgiwrap.socket;
+ fastcgi_read_timeout 1000s;
+ }
+
+ location /api/ {
+ rewrite ^/api/(.+)$ /cgi-bin/$1 last;
+ }
+ }
+
+
+}
diff --git a/0.7.55.4/etc/supervisord.conf b/0.7.55.4/etc/supervisord.conf
new file mode 100644
index 0000000..fc67b74
--- /dev/null
+++ b/0.7.55.4/etc/supervisord.conf
@@ -0,0 +1,36 @@
+[supervisord]
+nodaemon=true
+
+[program:overpass_dispatch]
+command=/app/bin/dispatcher_start.sh
+user=overpass
+redirect_stderr=true
+priority=1
+
+[program:nginx]
+command=nginx
+priority=2
+
+[program:fcgiwrap]
+command=/bin/bash -c "find /nginx -type s -print0 | xargs -0 --no-run-if-empty rm && fcgiwrap -s unix:/nginx/fcgiwrap.socket"
+redirect_stderr=true
+user=nginx
+priority=3
+
+[program:update_overpass]
+command=/app/bin/update_overpass.sh
+user=overpass
+redirect_stderr=true
+priority=5
+
+[program:dispatcher_areas]
+command=nice /app/bin/dispatcher --areas --db-dir="/db/db"
+user=overpass
+redirect_stderr=true
+priority=6
+
+[program:areas_rules]
+command=nice /app/bin/rules_loop.sh /db/db %(ENV_OVERPASS_RULES_LOAD)s
+user=overpass
+redirect_stderr=true
+priority=7
diff --git a/0.7.55.4/requirements.txt b/0.7.55.4/requirements.txt
new file mode 100644
index 0000000..8314012
--- /dev/null
+++ b/0.7.55.4/requirements.txt
@@ -0,0 +1 @@
+osmium
diff --git a/0.7.55.5/Dockerfile b/0.7.55.5/Dockerfile
new file mode 100644
index 0000000..c034d99
--- /dev/null
+++ b/0.7.55.5/Dockerfile
@@ -0,0 +1,79 @@
+FROM nginx:1.14
+
+RUN addgroup overpass && adduser --home /db --disabled-password --gecos overpass --ingroup overpass overpass
+
+ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.55.5.tar.gz /app/src.tar.gz
+
+RUN apt-get update \
+ && apt-get install --no-install-recommends --no-install-suggests -y \
+ autoconf \
+ automake \
+ bash \
+ bzip2 \
+ ca-certificates \
+ expat \
+ fcgiwrap \
+ g++ \
+ lftp \
+ libexpat1-dev \
+ liblz4-1 \
+ liblz4-dev \
+ libtool \
+ m4 \
+ make \
+ osmium-tool \
+ python3 \
+ python3-venv \
+ supervisor \
+ wget \
+ zlib1g \
+ zlib1g-dev \
+ && mkdir -p /app/src \
+ && cd /app/src \
+ && tar -x -z --strip-components 1 -f ../src.tar.gz \
+ && autoscan \
+ && aclocal \
+ && autoheader \
+ && libtoolize \
+ && automake --add-missing \
+ && autoconf \
+ && CXXFLAGS='-O2' CFLAGS='-O2' ./configure --prefix=/app --enable-lz4 \
+ && make -j $(grep -c ^processor /proc/cpuinfo) dist install clean \
+ && apt-get remove -y \
+ autoconf \
+ automake \
+ libexpat1-dev \
+ g++ \
+ libtool \
+ liblz4-dev \
+ m4 \
+ make \
+ zlib1g-dev \
+ && apt-get autoremove -y \
+ && mkdir -p /db/diffs /app/etc \
+ && cp -r /app/src/rules /app/etc/rules \
+ && rm -rf /app/src /app/src.tar.gz \
+ && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt /app/
+
+RUN python3 -m venv /app/venv \
+ && /app/venv/bin/pip install -r /app/requirements.txt
+
+RUN mkdir /nginx && chown nginx:nginx /nginx && chown -R overpass:overpass /db
+
+COPY etc/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+
+COPY etc/nginx-overpass.conf /etc/nginx/nginx.conf
+
+COPY bin/update_overpass.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/
+
+COPY docker-entrypoint.sh /app/
+
+RUN chmod a+x /app/docker-entrypoint.sh /app/bin/update_overpass.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh
+
+ENV OVERPASS_RULES_LOAD 1
+
+EXPOSE 80
+# CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"]
+CMD ["/app/docker-entrypoint.sh"]
diff --git a/0.7.55.5/bin/dispatcher_start.sh b/0.7.55.5/bin/dispatcher_start.sh
new file mode 100644
index 0000000..19a3c55
--- /dev/null
+++ b/0.7.55.5/bin/dispatcher_start.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e -o pipefail
+
+if [[ $OVERPASS_META == 'yes' ]] ; then
+ META_ARG='--meta'
+elif [[ $OVERPASS_META == 'attic' ]] ; then
+ META_ARG='--attic'
+else
+ META_ARG_=''
+fi
+
+find /db/db -type s -print0 | xargs -0 --no-run-if-empty rm && /app/bin/dispatcher --osm-base "${META_ARG}" --db-dir=/db/db
+
diff --git a/0.7.55.5/bin/rules_loop.sh b/0.7.55.5/bin/rules_loop.sh
new file mode 100644
index 0000000..a50f94f
--- /dev/null
+++ b/0.7.55.5/bin/rules_loop.sh
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht
+#
+# This file is part of Overpass_API.
+#
+# Overpass_API is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# Overpass_API is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Overpass_API. If not, see .
+
+if [[ -z $1 ]]; then
+{
+ echo "Usage: $0 database_dir [desired_cpu_load]"
+ exit 0
+};
+fi
+
+CPU_LOAD=${2:-100}
+DB_DIR="`pwd`/$1"
+
+EXEC_DIR="`dirname $0`/"
+if [[ ! ${EXEC_DIR:0:1} == "/" ]]; then
+{
+ EXEC_DIR="`pwd`/$EXEC_DIR"
+};
+fi
+
+pushd "$EXEC_DIR"
+
+while [[ true ]]; do
+{
+ START=$(date +%s)
+ echo "`date '+%F %T'`: update started" >>$DB_DIR/rules_loop.log
+ ./osm3s_query --progress --rules <$DB_DIR/rules/areas.osm3s
+ echo "`date '+%F %T'`: update finished" >>$DB_DIR/rules_loop.log
+ WORK_TIME=$(( $(date +%s) - START ))
+ SLEEP_TIME=$(( WORK_TIME * 100 / CPU_LOAD - WORK_TIME))
+ # let SLEEP_TIME be at least 3 seconds
+ SLEEP_TIME=$(( SLEEP_TIME < 3 ? 3 : SLEEP_TIME))
+ echo "It took $WORK_TIME to run the loop. Desired load is: ${CPU_LOAD}%. Sleeping: $SLEEP_TIME"
+ sleep $SLEEP_TIME
+}; done
diff --git a/0.7.55.5/bin/update_overpass.sh b/0.7.55.5/bin/update_overpass.sh
new file mode 100644
index 0000000..328e375
--- /dev/null
+++ b/0.7.55.5/bin/update_overpass.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env sh
+
+(
+ set -e
+
+ if [ "x$OVERPASS_META" = "xattic" ] ; then
+ META="--keep-attic"
+ elif [ "x$OVERPASS_META" = "xyes" ] ; then
+ META="--meta"
+ else
+ META=""
+ fi
+
+ if [ ! -d /db/diffs ] ; then
+ mkdir /db/diffs
+ fi
+
+ while `true` ; do
+ (
+ if [ ! -e /db/diffs/changes.osm ] ; then
+ ! /app/venv/bin/pyosmium-get-changes --server $OVERPASS_DIFF_URL -o /db/diffs/changes.osm -f /db/replicate_id
+ OSMIUM_STATUS=$?
+ if [ $OSMIUM_STATUS -eq 1 ]; then
+ echo "There are still some updates remainging"
+ fi
+ if [ $OSMIUM_STATUS -eq 2 ]; then
+ echo "Failure downloading updates"
+ exit 0
+ fi
+ else
+ echo "/db/diffs/changes.osm exists. Trying to apply again."
+ fi
+ /app/bin/update_from_dir --osc-dir=/db/diffs/ $META
+ rm /db/diffs/changes.osm
+ )
+ sleep 60
+ done
+) 2>&1 | tee -a /db/changes.log
\ No newline at end of file
diff --git a/0.7.55.5/docker-entrypoint.sh b/0.7.55.5/docker-entrypoint.sh
new file mode 100644
index 0000000..cb69627
--- /dev/null
+++ b/0.7.55.5/docker-entrypoint.sh
@@ -0,0 +1,49 @@
+#!/bin/bash
+
+set -eo pipefail
+shopt -s nullglob
+OVERPASS_META=${OVERPASS_META:-no}
+OVERPASS_MODE=${OVERPASS_MODE:-clone}
+OVERPASS_COMPRESSION=${OVERPASS_COMPRESSION:-gz}
+
+if [[ "$OVERPASS_META" == "attic" ]] ; then
+ META="--keep-attic"
+else
+ META="--meta"
+fi
+
+
+if [ ! -d /db/db ] ; then
+ if [ "$OVERPASS_MODE" = "clone" ]; then
+ mkdir -p /db/db \
+ && /app/bin/download_clone.sh --db-dir=/db/db --source=http://dev.overpass-api.de/api_drolbr/ $META \
+ && cp -r /app/etc/rules /db/db \
+ && chown -R overpass:overpass /db \
+ && echo "Overpass ready, you can start your container with docker start"
+ exit
+ fi
+
+ if [ "$OVERPASS_MODE" = "init" ]; then
+ lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet.osm.bz2; exit" \
+ && /app/bin/init_osm3s.sh /db/planet.osm.bz2 /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION --map-compression-method=$OVERPASS_COMPRESSION"\
+ && echo "Database created. Now updating it." && (
+ ! /app/venv/bin/pyosmium-get-changes -O /db/planet.osm.bz2 --server $OVERPASS_DIFF_URL -o /db/changes.osm -f /db/replicate_id
+ OSMIUM_STATUS=$?
+ if [ $OSMIUM_STATUS -eq 1 ]; then
+ echo "There are still some updates remainging"
+ fi
+ if [ $OSMIUM_STATUS -eq 2 ]; then
+ echo "Failure downloading updates"
+ exit 0
+ fi
+ (cat /db/changes.osm | /app/bin/update_database --db-dir=/db/db $META --compression-method=$OVERPASS_COMPRESSION) 2>&1 | tee -a /db/changes.log
+ ) \
+ && rm /db/planet.osm.bz2 /db/changes.osm \
+ && cp -r /app/etc/rules /db/db \
+ && chown -R overpass:overpass /db \
+ && echo "Overpass ready, you can start your container with docker start"
+ exit
+ fi
+fi
+
+exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf
diff --git a/0.7.55.5/etc/nginx-overpass.conf b/0.7.55.5/etc/nginx-overpass.conf
new file mode 100644
index 0000000..a19dde4
--- /dev/null
+++ b/0.7.55.5/etc/nginx-overpass.conf
@@ -0,0 +1,67 @@
+daemon off;
+
+user nginx;
+worker_processes 4;
+
+error_log /var/log/nginx/error.log warn;
+pid /var/run/nginx.pid;
+
+
+events {
+ worker_connections 1024;
+}
+
+
+http {
+ include /etc/nginx/mime.types;
+ default_type application/octet-stream;
+
+ log_format main '$remote_addr - $remote_user [$time_local] "$request" '
+ '$status $body_bytes_sent "$http_referer" '
+ '"$http_user_agent" "$http_x_forwarded_for"';
+
+ access_log /var/log/nginx/access.log main;
+
+ sendfile on;
+ #tcp_nopush on;
+
+ keepalive_timeout 1000s;
+
+ #gzip on;
+
+ server {
+ listen 80;
+ server_name localhost;
+
+ #charset koi8-r;
+ #access_log /var/log/nginx/log/host.access.log main;
+
+ location / {
+ root /usr/share/nginx/html;
+ index index.html index.htm;
+ }
+
+ #error_page 404 /404.html;
+
+ # redirect server error pages to the static page /50x.html
+ #
+ error_page 500 502 503 504 /50x.html;
+ location = /50x.html {
+ root /usr/share/nginx/html;
+ }
+
+ location /cgi-bin/ {
+ include fastcgi_params;
+ fastcgi_param SCRIPT_FILENAME /app/$fastcgi_script_name;
+ fastcgi_param PATH_INFO $fastcgi_path_info;
+ fastcgi_pass unix:/nginx/fcgiwrap.socket;
+ fastcgi_read_timeout 1000s;
+ }
+
+ location /api/ {
+ rewrite ^/api/(.+)$ /cgi-bin/$1 last;
+ }
+ }
+
+
+}
diff --git a/0.7.55.5/etc/supervisord.conf b/0.7.55.5/etc/supervisord.conf
new file mode 100644
index 0000000..fc67b74
--- /dev/null
+++ b/0.7.55.5/etc/supervisord.conf
@@ -0,0 +1,36 @@
+[supervisord]
+nodaemon=true
+
+[program:overpass_dispatch]
+command=/app/bin/dispatcher_start.sh
+user=overpass
+redirect_stderr=true
+priority=1
+
+[program:nginx]
+command=nginx
+priority=2
+
+[program:fcgiwrap]
+command=/bin/bash -c "find /nginx -type s -print0 | xargs -0 --no-run-if-empty rm && fcgiwrap -s unix:/nginx/fcgiwrap.socket"
+redirect_stderr=true
+user=nginx
+priority=3
+
+[program:update_overpass]
+command=/app/bin/update_overpass.sh
+user=overpass
+redirect_stderr=true
+priority=5
+
+[program:dispatcher_areas]
+command=nice /app/bin/dispatcher --areas --db-dir="/db/db"
+user=overpass
+redirect_stderr=true
+priority=6
+
+[program:areas_rules]
+command=nice /app/bin/rules_loop.sh /db/db %(ENV_OVERPASS_RULES_LOAD)s
+user=overpass
+redirect_stderr=true
+priority=7
diff --git a/0.7.55.5/requirements.txt b/0.7.55.5/requirements.txt
new file mode 100644
index 0000000..8314012
--- /dev/null
+++ b/0.7.55.5/requirements.txt
@@ -0,0 +1 @@
+osmium
diff --git a/0.7.55.6/Dockerfile b/0.7.55.6/Dockerfile
new file mode 100644
index 0000000..7e5676b
--- /dev/null
+++ b/0.7.55.6/Dockerfile
@@ -0,0 +1,79 @@
+FROM nginx:1.14
+
+RUN addgroup overpass && adduser --home /db --disabled-password --gecos overpass --ingroup overpass overpass
+
+ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.55.6.tar.gz /app/src.tar.gz
+
+RUN apt-get update \
+ && apt-get install --no-install-recommends --no-install-suggests -y \
+ autoconf \
+ automake \
+ bash \
+ bzip2 \
+ ca-certificates \
+ expat \
+ fcgiwrap \
+ g++ \
+ lftp \
+ libexpat1-dev \
+ liblz4-1 \
+ liblz4-dev \
+ libtool \
+ m4 \
+ make \
+ osmium-tool \
+ python3 \
+ python3-venv \
+ supervisor \
+ wget \
+ zlib1g \
+ zlib1g-dev \
+ && mkdir -p /app/src \
+ && cd /app/src \
+ && tar -x -z --strip-components 1 -f ../src.tar.gz \
+ && autoscan \
+ && aclocal \
+ && autoheader \
+ && libtoolize \
+ && automake --add-missing \
+ && autoconf \
+ && CXXFLAGS='-O2' CFLAGS='-O2' ./configure --prefix=/app --enable-lz4 \
+ && make -j $(grep -c ^processor /proc/cpuinfo) dist install clean \
+ && apt-get remove -y \
+ autoconf \
+ automake \
+ libexpat1-dev \
+ g++ \
+ libtool \
+ liblz4-dev \
+ m4 \
+ make \
+ zlib1g-dev \
+ && apt-get autoremove -y \
+ && mkdir -p /db/diffs /app/etc \
+ && cp -r /app/src/rules /app/etc/rules \
+ && rm -rf /app/src /app/src.tar.gz \
+ && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt /app/
+
+RUN python3 -m venv /app/venv \
+ && /app/venv/bin/pip install -r /app/requirements.txt
+
+RUN mkdir /nginx && chown nginx:nginx /nginx && chown -R overpass:overpass /db
+
+COPY etc/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+
+COPY etc/nginx-overpass.conf /etc/nginx/nginx.conf
+
+COPY bin/update_overpass.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/
+
+COPY docker-entrypoint.sh /app/
+
+RUN chmod a+x /app/docker-entrypoint.sh /app/bin/update_overpass.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh
+
+ENV OVERPASS_RULES_LOAD 1
+
+EXPOSE 80
+# CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"]
+CMD ["/app/docker-entrypoint.sh"]
diff --git a/0.7.55.6/bin/dispatcher_start.sh b/0.7.55.6/bin/dispatcher_start.sh
new file mode 100644
index 0000000..19a3c55
--- /dev/null
+++ b/0.7.55.6/bin/dispatcher_start.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e -o pipefail
+
+if [[ $OVERPASS_META == 'yes' ]] ; then
+ META_ARG='--meta'
+elif [[ $OVERPASS_META == 'attic' ]] ; then
+ META_ARG='--attic'
+else
+ META_ARG_=''
+fi
+
+find /db/db -type s -print0 | xargs -0 --no-run-if-empty rm && /app/bin/dispatcher --osm-base "${META_ARG}" --db-dir=/db/db
+
diff --git a/0.7.55.6/bin/rules_loop.sh b/0.7.55.6/bin/rules_loop.sh
new file mode 100644
index 0000000..a50f94f
--- /dev/null
+++ b/0.7.55.6/bin/rules_loop.sh
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht
+#
+# This file is part of Overpass_API.
+#
+# Overpass_API is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# Overpass_API is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Overpass_API. If not, see .
+
+if [[ -z $1 ]]; then
+{
+ echo "Usage: $0 database_dir [desired_cpu_load]"
+ exit 0
+};
+fi
+
+CPU_LOAD=${2:-100}
+DB_DIR="`pwd`/$1"
+
+EXEC_DIR="`dirname $0`/"
+if [[ ! ${EXEC_DIR:0:1} == "/" ]]; then
+{
+ EXEC_DIR="`pwd`/$EXEC_DIR"
+};
+fi
+
+pushd "$EXEC_DIR"
+
+while [[ true ]]; do
+{
+ START=$(date +%s)
+ echo "`date '+%F %T'`: update started" >>$DB_DIR/rules_loop.log
+ ./osm3s_query --progress --rules <$DB_DIR/rules/areas.osm3s
+ echo "`date '+%F %T'`: update finished" >>$DB_DIR/rules_loop.log
+ WORK_TIME=$(( $(date +%s) - START ))
+ SLEEP_TIME=$(( WORK_TIME * 100 / CPU_LOAD - WORK_TIME))
+ # let SLEEP_TIME be at least 3 seconds
+ SLEEP_TIME=$(( SLEEP_TIME < 3 ? 3 : SLEEP_TIME))
+ echo "It took $WORK_TIME to run the loop. Desired load is: ${CPU_LOAD}%. Sleeping: $SLEEP_TIME"
+ sleep $SLEEP_TIME
+}; done
diff --git a/0.7.55.6/bin/update_overpass.sh b/0.7.55.6/bin/update_overpass.sh
new file mode 100644
index 0000000..328e375
--- /dev/null
+++ b/0.7.55.6/bin/update_overpass.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env sh
+
+(
+ set -e
+
+ if [ "x$OVERPASS_META" = "xattic" ] ; then
+ META="--keep-attic"
+ elif [ "x$OVERPASS_META" = "xyes" ] ; then
+ META="--meta"
+ else
+ META=""
+ fi
+
+ if [ ! -d /db/diffs ] ; then
+ mkdir /db/diffs
+ fi
+
+ while `true` ; do
+ (
+ if [ ! -e /db/diffs/changes.osm ] ; then
+ ! /app/venv/bin/pyosmium-get-changes --server $OVERPASS_DIFF_URL -o /db/diffs/changes.osm -f /db/replicate_id
+ OSMIUM_STATUS=$?
+ if [ $OSMIUM_STATUS -eq 1 ]; then
+ echo "There are still some updates remainging"
+ fi
+ if [ $OSMIUM_STATUS -eq 2 ]; then
+ echo "Failure downloading updates"
+ exit 0
+ fi
+ else
+ echo "/db/diffs/changes.osm exists. Trying to apply again."
+ fi
+ /app/bin/update_from_dir --osc-dir=/db/diffs/ $META
+ rm /db/diffs/changes.osm
+ )
+ sleep 60
+ done
+) 2>&1 | tee -a /db/changes.log
\ No newline at end of file
diff --git a/0.7.55.6/docker-entrypoint.sh b/0.7.55.6/docker-entrypoint.sh
new file mode 100644
index 0000000..cb69627
--- /dev/null
+++ b/0.7.55.6/docker-entrypoint.sh
@@ -0,0 +1,49 @@
+#!/bin/bash
+
+set -eo pipefail
+shopt -s nullglob
+OVERPASS_META=${OVERPASS_META:-no}
+OVERPASS_MODE=${OVERPASS_MODE:-clone}
+OVERPASS_COMPRESSION=${OVERPASS_COMPRESSION:-gz}
+
+if [[ "$OVERPASS_META" == "attic" ]] ; then
+ META="--keep-attic"
+else
+ META="--meta"
+fi
+
+
+if [ ! -d /db/db ] ; then
+ if [ "$OVERPASS_MODE" = "clone" ]; then
+ mkdir -p /db/db \
+ && /app/bin/download_clone.sh --db-dir=/db/db --source=http://dev.overpass-api.de/api_drolbr/ $META \
+ && cp -r /app/etc/rules /db/db \
+ && chown -R overpass:overpass /db \
+ && echo "Overpass ready, you can start your container with docker start"
+ exit
+ fi
+
+ if [ "$OVERPASS_MODE" = "init" ]; then
+ lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet.osm.bz2; exit" \
+ && /app/bin/init_osm3s.sh /db/planet.osm.bz2 /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION --map-compression-method=$OVERPASS_COMPRESSION"\
+ && echo "Database created. Now updating it." && (
+ ! /app/venv/bin/pyosmium-get-changes -O /db/planet.osm.bz2 --server $OVERPASS_DIFF_URL -o /db/changes.osm -f /db/replicate_id
+ OSMIUM_STATUS=$?
+ if [ $OSMIUM_STATUS -eq 1 ]; then
+ echo "There are still some updates remainging"
+ fi
+ if [ $OSMIUM_STATUS -eq 2 ]; then
+ echo "Failure downloading updates"
+ exit 0
+ fi
+ (cat /db/changes.osm | /app/bin/update_database --db-dir=/db/db $META --compression-method=$OVERPASS_COMPRESSION) 2>&1 | tee -a /db/changes.log
+ ) \
+ && rm /db/planet.osm.bz2 /db/changes.osm \
+ && cp -r /app/etc/rules /db/db \
+ && chown -R overpass:overpass /db \
+ && echo "Overpass ready, you can start your container with docker start"
+ exit
+ fi
+fi
+
+exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf
diff --git a/0.7.55.6/etc/nginx-overpass.conf b/0.7.55.6/etc/nginx-overpass.conf
new file mode 100644
index 0000000..a19dde4
--- /dev/null
+++ b/0.7.55.6/etc/nginx-overpass.conf
@@ -0,0 +1,67 @@
+daemon off;
+
+user nginx;
+worker_processes 4;
+
+error_log /var/log/nginx/error.log warn;
+pid /var/run/nginx.pid;
+
+
+events {
+ worker_connections 1024;
+}
+
+
+http {
+ include /etc/nginx/mime.types;
+ default_type application/octet-stream;
+
+ log_format main '$remote_addr - $remote_user [$time_local] "$request" '
+ '$status $body_bytes_sent "$http_referer" '
+ '"$http_user_agent" "$http_x_forwarded_for"';
+
+ access_log /var/log/nginx/access.log main;
+
+ sendfile on;
+ #tcp_nopush on;
+
+ keepalive_timeout 1000s;
+
+ #gzip on;
+
+ server {
+ listen 80;
+ server_name localhost;
+
+ #charset koi8-r;
+ #access_log /var/log/nginx/log/host.access.log main;
+
+ location / {
+ root /usr/share/nginx/html;
+ index index.html index.htm;
+ }
+
+ #error_page 404 /404.html;
+
+ # redirect server error pages to the static page /50x.html
+ #
+ error_page 500 502 503 504 /50x.html;
+ location = /50x.html {
+ root /usr/share/nginx/html;
+ }
+
+ location /cgi-bin/ {
+ include fastcgi_params;
+ fastcgi_param SCRIPT_FILENAME /app/$fastcgi_script_name;
+ fastcgi_param PATH_INFO $fastcgi_path_info;
+ fastcgi_pass unix:/nginx/fcgiwrap.socket;
+ fastcgi_read_timeout 1000s;
+ }
+
+ location /api/ {
+ rewrite ^/api/(.+)$ /cgi-bin/$1 last;
+ }
+ }
+
+
+}
diff --git a/0.7.55.6/etc/supervisord.conf b/0.7.55.6/etc/supervisord.conf
new file mode 100644
index 0000000..fc67b74
--- /dev/null
+++ b/0.7.55.6/etc/supervisord.conf
@@ -0,0 +1,36 @@
+[supervisord]
+nodaemon=true
+
+[program:overpass_dispatch]
+command=/app/bin/dispatcher_start.sh
+user=overpass
+redirect_stderr=true
+priority=1
+
+[program:nginx]
+command=nginx
+priority=2
+
+[program:fcgiwrap]
+command=/bin/bash -c "find /nginx -type s -print0 | xargs -0 --no-run-if-empty rm && fcgiwrap -s unix:/nginx/fcgiwrap.socket"
+redirect_stderr=true
+user=nginx
+priority=3
+
+[program:update_overpass]
+command=/app/bin/update_overpass.sh
+user=overpass
+redirect_stderr=true
+priority=5
+
+[program:dispatcher_areas]
+command=nice /app/bin/dispatcher --areas --db-dir="/db/db"
+user=overpass
+redirect_stderr=true
+priority=6
+
+[program:areas_rules]
+command=nice /app/bin/rules_loop.sh /db/db %(ENV_OVERPASS_RULES_LOAD)s
+user=overpass
+redirect_stderr=true
+priority=7
diff --git a/0.7.55.6/requirements.txt b/0.7.55.6/requirements.txt
new file mode 100644
index 0000000..8314012
--- /dev/null
+++ b/0.7.55.6/requirements.txt
@@ -0,0 +1 @@
+osmium
diff --git a/0.7.55/Dockerfile b/0.7.55/Dockerfile
index eb993af..0bf3faa 100644
--- a/0.7.55/Dockerfile
+++ b/0.7.55/Dockerfile
@@ -2,26 +2,33 @@ FROM nginx:1.14
RUN addgroup overpass && adduser --home /db --disabled-password --gecos overpass --ingroup overpass overpass
+ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.55.tar.gz /app/src.tar.gz
+
RUN apt-get update \
&& apt-get install --no-install-recommends --no-install-suggests -y \
autoconf \
automake \
+ bash \
+ bzip2 \
ca-certificates \
expat \
- libexpat1-dev \
+ fcgiwrap \
g++ \
+ lftp \
+ libexpat1-dev \
liblz4-1 \
liblz4-dev \
libtool \
m4 \
make \
+ osmium-tool \
+ python3 \
+ python3-venv \
+ supervisor \
+ wget \
zlib1g \
- zlib1g-dev
-
-
-ADD http://dev.overpass-api.de/releases/osm-3s_v0.7.55.tar.gz /app/src.tar.gz
-
-RUN mkdir -p /app/src \
+ zlib1g-dev \
+ && mkdir -p /app/src \
&& cd /app/src \
&& tar -x -z --strip-components 1 -f ../src.tar.gz \
&& autoscan \
@@ -45,26 +52,28 @@ RUN mkdir -p /app/src \
&& apt-get autoremove -y \
&& mkdir -p /db/diffs /app/etc \
&& cp -r /app/src/rules /app/etc/rules \
- && rm -rf /app/src /app/src.tar.gz
+ && rm -rf /app/src /app/src.tar.gz \
+ && rm -rf /var/lib/apt/lists/*
-RUN apt-get install --no-install-recommends --no-install-suggests -y \
- supervisor \
- bash \
- lftp \
- wget \
- fcgiwrap \
- bzip2
+COPY requirements.txt /app/
+RUN python3 -m venv /app/venv \
+ && /app/venv/bin/pip install -r /app/requirements.txt
RUN mkdir /nginx && chown nginx:nginx /nginx && chown -R overpass:overpass /db
+
COPY etc/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+
COPY etc/nginx-overpass.conf /etc/nginx/nginx.conf
-COPY bin/apply_osc_to_db.sh bin/fetch_osc.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/
+
+COPY bin/update_overpass.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/
+
COPY docker-entrypoint.sh /app/
-RUN chmod a+x /app/docker-entrypoint.sh /app/bin/apply_osc_to_db.sh /app/bin/fetch_osc.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh
-VOLUME /db
+
+RUN chmod a+x /app/docker-entrypoint.sh /app/bin/update_overpass.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh
ENV OVERPASS_RULES_LOAD 1
+
EXPOSE 80
# CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"]
CMD ["/app/docker-entrypoint.sh"]
diff --git a/0.7.55/bin/apply_osc_to_db.sh b/0.7.55/bin/apply_osc_to_db.sh
deleted file mode 100755
index 94d1879..0000000
--- a/0.7.55/bin/apply_osc_to_db.sh
+++ /dev/null
@@ -1,163 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht
-#
-# This file is part of Overpass_API.
-#
-# Overpass_API is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# Overpass_API is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with Overpass_API. If not, see .
-
-if [[ -z $3 ]]; then
-{
- echo "Usage: $0 replicate_dir start_id --meta=(attic|yes|no)"
- exit 0
-}; fi
-
-EXEC_DIR="`dirname $0`/"
-if [[ ! ${EXEC_DIR:0:1} == "/" ]]; then
-{
- EXEC_DIR="`pwd`/$EXEC_DIR"
-}; fi
-
-DB_DIR=`$EXEC_DIR/dispatcher --show-dir`
-
-if [[ ! -d $DB_DIR ]] ; then
- echo "Can't find DB_DIR. Returned value: $DB_DIR"
- exit 1;
-fi
-
-REPLICATE_DIR="$1"
-if [[ ! ${REPLICATE_DIR:0:1} == "/" ]]; then
-{
- REPLICATE_DIR="`pwd`/$REPLICATE_DIR"
-}; fi
-
-START=$2
-META=
-
-if [[ $3 == "--meta=attic" ]]; then
- META="--keep-attic"
-elif [[ $3 == "--meta=yes" || $3 == "--meta" ]]; then
- META="--meta"
-elif [[ $3 == "--meta=no" ]]; then
- META=
-else
-{
- echo "You must specify --meta=yes or --meta=no"
- exit 0
-}; fi
-
-PRODUCE_DIFF=
-
-get_replicate_filename()
-{
- printf -v TDIGIT3 %03u $(($1 % 1000))
- ARG=$(($1 / 1000))
- printf -v TDIGIT2 %03u $(($ARG % 1000))
- ARG=$(($ARG / 1000))
- printf -v TDIGIT1 %03u $ARG
- REPLICATE_TRUNK_DIR=$TDIGIT1/$TDIGIT2/
- REPLICATE_FILENAME=$TDIGIT1/$TDIGIT2/$TDIGIT3
-};
-
-
-collect_minute_diffs()
-{
- TEMP_DIR=$1
- TARGET=$(($START + 1))
-
- get_replicate_filename $TARGET
-
- while [[ ( -s $REPLICATE_DIR/$REPLICATE_FILENAME.state.txt ) && ( $(($START + 1440)) -ge $(($TARGET)) ) && ( `du -m $TEMP_DIR | awk '{ print $1; }'` -le 512 ) ]];
- do
- {
- printf -v TARGET_FILE %09u $TARGET
- gunzip <$REPLICATE_DIR/$REPLICATE_FILENAME.osc.gz >$TEMP_DIR/$TARGET_FILE.osc
- TARGET=$(($TARGET + 1))
- get_replicate_filename $TARGET
- };
- done
- TARGET=$(($TARGET - 1))
-};
-
-
-apply_minute_diffs()
-{
- ./update_from_dir --osc-dir=$1 --version=$DATA_VERSION $META --flush-size=0
- EXITCODE=$?
- while [[ $EXITCODE -ne 0 ]];
- do
- {
- sleep 60
- ./update_from_dir --osc-dir=$1 --version=$DATA_VERSION $META --flush-size=0
- EXITCODE=$?
- };
- done
- DIFF_COUNT=$(($DIFF_COUNT + 1))
-};
-
-
-update_state()
-{
- get_replicate_filename $TARGET
- TIMESTAMP_LINE=`grep "^timestamp" <$REPLICATE_DIR/$REPLICATE_FILENAME.state.txt`
- while [[ -z $TIMESTAMP_LINE ]]; do
- {
- sleep 5
- TIMESTAMP_LINE=`grep "^timestamp" <$REPLICATE_DIR/$REPLICATE_FILENAME.state.txt`
- }; done
- DATA_VERSION=${TIMESTAMP_LINE:10}
-};
-
-
-echo >>$DB_DIR/apply_osc_to_db.log
-
-mkdir -p $DB_DIR/augmented_diffs/
-DIFF_COUNT=0
-
-# update_state
-
-pushd "$EXEC_DIR"
-
-while [[ true ]]; do
-{
- if [[ $START == "auto" ]]; then
- {
- START=`cat $DB_DIR/replicate_id`
- }; fi
-
- echo "`date -u '+%F %T'`: updating from $START" >>$DB_DIR/apply_osc_to_db.log
-
- TEMP_DIR=`mktemp -d /tmp/osm-3s_update_XXXXXX`
- collect_minute_diffs $TEMP_DIR
-
- if [[ $TARGET -gt $START ]]; then
- {
- echo "`date -u '+%F %T'`: updating to $TARGET" >>$DB_DIR/apply_osc_to_db.log
-
- update_state
- apply_minute_diffs $TEMP_DIR
- echo "$TARGET" >$DB_DIR/replicate_id
-
- echo "`date -u '+%F %T'`: update complete" $TARGET >>$DB_DIR/apply_osc_to_db.log
- };
- else
- {
- sleep 5
- }; fi
-
- rm -f $TEMP_DIR/*
- rmdir $TEMP_DIR
-
- START=$TARGET
-}; done
diff --git a/0.7.55/bin/fetch_osc.sh b/0.7.55/bin/fetch_osc.sh
deleted file mode 100755
index a9d163f..0000000
--- a/0.7.55/bin/fetch_osc.sh
+++ /dev/null
@@ -1,163 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht
-#
-# This file is part of Overpass_API.
-#
-# Overpass_API is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# Overpass_API is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with Overpass_API. If not, see .
-
-if [[ -z $1 ]]; then
-{
- echo Usage: $0 Replicate_id Source_dir Local_dir [Sleep]
- exit 0
-};
-fi
-
-REPLICATE_ID=$1
-SOURCE_DIR=$2
-LOCAL_DIR=$3
-SLEEP_BETWEEN_DLS=15 # How long to sleep between download attempts (sec). Default: 15. See also FAIL_COUNTER_ALERT
-FAIL_COUNTER_ALERT=20 # After how many sleep cycles do we get nervous
-FAIL_COUNTER=0
-FILE_PANIC=
-
-if [[ ! -d $LOCAL_DIR ]];
- then {
- mkdir $LOCAL_DIR
-};
-fi
-
-if [[ $REPLICATE_ID == "auto" ]] ; then
- REPLICATE_ID=`find $LOCAL_DIR -type f -name '*state.txt' -not -size 0 | sort | tail -n 1 | sed -e "s#^$LOCAL_DIR##" -e 's#[./]##g' -e 's#statetxt##' -e 's/^0\+//g'`
-
- EXEC_DIR="`dirname $0`/"
- DB_DIR=`$EXEC_DIR/dispatcher --show-dir`
-
- if [[ ! -d $DB_DIR ]] ; then
- echo "Can't find DB_DIR. Returned value: $DB_DIR"
- exit 1;
- fi
-
- if [[ "x$REPLICATE_ID" == "x" ]] ; then
- REPLICATE_ID=`cat $DB_DIR/replicate_id`
- fi
- if [[ "x$REPLICATE_ID" == "x" ]] ; then
- echo "Could not determine REPLICATE_ID. Exiting"
- exit 1
- fi
-fi
-
-# $1 - remote source
-# $2 - local destination
-fetch_file()
-{
- wget -nv -O "$2" "$1"
-};
-
-retry_fetch_file()
-{
- FILE_PANIC=
- FAIL_COUNTER=0
- if [[ ! -s "$2" ]]; then {
- fetch_file "$1" "$2"
- if [[ "$3" == "gzip" ]]; then {
- gunzip -t <"$2"
- if [[ $? -ne 0 ]]; then {
- rm "$2"
- }; fi
- }; fi
- }; fi
- until [[ -s "$2" || $FAIL_COUNTER -ge $FAIL_COUNTER_ALERT ]]; do {
- FAIL_COUNTER=$(($FAIL_COUNTER+1))
- sleep $SLEEP_BETWEEN_DLS
- fetch_file "$1" "$2"
- if [[ "$3" == "gzip" ]]; then {
- gunzip -t <"$2"
- if [[ $? -ne 0 ]]; then {
- rm "$2"
- }; fi
- }; fi
- }; done
- if [[ $FAIL_COUNTER -ge $FAIL_COUNTER_ALERT ]]; then
- FILE_PANIC="true"
- fi
-};
-
-file_panic()
-{
- echo "fetch_osc()@"`date -u "+%F %T"`": upstream_delay $REPLICATE_ID" >>$LOCAL_DIR/fetch_osc.log
- REPLICATE_ID=$(($REPLICATE_ID - 1))
-
- printf -v TDIGIT3 %03u $(($REPLICATE_ID % 1000))
- ARG=$(($REPLICATE_ID / 1000))
- printf -v TDIGIT2 %03u $(($ARG % 1000))
- ARG=$(($ARG / 1000))
- printf -v TDIGIT1 %03u $ARG
-
- FILE_PANIC="true"
- until [[ ! -n $FILE_PANIC ]]; do {
- retry_fetch_file "$REMOTE_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.new.osc.gz" "gzip"
- }; done
- FILE_PANIC="true"
- until [[ ! -n $FILE_PANIC ]]; do {
- retry_fetch_file "$REMOTE_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.new.state.txt" "text"
- }; done
-
- RES_GZIP=`diff -q "$LOCAL_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.new.osc.gz"`
- RES_TEXT=`diff -q "$LOCAL_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.new.state.txt"`
- if [[ -n $RES_GZIP || -n $RES_TEXT ]]; then
- echo "fetch_osc()@"`date -u "+%F %T"`": file_panic $REPLICATE_ID" >>$LOCAL_DIR/fetch_osc.log
- echo "fetch_osc()@"`date -u "+%F %T"`": $RES_GZIP" >>$LOCAL_DIR/fetch_osc.log
- echo "fetch_osc()@"`date -u "+%F %T"`": $RES_TEXT" >>$LOCAL_DIR/fetch_osc.log
- exit 1
- fi
-
- rm "$LOCAL_PATH/$TDIGIT3.new.osc.gz"
- rm "$LOCAL_PATH/$TDIGIT3.new.state.txt"
-};
-
-fetch_minute_diff()
-{
- printf -v TDIGIT3 %03u $(($REPLICATE_ID % 1000))
- ARG=$(($REPLICATE_ID / 1000))
- printf -v TDIGIT2 %03u $(($ARG % 1000))
- ARG=$(($ARG / 1000))
- printf -v TDIGIT1 %03u $ARG
-
- LOCAL_PATH="$LOCAL_DIR/$TDIGIT1/$TDIGIT2"
- REMOTE_PATH="$SOURCE_DIR/$TDIGIT1/$TDIGIT2"
- mkdir -p "$LOCAL_DIR/$TDIGIT1/$TDIGIT2"
-
- retry_fetch_file "$REMOTE_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.osc.gz" "gzip"
- if [[ -n $FILE_PANIC ]]; then
- file_panic
- fi
- retry_fetch_file "$REMOTE_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.state.txt" "text"
- if [[ -n $FILE_PANIC ]]; then
- file_panic
- fi
-
- TIMESTAMP_LINE=`grep timestamp $LOCAL_DIR/$TDIGIT1/$TDIGIT2/$TDIGIT3.state.txt`
- TIMESTAMP=${TIMESTAMP_LINE:10}
-};
-
-while [[ true ]];
-do
-{
- REPLICATE_ID=$(($REPLICATE_ID + 1))
- fetch_minute_diff
- echo "fetch_osc()@"`date -u "+%F %T"`": new_replicate_diff $REPLICATE_ID $TIMESTAMP" >>$LOCAL_DIR/fetch_osc.log
- sleep 1
-};
-done
diff --git a/0.7.55/bin/update_overpass.sh b/0.7.55/bin/update_overpass.sh
new file mode 100644
index 0000000..328e375
--- /dev/null
+++ b/0.7.55/bin/update_overpass.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env sh
+
+(
+ set -e
+
+ if [ "x$OVERPASS_META" = "xattic" ] ; then
+ META="--keep-attic"
+ elif [ "x$OVERPASS_META" = "xyes" ] ; then
+ META="--meta"
+ else
+ META=""
+ fi
+
+ if [ ! -d /db/diffs ] ; then
+ mkdir /db/diffs
+ fi
+
+ while `true` ; do
+ (
+ if [ ! -e /db/diffs/changes.osm ] ; then
+ ! /app/venv/bin/pyosmium-get-changes --server $OVERPASS_DIFF_URL -o /db/diffs/changes.osm -f /db/replicate_id
+ OSMIUM_STATUS=$?
+ if [ $OSMIUM_STATUS -eq 1 ]; then
+ echo "There are still some updates remainging"
+ fi
+ if [ $OSMIUM_STATUS -eq 2 ]; then
+ echo "Failure downloading updates"
+ exit 0
+ fi
+ else
+ echo "/db/diffs/changes.osm exists. Trying to apply again."
+ fi
+ /app/bin/update_from_dir --osc-dir=/db/diffs/ $META
+ rm /db/diffs/changes.osm
+ )
+ sleep 60
+ done
+) 2>&1 | tee -a /db/changes.log
\ No newline at end of file
diff --git a/0.7.55/docker-entrypoint.sh b/0.7.55/docker-entrypoint.sh
index 5a43603..cb69627 100644
--- a/0.7.55/docker-entrypoint.sh
+++ b/0.7.55/docker-entrypoint.sh
@@ -6,13 +6,15 @@ OVERPASS_META=${OVERPASS_META:-no}
OVERPASS_MODE=${OVERPASS_MODE:-clone}
OVERPASS_COMPRESSION=${OVERPASS_COMPRESSION:-gz}
+if [[ "$OVERPASS_META" == "attic" ]] ; then
+ META="--keep-attic"
+else
+ META="--meta"
+fi
+
+
if [ ! -d /db/db ] ; then
if [ "$OVERPASS_MODE" = "clone" ]; then
- if [[ "$OVERPASS_META" == "attic" ]] ; then
- META="--keep-attic"
- else
- META="--meta=$OVERPASS_META"
- fi
mkdir -p /db/db \
&& /app/bin/download_clone.sh --db-dir=/db/db --source=http://dev.overpass-api.de/api_drolbr/ $META \
&& cp -r /app/etc/rules /db/db \
@@ -22,10 +24,21 @@ if [ ! -d /db/db ] ; then
fi
if [ "$OVERPASS_MODE" = "init" ]; then
- lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet; exit" \
- && /app/bin/init_osm3s.sh /db/planet /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION" \
- && echo $OVERPASS_PLANET_SEQUENCE_ID > /db/db/replicate_id \
- && rm /db/planet \
+ lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet.osm.bz2; exit" \
+ && /app/bin/init_osm3s.sh /db/planet.osm.bz2 /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION --map-compression-method=$OVERPASS_COMPRESSION"\
+ && echo "Database created. Now updating it." && (
+ ! /app/venv/bin/pyosmium-get-changes -O /db/planet.osm.bz2 --server $OVERPASS_DIFF_URL -o /db/changes.osm -f /db/replicate_id
+ OSMIUM_STATUS=$?
+ if [ $OSMIUM_STATUS -eq 1 ]; then
+ echo "There are still some updates remainging"
+ fi
+ if [ $OSMIUM_STATUS -eq 2 ]; then
+ echo "Failure downloading updates"
+ exit 0
+ fi
+ (cat /db/changes.osm | /app/bin/update_database --db-dir=/db/db $META --compression-method=$OVERPASS_COMPRESSION) 2>&1 | tee -a /db/changes.log
+ ) \
+ && rm /db/planet.osm.bz2 /db/changes.osm \
&& cp -r /app/etc/rules /db/db \
&& chown -R overpass:overpass /db \
&& echo "Overpass ready, you can start your container with docker start"
diff --git a/0.7.55/etc/supervisord.conf b/0.7.55/etc/supervisord.conf
index 6c4623b..fc67b74 100644
--- a/0.7.55/etc/supervisord.conf
+++ b/0.7.55/etc/supervisord.conf
@@ -17,18 +17,12 @@ redirect_stderr=true
user=nginx
priority=3
-[program:fetch_diff]
-command=/app/bin/fetch_osc.sh auto %(ENV_OVERPASS_DIFF_URL)s /db/diffs
+[program:update_overpass]
+command=/app/bin/update_overpass.sh
user=overpass
redirect_stderr=true
priority=5
-[program:apply_diff]
-command=/app/bin/apply_osc_to_db.sh /db/diffs/ auto --meta=%(ENV_OVERPASS_META)s
-redirect_stderr=true
-user=overpass
-priority=4
-
[program:dispatcher_areas]
command=nice /app/bin/dispatcher --areas --db-dir="/db/db"
user=overpass
diff --git a/0.7.55/requirements.txt b/0.7.55/requirements.txt
new file mode 100644
index 0000000..8314012
--- /dev/null
+++ b/0.7.55/requirements.txt
@@ -0,0 +1 @@
+osmium
diff --git a/Dockerfile.template b/Dockerfile.template
index 637b452..4eecf18 100644
--- a/Dockerfile.template
+++ b/Dockerfile.template
@@ -2,26 +2,33 @@ FROM nginx:1.14
RUN addgroup overpass && adduser --home /db --disabled-password --gecos overpass --ingroup overpass overpass
+ADD http://dev.overpass-api.de/releases/osm-3s_v{version}.tar.gz /app/src.tar.gz
+
RUN apt-get update \
&& apt-get install --no-install-recommends --no-install-suggests -y \
autoconf \
automake \
+ bash \
+ bzip2 \
ca-certificates \
expat \
- libexpat1-dev \
+ fcgiwrap \
g++ \
+ lftp \
+ libexpat1-dev \
liblz4-1 \
liblz4-dev \
libtool \
m4 \
make \
+ osmium-tool \
+ python3 \
+ python3-venv \
+ supervisor \
+ wget \
zlib1g \
- zlib1g-dev
-
-
-ADD http://dev.overpass-api.de/releases/osm-3s_v{version}.tar.gz /app/src.tar.gz
-
-RUN mkdir -p /app/src \
+ zlib1g-dev \
+ && mkdir -p /app/src \
&& cd /app/src \
&& tar -x -z --strip-components 1 -f ../src.tar.gz \
&& autoscan \
@@ -45,26 +52,28 @@ RUN mkdir -p /app/src \
&& apt-get autoremove -y \
&& mkdir -p /db/diffs /app/etc \
&& cp -r /app/src/rules /app/etc/rules \
- && rm -rf /app/src /app/src.tar.gz
+ && rm -rf /app/src /app/src.tar.gz \
+ && rm -rf /var/lib/apt/lists/*
-RUN apt-get install --no-install-recommends --no-install-suggests -y \
- supervisor \
- bash \
- lftp \
- wget \
- fcgiwrap \
- bzip2
+COPY requirements.txt /app/
+RUN python3 -m venv /app/venv \
+ && /app/venv/bin/pip install -r /app/requirements.txt
RUN mkdir /nginx && chown nginx:nginx /nginx && chown -R overpass:overpass /db
+
COPY etc/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+
COPY etc/nginx-overpass.conf /etc/nginx/nginx.conf
-COPY bin/apply_osc_to_db.sh bin/fetch_osc.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/
+
+COPY bin/update_overpass.sh bin/rules_loop.sh bin/dispatcher_start.sh /app/bin/
+
COPY docker-entrypoint.sh /app/
-RUN chmod a+x /app/docker-entrypoint.sh /app/bin/apply_osc_to_db.sh /app/bin/fetch_osc.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh
-VOLUME /db
+
+RUN chmod a+x /app/docker-entrypoint.sh /app/bin/update_overpass.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh
ENV OVERPASS_RULES_LOAD 1
+
EXPOSE 80
# CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"]
CMD ["/app/docker-entrypoint.sh"]
diff --git a/README.md b/README.md
index f6805ba..f08eb16 100644
--- a/README.md
+++ b/README.md
@@ -6,7 +6,6 @@ To do it properly it needs a bit of guidance in form of enviroment variables.
* `OVERPASS_META` - `yes`, `no` or `attic` - passed to Overpass as `--meta` or `--keep-attic`
* `OVERPASS_DIFF_URL` - url to diff's directory for updating the instance (eg. https://planet.openstreetmap.org/replication/minute/)
* `OVERPASS_PLANET_URL` - url to "planet" file in `init` mode
-* `OVERPASS_PLANET_SEQUENCE_ID` - sequence identifier corresponding to planet file above. All files after this one will be applied
* `OVERPASS_COMPRESSION` - takes values of `no`, `gz` or `lz4`. Specifies compression mode of the Overpass database.
Ony useful in `init` mode. Defaults to `gz`
* `OVERPASS_RULES_LOAD` - desired load generated by areas generation. Controls how long the script will sleep before regenerating
@@ -19,9 +18,7 @@ then data is copied from http://dev.overpass-api.de/api_drolbr/ and then updated
covering whole world. This mode works only with minute diffs.
In `init` mode you need to point `OVERPASS_PLANET_URL` to address with planet (partial) dump. This file will be downloaded,
-indexed by Overpass and later - updated using `OVERPASS_PLANET_SEQUENCE_ID` and `OVERPASS_DIFF_URL`. You need to check which
-sequence number is for your planet file. Take it from desctiption or just take a sequence number a day before your planet
-file is dated.
+indexed by Overpass and later - updated using `OVERPASS_DIFF_URL`.
Start initalization mode with `-i` and `-t` options to `docker run` so you will have a chance to monitor the progress of
file downloads etc.
@@ -43,7 +40,6 @@ docker run \
-e OVERPASS_MODE=init \
-e OVERPASS_PLANET_URL=http://download.geofabrik.de/europe/monaco-latest.osm.bz2 \
-e OVERPASS_DIFF_URL=http://download.openstreetmap.fr/replication/europe/monaco/minute/ \
- -e OVERPASS_PLANET_SEQUENCE_ID=2946200 \
-e OVERPASS_RULES_LOAD=10 \
-v /big/docker/overpass_db/:/db \
-p 12345:80 \
diff --git a/bin/apply_osc_to_db.sh b/bin/apply_osc_to_db.sh
deleted file mode 100755
index 94d1879..0000000
--- a/bin/apply_osc_to_db.sh
+++ /dev/null
@@ -1,163 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht
-#
-# This file is part of Overpass_API.
-#
-# Overpass_API is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# Overpass_API is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with Overpass_API. If not, see .
-
-if [[ -z $3 ]]; then
-{
- echo "Usage: $0 replicate_dir start_id --meta=(attic|yes|no)"
- exit 0
-}; fi
-
-EXEC_DIR="`dirname $0`/"
-if [[ ! ${EXEC_DIR:0:1} == "/" ]]; then
-{
- EXEC_DIR="`pwd`/$EXEC_DIR"
-}; fi
-
-DB_DIR=`$EXEC_DIR/dispatcher --show-dir`
-
-if [[ ! -d $DB_DIR ]] ; then
- echo "Can't find DB_DIR. Returned value: $DB_DIR"
- exit 1;
-fi
-
-REPLICATE_DIR="$1"
-if [[ ! ${REPLICATE_DIR:0:1} == "/" ]]; then
-{
- REPLICATE_DIR="`pwd`/$REPLICATE_DIR"
-}; fi
-
-START=$2
-META=
-
-if [[ $3 == "--meta=attic" ]]; then
- META="--keep-attic"
-elif [[ $3 == "--meta=yes" || $3 == "--meta" ]]; then
- META="--meta"
-elif [[ $3 == "--meta=no" ]]; then
- META=
-else
-{
- echo "You must specify --meta=yes or --meta=no"
- exit 0
-}; fi
-
-PRODUCE_DIFF=
-
-get_replicate_filename()
-{
- printf -v TDIGIT3 %03u $(($1 % 1000))
- ARG=$(($1 / 1000))
- printf -v TDIGIT2 %03u $(($ARG % 1000))
- ARG=$(($ARG / 1000))
- printf -v TDIGIT1 %03u $ARG
- REPLICATE_TRUNK_DIR=$TDIGIT1/$TDIGIT2/
- REPLICATE_FILENAME=$TDIGIT1/$TDIGIT2/$TDIGIT3
-};
-
-
-collect_minute_diffs()
-{
- TEMP_DIR=$1
- TARGET=$(($START + 1))
-
- get_replicate_filename $TARGET
-
- while [[ ( -s $REPLICATE_DIR/$REPLICATE_FILENAME.state.txt ) && ( $(($START + 1440)) -ge $(($TARGET)) ) && ( `du -m $TEMP_DIR | awk '{ print $1; }'` -le 512 ) ]];
- do
- {
- printf -v TARGET_FILE %09u $TARGET
- gunzip <$REPLICATE_DIR/$REPLICATE_FILENAME.osc.gz >$TEMP_DIR/$TARGET_FILE.osc
- TARGET=$(($TARGET + 1))
- get_replicate_filename $TARGET
- };
- done
- TARGET=$(($TARGET - 1))
-};
-
-
-apply_minute_diffs()
-{
- ./update_from_dir --osc-dir=$1 --version=$DATA_VERSION $META --flush-size=0
- EXITCODE=$?
- while [[ $EXITCODE -ne 0 ]];
- do
- {
- sleep 60
- ./update_from_dir --osc-dir=$1 --version=$DATA_VERSION $META --flush-size=0
- EXITCODE=$?
- };
- done
- DIFF_COUNT=$(($DIFF_COUNT + 1))
-};
-
-
-update_state()
-{
- get_replicate_filename $TARGET
- TIMESTAMP_LINE=`grep "^timestamp" <$REPLICATE_DIR/$REPLICATE_FILENAME.state.txt`
- while [[ -z $TIMESTAMP_LINE ]]; do
- {
- sleep 5
- TIMESTAMP_LINE=`grep "^timestamp" <$REPLICATE_DIR/$REPLICATE_FILENAME.state.txt`
- }; done
- DATA_VERSION=${TIMESTAMP_LINE:10}
-};
-
-
-echo >>$DB_DIR/apply_osc_to_db.log
-
-mkdir -p $DB_DIR/augmented_diffs/
-DIFF_COUNT=0
-
-# update_state
-
-pushd "$EXEC_DIR"
-
-while [[ true ]]; do
-{
- if [[ $START == "auto" ]]; then
- {
- START=`cat $DB_DIR/replicate_id`
- }; fi
-
- echo "`date -u '+%F %T'`: updating from $START" >>$DB_DIR/apply_osc_to_db.log
-
- TEMP_DIR=`mktemp -d /tmp/osm-3s_update_XXXXXX`
- collect_minute_diffs $TEMP_DIR
-
- if [[ $TARGET -gt $START ]]; then
- {
- echo "`date -u '+%F %T'`: updating to $TARGET" >>$DB_DIR/apply_osc_to_db.log
-
- update_state
- apply_minute_diffs $TEMP_DIR
- echo "$TARGET" >$DB_DIR/replicate_id
-
- echo "`date -u '+%F %T'`: update complete" $TARGET >>$DB_DIR/apply_osc_to_db.log
- };
- else
- {
- sleep 5
- }; fi
-
- rm -f $TEMP_DIR/*
- rmdir $TEMP_DIR
-
- START=$TARGET
-}; done
diff --git a/bin/fetch_osc.sh b/bin/fetch_osc.sh
deleted file mode 100755
index a9d163f..0000000
--- a/bin/fetch_osc.sh
+++ /dev/null
@@ -1,163 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht
-#
-# This file is part of Overpass_API.
-#
-# Overpass_API is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# Overpass_API is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with Overpass_API. If not, see .
-
-if [[ -z $1 ]]; then
-{
- echo Usage: $0 Replicate_id Source_dir Local_dir [Sleep]
- exit 0
-};
-fi
-
-REPLICATE_ID=$1
-SOURCE_DIR=$2
-LOCAL_DIR=$3
-SLEEP_BETWEEN_DLS=15 # How long to sleep between download attempts (sec). Default: 15. See also FAIL_COUNTER_ALERT
-FAIL_COUNTER_ALERT=20 # After how many sleep cycles do we get nervous
-FAIL_COUNTER=0
-FILE_PANIC=
-
-if [[ ! -d $LOCAL_DIR ]];
- then {
- mkdir $LOCAL_DIR
-};
-fi
-
-if [[ $REPLICATE_ID == "auto" ]] ; then
- REPLICATE_ID=`find $LOCAL_DIR -type f -name '*state.txt' -not -size 0 | sort | tail -n 1 | sed -e "s#^$LOCAL_DIR##" -e 's#[./]##g' -e 's#statetxt##' -e 's/^0\+//g'`
-
- EXEC_DIR="`dirname $0`/"
- DB_DIR=`$EXEC_DIR/dispatcher --show-dir`
-
- if [[ ! -d $DB_DIR ]] ; then
- echo "Can't find DB_DIR. Returned value: $DB_DIR"
- exit 1;
- fi
-
- if [[ "x$REPLICATE_ID" == "x" ]] ; then
- REPLICATE_ID=`cat $DB_DIR/replicate_id`
- fi
- if [[ "x$REPLICATE_ID" == "x" ]] ; then
- echo "Could not determine REPLICATE_ID. Exiting"
- exit 1
- fi
-fi
-
-# $1 - remote source
-# $2 - local destination
-fetch_file()
-{
- wget -nv -O "$2" "$1"
-};
-
-retry_fetch_file()
-{
- FILE_PANIC=
- FAIL_COUNTER=0
- if [[ ! -s "$2" ]]; then {
- fetch_file "$1" "$2"
- if [[ "$3" == "gzip" ]]; then {
- gunzip -t <"$2"
- if [[ $? -ne 0 ]]; then {
- rm "$2"
- }; fi
- }; fi
- }; fi
- until [[ -s "$2" || $FAIL_COUNTER -ge $FAIL_COUNTER_ALERT ]]; do {
- FAIL_COUNTER=$(($FAIL_COUNTER+1))
- sleep $SLEEP_BETWEEN_DLS
- fetch_file "$1" "$2"
- if [[ "$3" == "gzip" ]]; then {
- gunzip -t <"$2"
- if [[ $? -ne 0 ]]; then {
- rm "$2"
- }; fi
- }; fi
- }; done
- if [[ $FAIL_COUNTER -ge $FAIL_COUNTER_ALERT ]]; then
- FILE_PANIC="true"
- fi
-};
-
-file_panic()
-{
- echo "fetch_osc()@"`date -u "+%F %T"`": upstream_delay $REPLICATE_ID" >>$LOCAL_DIR/fetch_osc.log
- REPLICATE_ID=$(($REPLICATE_ID - 1))
-
- printf -v TDIGIT3 %03u $(($REPLICATE_ID % 1000))
- ARG=$(($REPLICATE_ID / 1000))
- printf -v TDIGIT2 %03u $(($ARG % 1000))
- ARG=$(($ARG / 1000))
- printf -v TDIGIT1 %03u $ARG
-
- FILE_PANIC="true"
- until [[ ! -n $FILE_PANIC ]]; do {
- retry_fetch_file "$REMOTE_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.new.osc.gz" "gzip"
- }; done
- FILE_PANIC="true"
- until [[ ! -n $FILE_PANIC ]]; do {
- retry_fetch_file "$REMOTE_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.new.state.txt" "text"
- }; done
-
- RES_GZIP=`diff -q "$LOCAL_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.new.osc.gz"`
- RES_TEXT=`diff -q "$LOCAL_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.new.state.txt"`
- if [[ -n $RES_GZIP || -n $RES_TEXT ]]; then
- echo "fetch_osc()@"`date -u "+%F %T"`": file_panic $REPLICATE_ID" >>$LOCAL_DIR/fetch_osc.log
- echo "fetch_osc()@"`date -u "+%F %T"`": $RES_GZIP" >>$LOCAL_DIR/fetch_osc.log
- echo "fetch_osc()@"`date -u "+%F %T"`": $RES_TEXT" >>$LOCAL_DIR/fetch_osc.log
- exit 1
- fi
-
- rm "$LOCAL_PATH/$TDIGIT3.new.osc.gz"
- rm "$LOCAL_PATH/$TDIGIT3.new.state.txt"
-};
-
-fetch_minute_diff()
-{
- printf -v TDIGIT3 %03u $(($REPLICATE_ID % 1000))
- ARG=$(($REPLICATE_ID / 1000))
- printf -v TDIGIT2 %03u $(($ARG % 1000))
- ARG=$(($ARG / 1000))
- printf -v TDIGIT1 %03u $ARG
-
- LOCAL_PATH="$LOCAL_DIR/$TDIGIT1/$TDIGIT2"
- REMOTE_PATH="$SOURCE_DIR/$TDIGIT1/$TDIGIT2"
- mkdir -p "$LOCAL_DIR/$TDIGIT1/$TDIGIT2"
-
- retry_fetch_file "$REMOTE_PATH/$TDIGIT3.osc.gz" "$LOCAL_PATH/$TDIGIT3.osc.gz" "gzip"
- if [[ -n $FILE_PANIC ]]; then
- file_panic
- fi
- retry_fetch_file "$REMOTE_PATH/$TDIGIT3.state.txt" "$LOCAL_PATH/$TDIGIT3.state.txt" "text"
- if [[ -n $FILE_PANIC ]]; then
- file_panic
- fi
-
- TIMESTAMP_LINE=`grep timestamp $LOCAL_DIR/$TDIGIT1/$TDIGIT2/$TDIGIT3.state.txt`
- TIMESTAMP=${TIMESTAMP_LINE:10}
-};
-
-while [[ true ]];
-do
-{
- REPLICATE_ID=$(($REPLICATE_ID + 1))
- fetch_minute_diff
- echo "fetch_osc()@"`date -u "+%F %T"`": new_replicate_diff $REPLICATE_ID $TIMESTAMP" >>$LOCAL_DIR/fetch_osc.log
- sleep 1
-};
-done
diff --git a/bin/update_overpass.sh b/bin/update_overpass.sh
new file mode 100644
index 0000000..328e375
--- /dev/null
+++ b/bin/update_overpass.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env sh
+
+(
+ set -e
+
+ if [ "x$OVERPASS_META" = "xattic" ] ; then
+ META="--keep-attic"
+ elif [ "x$OVERPASS_META" = "xyes" ] ; then
+ META="--meta"
+ else
+ META=""
+ fi
+
+ if [ ! -d /db/diffs ] ; then
+ mkdir /db/diffs
+ fi
+
+ while `true` ; do
+ (
+ if [ ! -e /db/diffs/changes.osm ] ; then
+ ! /app/venv/bin/pyosmium-get-changes --server $OVERPASS_DIFF_URL -o /db/diffs/changes.osm -f /db/replicate_id
+ OSMIUM_STATUS=$?
+ if [ $OSMIUM_STATUS -eq 1 ]; then
+ echo "There are still some updates remainging"
+ fi
+ if [ $OSMIUM_STATUS -eq 2 ]; then
+ echo "Failure downloading updates"
+ exit 0
+ fi
+ else
+ echo "/db/diffs/changes.osm exists. Trying to apply again."
+ fi
+ /app/bin/update_from_dir --osc-dir=/db/diffs/ $META
+ rm /db/diffs/changes.osm
+ )
+ sleep 60
+ done
+) 2>&1 | tee -a /db/changes.log
\ No newline at end of file
diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh
index 5a43603..cb69627 100755
--- a/docker-entrypoint.sh
+++ b/docker-entrypoint.sh
@@ -6,13 +6,15 @@ OVERPASS_META=${OVERPASS_META:-no}
OVERPASS_MODE=${OVERPASS_MODE:-clone}
OVERPASS_COMPRESSION=${OVERPASS_COMPRESSION:-gz}
+if [[ "$OVERPASS_META" == "attic" ]] ; then
+ META="--keep-attic"
+else
+ META="--meta"
+fi
+
+
if [ ! -d /db/db ] ; then
if [ "$OVERPASS_MODE" = "clone" ]; then
- if [[ "$OVERPASS_META" == "attic" ]] ; then
- META="--keep-attic"
- else
- META="--meta=$OVERPASS_META"
- fi
mkdir -p /db/db \
&& /app/bin/download_clone.sh --db-dir=/db/db --source=http://dev.overpass-api.de/api_drolbr/ $META \
&& cp -r /app/etc/rules /db/db \
@@ -22,10 +24,21 @@ if [ ! -d /db/db ] ; then
fi
if [ "$OVERPASS_MODE" = "init" ]; then
- lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet; exit" \
- && /app/bin/init_osm3s.sh /db/planet /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION" \
- && echo $OVERPASS_PLANET_SEQUENCE_ID > /db/db/replicate_id \
- && rm /db/planet \
+ lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet.osm.bz2; exit" \
+ && /app/bin/init_osm3s.sh /db/planet.osm.bz2 /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION --map-compression-method=$OVERPASS_COMPRESSION"\
+ && echo "Database created. Now updating it." && (
+ ! /app/venv/bin/pyosmium-get-changes -O /db/planet.osm.bz2 --server $OVERPASS_DIFF_URL -o /db/changes.osm -f /db/replicate_id
+ OSMIUM_STATUS=$?
+ if [ $OSMIUM_STATUS -eq 1 ]; then
+ echo "There are still some updates remainging"
+ fi
+ if [ $OSMIUM_STATUS -eq 2 ]; then
+ echo "Failure downloading updates"
+ exit 0
+ fi
+ (cat /db/changes.osm | /app/bin/update_database --db-dir=/db/db $META --compression-method=$OVERPASS_COMPRESSION) 2>&1 | tee -a /db/changes.log
+ ) \
+ && rm /db/planet.osm.bz2 /db/changes.osm \
&& cp -r /app/etc/rules /db/db \
&& chown -R overpass:overpass /db \
&& echo "Overpass ready, you can start your container with docker start"
diff --git a/etc/supervisord.conf b/etc/supervisord.conf
index 6c4623b..fc67b74 100644
--- a/etc/supervisord.conf
+++ b/etc/supervisord.conf
@@ -17,18 +17,12 @@ redirect_stderr=true
user=nginx
priority=3
-[program:fetch_diff]
-command=/app/bin/fetch_osc.sh auto %(ENV_OVERPASS_DIFF_URL)s /db/diffs
+[program:update_overpass]
+command=/app/bin/update_overpass.sh
user=overpass
redirect_stderr=true
priority=5
-[program:apply_diff]
-command=/app/bin/apply_osc_to_db.sh /db/diffs/ auto --meta=%(ENV_OVERPASS_META)s
-redirect_stderr=true
-user=overpass
-priority=4
-
[program:dispatcher_areas]
command=nice /app/bin/dispatcher --areas --db-dir="/db/db"
user=overpass
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..8314012
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1 @@
+osmium
diff --git a/update.py b/update.py
index 55105b5..d8209b6 100644
--- a/update.py
+++ b/update.py
@@ -8,6 +8,9 @@
class VersionFinder(html.parser.HTMLParser):
+ def error(self, message):
+ raise RuntimeError(message)
+
def __init__(self):
super().__init__()
self.versions = []
@@ -28,12 +31,23 @@ def main():
with open("Dockerfile.template") as f:
template = f.read()
for ver in parser.versions:
+ if any((ver.startswith(x) for x in ('0.6', 'eta', '0.7.1', '0.7.2', '0.7.3', '0.7.4', '0.7.50', '0.7.52',
+ '0.7.54.11', # invalid CRC in archive
+ '0.7.51', # no autoconf
+ ))) or \
+ ver == '0.7':
+ # ignore old releases
+ continue
+ if os.path.exists(ver):
+ shutil.rmtree(ver)
os.mkdir(ver)
with open(pathlib.Path(ver) / "Dockerfile", "w+") as f:
f.write(template.format(version=ver))
for i in ("etc", "bin"):
- shutil.copytree(i, pathlib.Path(ver) / i)
+ shutil.copytree(i, pathlib.Path(ver) / i)
shutil.copyfile("docker-entrypoint.sh", pathlib.Path(ver) / "docker-entrypoint.sh")
+ shutil.copyfile("requirements.txt", pathlib.Path(ver) / "requirements.txt")
+
if __name__ == '__main__':
main()