1.1 --- a/examples/teleios/runChain.sh	Fri Dec 14 18:12:51 2012 +0200
     1.2 +++ b/examples/teleios/runChain.sh	Sun Dec 16 14:29:01 2012 +0200
     1.3 @@ -1,25 +1,35 @@
     1.4  #! /bin/bash
     1.5 -
     1.6 -#
     1.7 -# This Source Code Form is subject to the terms of the Mozilla Public
     1.8 -# License, v. 2.0. If a copy of the MPL was not distributed with this
     1.9 -# file, You can obtain one at http://mozilla.org/MPL/2.0/.
    1.10 -#
    1.11 -# Copyright (C) 2010, 2011, 2012, Pyravlos Team
    1.12 -#
    1.13 -# http://www.strabon.di.uoa.gr/
    1.14 +# 
    1.15 +# This Source Code Form is subject to the terms of the Mozilla Public 
    1.16 +# License, v. 2.0. If a copy of the MPL was not distributed with this file, you
    1.17 +# can obtain one at http://mozilla.org/MPL/2.0/. 
    1.18 +# 
    1.19 +# Copyright (C) 2010, 2011, 2012, Pyravlos Team 
    1.20 +# 
    1.21 +# http://www.strabon.di.uoa.gr/ 
    1.22  #
    1.23  
    1.24 -LOC="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
    1.25 +# Command name
    1.26 +cmd="$(basename ${0})" 
    1.27 +# Get the directory where the script resides
    1.28 +loc="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
    1.29  
    1.30 -ENDPOINT="http://teleios4.di.uoa.gr:8080/NOA"
    1.31 -DB="NOA2012"
    1.32 +function help() {                                                               
    1.33 +    echo "Usage: ${cmd} [OPTIONS] "                        
    1.34 +    echo                                                                        
    1.35 +    echo "Execute NOA chain with refinements and measure time."
    1.36 +    echo                                                                        
    1.37 +    echo "OPTIONS can be any of the following"                                  
    1.38 +    echo "  -d,--db   		: PostGIS database"                
    1.39 +    echo "  -e,--endpoint   : Strabon Endpoint"
    1.40 +    echo "  -h,--hotposts	: URL where hotspots are stored"
    1.41 +	echo "  -b,--background	: Background data"                                           
    1.42 +	echo "  -l,--log		: Log file"                                           
    1.43 +}
    1.44  
    1.45 -HOTSPOTS_URL="http://challenge.strabon.di.uoa.gr/MSG1"
    1.46 -ENDPOINT_SCRIPT=../../scripts/endpoint
    1.47 -
    1.48 -logFile="chain.log"
    1.49 -
    1.50 +# If no arguments are given it returns miliseconds from 1970-01-01 00:00:00 UTC
    1.51 +# Else if a time (miliseconds form ...) is given it returns the delta between
    1.52 +# given time and current time
    1.53  function timer()
    1.54  {
    1.55     if [[ $# -eq 0 ]]; then
    1.56 @@ -42,12 +52,63 @@
    1.57     fi
    1.58  }
    1.59  
    1.60 -# find out the postgres service to use
    1.61 -postgres=$(ls -1 /etc/init.d/| grep postgres | head -1)
    1.62 +# Handle the postgres service
    1.63 +# -$1: Command for the service
    1.64 +function handlePostgresService()
    1.65 +{
    1.66 +	# find out the postgres service to use
    1.67 +	postgres=$(ls -1 /etc/init.d/| grep postgres | head -1)
    1.68  
    1.69 -tomcat=
    1.70 -function chooseTomcat()
    1.71 +	echo "Postgres service received command: $1"
    1.72 +	sudo service ${postgres} $1
    1.73 +}
    1.74 +
    1.75 +# Handled a postgres database
    1.76 +# -$1: Command (create/drop/store)
    1.77 +# -$2: Dump file to store (if store is given as $1)
    1.78 +#	   'spatial' to create a spatial database (if create is given as $1)
    1.79 +function handlePostgresDatabase() {
    1.80 +	local command=$1
    1.81 +	local db=$2
    1.82 +	shift; shift
    1.83 +	local options="$*"
    1.84 +	case "${command}" in
    1.85 +		create)
    1.86 +			echo "Creating database ${db}..."
    1.87 +			createdb -U postgres ${db} ${options}	
    1.88 +			;;
    1.89 +		drop)
    1.90 +			if test ! -z "${options}"; then
    1.91 +				echo "ERROR: dropdb takes no extra options"
    1.92 +				echo "options: ${options}"
    1.93 +				exit -1
    1.94 +			fi
    1.95 +			echo "Dropping database ${db}..."
    1.96 +			dropdb -U postgres ${db} 
    1.97 +			;;
    1.98 +		vacuum)
    1.99 +			if test "${options}" = "analyze"; then
   1.100 +				psql -U postgres ${DB} -c 'VACUUM ANALYZE' 
   1.101 +			else
   1.102 +				psql -U postgres ${DB} -c 'VACUUM' 
   1.103 +			fi
   1.104 +			;;
   1.105 +		run)
   1.106 +			if test ! [ -f "${options}" ]; then
   1.107 +				echo "ERROR: No dump file to run"
   1.108 +				exit -1
   1.109 +			fi
   1.110 +			echo "Storing dump file ${options} in database ${db}..."
   1.111 +			psql ${db} -f ${options}
   1.112 +			;;
   1.113 +	esac
   1.114 +}
   1.115 +
   1.116 +# Handle the tomcat service
   1.117 +# -$1: Command for the service
   1.118 +function handleTomcatService()
   1.119  {
   1.120 +	# find out the tomcat service to use
   1.121  	if test -s /etc/fedora-release ; then
   1.122  		tomcat="tomcat"
   1.123  	#elif test -s /etc/centos-release ; then
   1.124 @@ -56,220 +117,303 @@
   1.125  	#elif test -s /etc/SuSE-release ; then
   1.126  	#elif test -s /etc/gentoo-release ; then
   1.127  	elif test -s /etc/lsb-release ; then # Ubuntu
   1.128 -			tomcat=$(ls -1 /etc/init.d/| grep tomcat | head -1)
   1.129 +		tomcat=$(ls -1 /etc/init.d/| grep tomcat | head -1)
   1.130  	elif test -s /etc/debian_version ; then
   1.131  		tomcat="tomcat"
   1.132  	fi
   1.133  
   1.134  	# check for service availability
   1.135  	if ! test -e "/etc/init.d/${tomcat}"; then
   1.136 -		tomcat=
   1.137 +		echo "ERROR: No tomcat service found"
   1.138 +		exit -1
   1.139  	fi
   1.140 +
   1.141 +	echo "Service ${tomcat} received command: $1"
   1.142 +	sudo service ${tomcat} $1
   1.143  }
   1.144  
   1.145 -insertMunicipalities=`cat ${LOC}/insertMunicipalities.rq`
   1.146 -deleteSeaHotspots=`cat ${LOC}/deleteSeaHotspots.rq` # | sed 's/\"/\\\"/g'`
   1.147 -refinePartialSeaHotspots=`cat ${LOC}/refinePartialSeaHotspots.rq` # | sed 's/\"/\\\"/g'`
   1.148 -invalidForFires=`cat ${LOC}/landUseInvalidForFires.rq`
   1.149 -refineTimePersistence=`cat ${LOC}/refineTimePersistence.rq` # | sed 's/\"/\\\"/g'`
   1.150 -discover=`cat ${LOC}/discover.rq`
   1.151 -#InsertMunicipalities =`cat ${LOC}/InsertMunicipalities.sparql` # | sed 's/\"/\\\"/g'`
   1.152 +# get the main version of postgres
   1.153 +function getPostgresMainVersion() {
   1.154 +	echo $(sudo service ${postgres} status | grep -o '.\..' | cut -b 1)
   1.155 +}
   1.156  
   1.157 -# Initialize (stop tomcat, restart postgres, drop/create database, start tomcat)
   1.158 -chooseTomcat
   1.159 -echo "stopping tomcat"
   1.160 -if test -z "${tomcat}"; then
   1.161 -	# work-around for babis (standalone tomcat, with start_tomcat.sh and stop_tomcat.sh scripts)
   1.162 -	stop_tomcat.sh
   1.163 -else
   1.164 -	sudo service ${tomcat} stop
   1.165 -fi
   1.166 +# It stores the backgroud data
   1.167 +# - $1: database
   1.168 +# - $2: backgound data file
   1.169 +function storeBackgroundData() {
   1.170 +	local db=$1
   1.171 +	local bgFile=$2
   1.172  
   1.173 -sudo service ${postgres} restart
   1.174 +	if test -f ${bgFile}; then
   1.175 +		handlePostgresDatabase run ${db} ${bgFile}	
   1.176 +	elif test "${bgFile:0:7}" = "http://"; then
   1.177 +		#curl -s ${bgFile} | tar xz -O | psql -d ${DB}
   1.178 +		wget ${bgFile} -O /tmp/bgFile$$.tar.gz
   1.179 +		tar xz -O /tmp/bgFile$$.tar.gz
   1.180 +		
   1.181 +		handlePostgresDatabase run ${db} /tmp/bgFile$$.tar.tz
   1.182 +		rm /tmp/bgFiile$$.tar.gz
   1.183 +	else
   1.184 +		echo "Backgound file not foung"
   1.185 +		exit -1
   1.186 +	fi
   1.187 +	handlePostgresDatabase vacuum ${db} analyze
   1.188 +}
   1.189 +
   1.190 +# Handle Stabon Endpoint
   1.191 +# - $1: endpoint
   1.192 +# - $2: command (store/query)
   1.193 +# - $2: file/query
   1.194 +function handleStrabonEndpoint(){
   1.195 +	endpoint=$1
   1.196 +	command=$2
   1.197 +	option=$3
   1.198 +
   1.199 +	endpointScript=${loc}/../../scripts/endpoint
   1.200 +	case ${command} in
   1.201 +		store)
   1.202 +			url=${options}
   1.203 +
   1.204 +			# print timestamp
   1.205 +			echo -n "${TIMESTAMP} " >> ${logFile}
   1.206 +
   1.207 +			tmr1=$(timer)
   1.208 +			${endpointScript} store ${endpoint} N-Triples -u ${url}
   1.209 +			tmr2=$(timer)
   1.210 +			printf '%s ' $((tmr2-tmr1)) >> ${logFile}
   1.211 +			;;
   1.212 +		query)
   1.213 +			query=${option}
   1.214 +
   1.215 +			tmr1=$(timer)
   1.216 +			${endpointScript} query ${endpoint} "${query}"
   1.217 +			tmr2=$(timer)
   1.218 +			printf '%s ' $((tmr2-tmr1)) >> ${logFile} 
   1.219 +			;;
   1.220 +		update)
   1.221 +			update=${option}
   1.222 +			tmr1=$(timer)
   1.223 +			${endpointScript} update ${endpoint} "${update}"
   1.224 +			tmr2=$(timer)
   1.225 +			printf '%s ' $((tmr2-tmr1)) >> stderr.txt
   1.226 +			;;
   1.227 +		*)
   1.228 +			echo "ERROR: ..."
   1.229 +			exit -1
   1.230 +			;;
   1.231 +	esac
   1.232 +}
   1.233 +
   1.234 +# $1: queryFile
   1.235 +function makeUpdate() {
   1.236 +	queryFile=$1
   1.237 +	instansiateScript=${loc}/../scripts/instantiate.sh
   1.238 +}
   1.239 +# default values
   1.240 +endpoint="http://pathway.di.uoa.gr:8080/endpoint"
   1.241 +db="NOA2012"
   1.242 +hotspots_url="http://pathway.di.uoag.r/hotspots/2012"
   1.243 +bgFile="http://dev.strabon.di.uoa.gr/rdf/Kallikratis-Coastline-Corine-dump-postgres-9.tgz"
   1.244 +logFile="runChain.log"
   1.245 +
   1.246 +# read script options
   1.247 +while test $# -gt 0 -a "X${1:0:1}" == "X-"; do
   1.248 +    case "${1}" in
   1.249 +        --help)
   1.250 +            help
   1.251 +            exit 0
   1.252 +            ;;
   1.253 +        -e|--endpoint)
   1.254 +            shift
   1.255 +			endpoint=${1}
   1.256 +            shift
   1.257 +            ;;
   1.258 +        -d|--db)
   1.259 +            shift
   1.260 +			db=${1}
   1.261 +			shift
   1.262 +			;;
   1.263 +		-h|--hotspots)
   1.264 +            shift
   1.265 +			hotspots_url=${1}
   1.266 +			shift
   1.267 +			;;
   1.268 +		-b|--background)
   1.269 +            shift
   1.270 +			bgFile=${1}
   1.271 +			shift
   1.272 +			;;
   1.273 +		-l|--log)
   1.274 +            shift
   1.275 +			logFile=${1}
   1.276 +			shift
   1.277 +			;;
   1.278 +		*)
   1.279 +			echo "unknown argument ${1}"
   1.280 +			help
   1.281 +			exit -1
   1.282 +			;;
   1.283 +	esac
   1.284 +done
   1.285 +
   1.286 +echo "endpoint: ${endpoint}"
   1.287 +echo "db: ${db}"
   1.288 +echo "hotspots: ${hotspots_url}"
   1.289 +echo "background: ${bgFile}"
   1.290 +echo "logFile: ${logfile}"
   1.291 +
   1.292 +
   1.293 +# Get queries and updates
   1.294 +insertMunicipalities=`cat ${loc}/insertMunicipalities.rq`
   1.295 +deleteSeaHotspots=`cat ${loc}/deleteSeaHotspots.rq` # | sed 's/\"/\\\"/g'`
   1.296 +refinePartialSeaHotspots=`cat ${loc}/refinePartialSeaHotspots.rq` # | sed 's/\"/\\\"/g'`
   1.297 +invalidForFires=`cat ${loc}/landUseInvalidForFires.rq`
   1.298 +refineTimePersistence=`cat ${loc}/refineTimePersistence.rq` # | sed 's/\"/\\\"/g'`
   1.299 +discover=`cat ${loc}/discover.rq`
   1.300 +InsertMunicipalities=`cat ${loc}/InsertMunicipalities.sparql` # | sed 's/\"/\\\"/g'`
   1.301 +
   1.302 +
   1.303 +#Initialize (stop tomcat, restart postgres, drop/create database, start tomcat)
   1.304 +handleTomcatService stop
   1.305 +handlePostgresService restart
   1.306  
   1.307  # get the main version of postgres
   1.308 -POSTGRES_MAIN_VERSION=$(sudo service ${postgres} status | grep -o '.\..' | cut -b 1)
   1.309 +postgres_main_version=$(getPostgresMainVersion)
   1.310  
   1.311 -#echo "Dropping endpoint database";
   1.312 -dropdb -U postgres ${DB}
   1.313 +handlePostgresDatabase drop ${db}
   1.314 +handlePostgresDatabase create ${db}
   1.315  
   1.316 -#echo "Creating endpoint database"
   1.317 -createdb -U postgres ${DB} 
   1.318 +storeBackgroundData ${db} ${bgFile}
   1.319  
   1.320 -# load data
   1.321 -#curl -s http://dev.strabon.di.uoa.gr/rdf/Kallikratis-Coastline-Corine-dump-postgres-${POSTGRES_MAIN_VERSION}.tgz | tar xz -O | psql -d ${DB}
   1.322 -psql -U postgres -d ${DB} -f /tmp/Kallikratis-Coastline-ExcludeArea-dump.sql
   1.323 -#psql -U postgres ${DB} -c 'VACUUM ANALYZE' 
   1.324 +handleTomcatService start
   1.325  
   1.326 -echo "starting tomcat"
   1.327 -if test -z "${tomcat}"; then
   1.328 -	# work-around for babis (standalone tomcat, with start_tomcat.sh and stop_tomcat.sh scripts)
   1.329 -	start_tomcat.sh
   1.330 -	sleep 2
   1.331 -else
   1.332 -	sudo service ${tomcat} start
   1.333 -fi
   1.334 +echo "Timestamp Store Municipalities DeleteInSea InvalidForFires RefineInCoast
   1.335 +TimePersistence" $ > stderr.txt
   1.336  
   1.337 -echo "initializing database"
   1.338 -echo "Timestamp Store Municipalities DeleteInSea InvalidForFires RefineInCoast TimePersistence" > stderr.txt
   1.339 +years="2007 2008 2010 2011"
   1.340 +for year in "${years}"; do
   1.341 +for month in `seq 5 9`; do
   1.342 +for day in `seq 1 31`; do
   1.343 +for hour in `seq 0 23`; do
   1.344 +for minute in `seq 0 60 5`; do
   1.345  
   1.346 -
   1.347 -#./scripts/endpoint query ${ENDPOINT} "SELECT (COUNT(*) AS ?C) WHERE {?s ?p ?o}"
   1.348 -#sudo -u postgres psql -d endpoint -c 'CREATE INDEX datetime_values_idx_value ON datetime_values USING btree(value)';
   1.349 -#sudo -u postgres psql -d endpoint -c 'VACUUM ANALYZE;';
   1.350 -
   1.351 -#for y in 2007 2008 2010 2011; do
   1.352 -for y in 2012; do
   1.353 -	# get hotpost URLS
   1.354 -	for hot in $(curl -s ${HOTSPOTS_URL}/${y}/ | grep -o '>HMSG.*\.nt' | colrm 1 1); do
   1.355 -		file="${HOTSPOTS_URL}/${y}/${hot}"
   1.356 -
   1.357 -		time_status=$(echo ${hot} | egrep -o '[[:digit:]]{6}_[[:digit:]]{4}')
   1.358 -		
   1.359 -		# get sensor
   1.360 -		SENSOR=$(echo ${hot} | grep -o 'MSG.')
   1.361 -
   1.362 -		# get satellite and set number of acquisitions per hour
   1.363 -		if test "${SENSOR}" = "MSG2"; then
   1.364 -			SAT="METEOSAT9"
   1.365 -
   1.366 -			N_ACQUISITIONS=3.0
   1.367 -		else
   1.368 -			SAT="METEOSAT8"
   1.369 -			SENSOR="MSG1_RSS"
   1.370 -
   1.371 -			N_ACQUISITIONS=7.0
   1.372 -		fi
   1.373 -
   1.374 -		# get time information for acquisition and construct timestamp
   1.375 -		year="20$(expr substr ${time_status} 1 2)"
   1.376 -		month=$(expr substr ${time_status} 3 2)
   1.377 -		day=$(expr substr ${time_status} 5 2)
   1.378 -		time2=$(expr substr ${time_status} 8 2)
   1.379 -		time2="${time2}:$(expr substr ${time_status} 10 2)"
   1.380 +		month=$(printf "%02d" ${month})
   1.381 +		day=$(printf "%02d" ${day})k
   1.382 +		hour=$(printf "%02d" ${hour})
   1.383 +		minute=$(printf "%02d" ${minute})k
   1.384  
   1.385  		# construct timestamp
   1.386 -		TIMESTAMP="${year}-${month}-${day}T${time2}:00"
   1.387 +		timestamp="${year}-${month}-${day}T${hour}:${minute}:00"
   1.388  
   1.389 -		# store file
   1.390 -		echo -n "storing " $file; echo; echo; 
   1.391 -		# ${countTime} ./strabon -db endpoint store $file
   1.392 +		fileURL="HMSG1_RSS_039_s7_${year}${month}${day}_${hour}${minute}.hotspots.nt"
   1.393 +		handleStrabonEndpoint store ${fileURL} ${fileURL}
   1.394  
   1.395 -		# print timestamp
   1.396 -		echo -n "${TIMESTAMP} " >> stderr.txt
   1.397 +##		# sudo -u postgres psql -d endpoint -c 'VACUUM ANALYZE;';
   1.398  
   1.399 -		tmr1=$(timer)
   1.400 -		${ENDPOINT_SCRIPT} store ${ENDPOINT} N-Triples -u ${file}
   1.401 -		tmr2=$(timer)
   1.402 -		printf '%s ' $((tmr2-tmr1)) >> stderr.txt
   1.403  
   1.404 -		# sudo -u postgres psql -d endpoint -c 'VACUUM ANALYZE;';
   1.405 -
   1.406 -		echo;echo;echo;echo "File ${file} stored!" >> ${logFile}
   1.407 -
   1.408 -		# insertMunicipalities
   1.409 -		echo -n "inserting Municipalities " ;echo; echo; echo;
   1.410 -		# query=`echo "${insertMunicipalities}" `
   1.411 -		# ${countTime} ./strabon -db endpoint update "${query}"
   1.412 -
   1.413 -		tmr1=$(timer)
   1.414 -
   1.415 -		query=`echo "${insertMunicipalities}" | sed "s/TIMESTAMP/${TIMESTAMP}/g" | \
   1.416 -		sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
   1.417 -		sed "s/SENSOR/${SENSOR}/g"`
   1.418 -
   1.419 -		${ENDPOINT_SCRIPT} update ${ENDPOINT} "${query}"
   1.420 -		
   1.421 -		tmr2=$(timer)
   1.422 -		printf '%s ' $((tmr2-tmr1)) >>stderr.txt
   1.423 -		echo;echo;echo;echo "File ${file} inserted Municipalities!"
   1.424 -
   1.425 -		# execute an explicit VACUUM ANALYZE when a query takes longer than it should
   1.426 -		duration=$((tmr2-tmr1))
   1.427 -		if test ${duration} -ge 30000; then
   1.428 -			psql -U postgres ${DB} -c 'VACUUM ANALYZE' 
   1.429 -			echo "Explicit VACUUM ANALYZE"
   1.430 -		fi
   1.431 -		
   1.432 -		# deleteSeaHotspots
   1.433 -		echo -n "Going to deleteSeaHotspots ${TIMESTAMP} " ;echo; echo; echo;
   1.434 -		query=`echo "${deleteSeaHotspots}" | sed "s/TIMESTAMP/${TIMESTAMP}/g" | \
   1.435 -		sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
   1.436 -		sed "s/SENSOR/${SENSOR}/g"`
   1.437 -		# ${countTime} ./strabon -db endpoint update "${query}"
   1.438 -
   1.439 -		tmr1=$(timer)
   1.440 -		${ENDPOINT_SCRIPT} update ${ENDPOINT} "${query}"
   1.441 -
   1.442 -		tmr2=$(timer)
   1.443 -		printf '%s ' $((tmr2-tmr1)) >>stderr.txt
   1.444 -		echo;echo;echo;echo "File ${file} deleteSeaHotspots done!"
   1.445 -
   1.446 -		# invalidForFires
   1.447 -		echo -n "invalidForFires ${TIMESTAMP} "  ; echo; echo ; echo;
   1.448 -		query=`echo "${invalidForFires}" | sed "s/TIMESTAMP/${TIMESTAMP}/g" | \
   1.449 -		sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
   1.450 -		sed "s/SENSOR/${SENSOR}/g"` 
   1.451 -
   1.452 -		# ${countTime} ./strabon -db endpoint update "${query}"
   1.453 -		tmr1=$(timer)
   1.454 -		${ENDPOINT_SCRIPT} update ${ENDPOINT} "${query}"
   1.455 -		tmr2=$(timer)
   1.456 -		printf '%s ' $((tmr2-tmr1)) >>stderr.txt
   1.457 -		echo "File ${file} invalidForFires done!"
   1.458 - 
   1.459 -		# refinePartialSeaHotspots
   1.460 -		echo -n "refinePartialSeaHotspots ${TIMESTAMP} "  ; echo; echo ; echo;
   1.461 -		query=`echo "${refinePartialSeaHotspots}" | sed "s/TIMESTAMP/${TIMESTAMP}/g" | \
   1.462 -		sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
   1.463 -		sed "s/SENSOR/${SENSOR}/g" |\
   1.464 -		sed "s/SAT/${SAT}/g"`
   1.465 -		# ${countTime} ./strabon -db endpoint update "${query}"
   1.466 -		tmr1=$(timer)
   1.467 -		${ENDPOINT_SCRIPT} update ${ENDPOINT} "${query}"
   1.468 -		tmr2=$(timer)
   1.469 -		printf '%s ' $((tmr2-tmr1)) >>stderr.txt
   1.470 -
   1.471 -		echo "File ${file} refinePartialSeaHotspots done!"
   1.472 -		# echo "Continue?"
   1.473 -		# read a
   1.474 -
   1.475 -		# refineTimePersistence
   1.476 -		echo -n "Going to refineTimePersistence ${TIMESTAMP} ";echo;echo;echo; 
   1.477 -		min_acquisition_time=`date --date="${year}-${month}-${day} ${time2}:00 EEST -30 minutes" +%Y-%m-%dT%H:%M:00`
   1.478 -		query=`echo "${refineTimePersistence}" | sed "s/TIMESTAMP/${TIMESTAMP}/g" | \
   1.479 -		sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
   1.480 -		sed "s/SENSOR/${SENSOR}/g" | \
   1.481 -		sed "s/ACQUISITIONS_IN_HALF_AN_HOUR/${N_ACQUISITIONS}/g" | \
   1.482 -		sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g" |\
   1.483 -		sed "s/SAT/${SAT}/g"`
   1.484 -
   1.485 -		#sudo -u postgres psql -d ${DB} -c 'VACUUM ANALYZE;';
   1.486 -
   1.487 -		tmr1=$(timer)
   1.488 -		${ENDPOINT_SCRIPT} update ${ENDPOINT} "${query}"
   1.489 -		 tmr2=$(timer)
   1.490 -		printf '%s \n' $((tmr2-tmr1)) >>stderr.txt
   1.491 -		echo;echo;echo;echo "File ${file} timePersistence done!"
   1.492 -		# echo "Continue?"
   1.493 -		# read a
   1.494 -
   1.495 -
   1.496 -		# discover
   1.497 -		echo -n "Going to discover ${TIMESTAMP} ";echo;echo;echo; 
   1.498 -		min_acquisition_time=`date --date="${year}-${month}-${day} 00:00 EEST" +%Y-%m-%dT%H:%M:00`
   1.499 -		max_acquisition_time=`date --date="${year}-${month}-${day} 23:59 EEST" +%Y-%m-%dT%H:%M:00`
   1.500 -		query=`echo "${discover}" | \
   1.501 -			sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
   1.502 -			sed "s/SENSOR/${SENSOR}/g" | \
   1.503 -			sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g" |\
   1.504 -			sed "s/MAX_ACQUISITION_TIME/${max_acquisition_time}/g"`
   1.505 -			
   1.506 -		tmr1=$(timer)
   1.507 -		${ENDPOINT_SCRIPT} query ${ENDPOINT} "${query}"
   1.508 -		tmr2=$(timer)
   1.509 -		printf '%s \n' $((tmr2-tmr1)) >>discover.txt
   1.510 -		echo;echo;echo;echo "Discovered hotspots done!"
   1.511 -	done
   1.512 -done
   1.513 -
   1.514 +##		# insertMunicipalities
   1.515 +##		echo -n "inserting Municipalities " ;echo; echo; echo;
   1.516 +##		# query=`echo "${insertMunicipalities}" `
   1.517 +##		# ${countTime} ./strabon -db endpoint update "${query}"
   1.518 +##
   1.519 +##		tmr1=$(timer)
   1.520 +##
   1.521 +##		query=`echo "${insertMunicipalities}" | sed "s/TIMESTAMP/${TIMESTAMP}/g" | \
   1.522 +##		sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
   1.523 +##		sed "s/SENSOR/${SENSOR}/g"`
   1.524 +##
   1.525 +##		${ENDPOINT_SCRIPT} update ${ENDPOINT} "${query}"
   1.526 +##		
   1.527 +##		tmr2=$(timer)
   1.528 +##		printf '%s ' $((tmr2-tmr1)) >>stderr.txt
   1.529 +##		echo;echo;echo;echo "File ${file} inserted Municipalities!"
   1.530 +##
   1.531 +##		# execute an explicit VACUUM ANALYZE when a query takes longer than it should
   1.532 +##		duration=$((tmr2-tmr1))
   1.533 +##		if test ${duration} -ge 30000; then
   1.534 +##			psql -U postgres ${DB} -c 'VACUUM ANALYZE' 
   1.535 +##			echo "Explicit VACUUM ANALYZE"
   1.536 +##		fi
   1.537 +##		
   1.538 +##		# deleteSeaHotspots
   1.539 +##		echo -n "Going to deleteSeaHotspots ${TIMESTAMP} " ;echo; echo; echo;
   1.540 +##		query=`echo "${deleteSeaHotspots}" | sed "s/TIMESTAMP/${TIMESTAMP}/g" | \
   1.541 +##		sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
   1.542 +##		sed "s/SENSOR/${SENSOR}/g"`
   1.543 +##		# ${countTime} ./strabon -db endpoint update "${query}"
   1.544 +##
   1.545 +##		tmr1=$(timer)
   1.546 +##		${ENDPOINT_SCRIPT} update ${ENDPOINT} "${query}"
   1.547 +##
   1.548 +##		tmr2=$(timer)
   1.549 +##		printf '%s ' $((tmr2-tmr1)) >>stderr.txt
   1.550 +##		echo;echo;echo;echo "File ${file} deleteSeaHotspots done!"
   1.551 +##
   1.552 +##		# invalidForFires
   1.553 +##		echo -n "invalidForFires ${TIMESTAMP} "  ; echo; echo ; echo;
   1.554 +##		query=`echo "${invalidForFires}" | sed "s/TIMESTAMP/${TIMESTAMP}/g" | \
   1.555 +##		sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
   1.556 +##		sed "s/SENSOR/${SENSOR}/g"` 
   1.557 +##
   1.558 +##		# ${countTime} ./strabon -db endpoint update "${query}"
   1.559 +##		tmr1=$(timer)
   1.560 +##		${ENDPOINT_SCRIPT} update ${ENDPOINT} "${query}"
   1.561 +##		tmr2=$(timer)
   1.562 +##		printf '%s ' $((tmr2-tmr1)) >>stderr.txt
   1.563 +##		echo "File ${file} invalidForFires done!"
   1.564 +## 
   1.565 +##		# refinePartialSeaHotspots
   1.566 +##		echo -n "refinePartialSeaHotspots ${TIMESTAMP} "  ; echo; echo ; echo;
   1.567 +##		query=`echo "${refinePartialSeaHotspots}" | sed "s/TIMESTAMP/${TIMESTAMP}/g" | \
   1.568 +##		sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
   1.569 +##		sed "s/SENSOR/${SENSOR}/g" |\
   1.570 +##		sed "s/SAT/${SAT}/g"`
   1.571 +##		# ${countTime} ./strabon -db endpoint update "${query}"
   1.572 +##		tmr1=$(timer)
   1.573 +##		${ENDPOINT_SCRIPT} update ${ENDPOINT} "${query}"
   1.574 +##		tmr2=$(timer)
   1.575 +##		printf '%s ' $((tmr2-tmr1)) >>stderr.txt
   1.576 +##
   1.577 +##		echo "File ${file} refinePartialSeaHotspots done!"
   1.578 +##		# echo "Continue?"
   1.579 +##		# read a
   1.580 +##
   1.581 +##		# refineTimePersistence
   1.582 +##		echo -n "Going to refineTimePersistence ${TIMESTAMP} ";echo;echo;echo; 
   1.583 +##		min_acquisition_time=`date --date="${year}-${month}-${day} ${time2}:00 EEST -30 minutes" +%Y-%m-%dT%H:%M:00`
   1.584 +##		query=`echo "${refineTimePersistence}" | sed "s/TIMESTAMP/${TIMESTAMP}/g" | \
   1.585 +##		sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
   1.586 +##		sed "s/SENSOR/${SENSOR}/g" | \
   1.587 +##		sed "s/ACQUISITIONS_IN_HALF_AN_HOUR/${N_ACQUISITIONS}/g" | \
   1.588 +##		sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g" |\
   1.589 +##		sed "s/SAT/${SAT}/g"`
   1.590 +##
   1.591 +##		#sudo -u postgres psql -d ${DB} -c 'VACUUM ANALYZE;';
   1.592 +##
   1.593 +##		tmr1=$(timer)
   1.594 +##		${ENDPOINT_SCRIPT} update ${ENDPOINT} "${query}"
   1.595 +##		 tmr2=$(timer)
   1.596 +##		printf '%s \n' $((tmr2-tmr1)) >>stderr.txt
   1.597 +##		echo;echo;echo;echo "File ${file} timePersistence done!"
   1.598 +##		# echo "Continue?"
   1.599 +##		# read a
   1.600 +##
   1.601 +##
   1.602 +##		# discover
   1.603 +##		echo -n "Going to discover ${TIMESTAMP} ";echo;echo;echo; 
   1.604 +##		min_acquisition_time=`date --date="${year}-${month}-${day} 00:00 EEST" +%Y-%m-%dT%H:%M:00`
   1.605 +##		max_acquisition_time=`date --date="${year}-${month}-${day} 23:59 EEST" +%Y-%m-%dT%H:%M:00`
   1.606 +##		query=`echo "${discover}" | \
   1.607 +##			sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
   1.608 +##			sed "s/SENSOR/${SENSOR}/g" | \
   1.609 +##			sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g" |\
   1.610 +##			sed "s/MAX_ACQUISITION_TIME/${max_acquisition_time}/g"`
   1.611 +##			
   1.612 +##		tmr1=$(timer)
   1.613 +##		${ENDPOINT_SCRIPT} query ${ENDPOINT} "${query}"
   1.614 +##		tmr2=$(timer)
   1.615 +##		printf '%s \n' $((tmr2-tmr1)) >>discover.txt
   1.616 +##		echo;echo;echo;echo "Discovered hotspots done!"
   1.617 +##	done
   1.618 +##done
   1.619 +##