Strabon
changeset 808:299566cf0392 DiffDateTime
Fixing runChain.sh...
author | George Garbis <ggarbis@di.uoa.gr> |
---|---|
date | Sun Dec 16 14:29:01 2012 +0200 (2012-12-16) |
parents | 1163025f7c46 |
children | 3b3e9ed00cf0 |
files | examples/teleios/runChain.sh |
line diff
1.1 --- a/examples/teleios/runChain.sh Fri Dec 14 18:12:51 2012 +0200 1.2 +++ b/examples/teleios/runChain.sh Sun Dec 16 14:29:01 2012 +0200 1.3 @@ -1,25 +1,35 @@ 1.4 #! /bin/bash 1.5 - 1.6 -# 1.7 -# This Source Code Form is subject to the terms of the Mozilla Public 1.8 -# License, v. 2.0. If a copy of the MPL was not distributed with this 1.9 -# file, You can obtain one at http://mozilla.org/MPL/2.0/. 1.10 -# 1.11 -# Copyright (C) 2010, 2011, 2012, Pyravlos Team 1.12 -# 1.13 -# http://www.strabon.di.uoa.gr/ 1.14 +# 1.15 +# This Source Code Form is subject to the terms of the Mozilla Public 1.16 +# License, v. 2.0. If a copy of the MPL was not distributed with this file, you 1.17 +# can obtain one at http://mozilla.org/MPL/2.0/. 1.18 +# 1.19 +# Copyright (C) 2010, 2011, 2012, Pyravlos Team 1.20 +# 1.21 +# http://www.strabon.di.uoa.gr/ 1.22 # 1.23 1.24 -LOC="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 1.25 +# Command name 1.26 +cmd="$(basename ${0})" 1.27 +# Get the directory where the script resides 1.28 +loc="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 1.29 1.30 -ENDPOINT="http://teleios4.di.uoa.gr:8080/NOA" 1.31 -DB="NOA2012" 1.32 +function help() { 1.33 + echo "Usage: ${cmd} [OPTIONS] " 1.34 + echo 1.35 + echo "Execute NOA chain with refinements and measure time." 1.36 + echo 1.37 + echo "OPTIONS can be any of the following" 1.38 + echo " -d,--db : PostGIS database" 1.39 + echo " -e,--endpoint : Strabon Endpoint" 1.40 + echo " -h,--hotposts : URL where hotspots are stored" 1.41 + echo " -b,--background : Background data" 1.42 + echo " -l,--log : Log file" 1.43 +} 1.44 1.45 -HOTSPOTS_URL="http://challenge.strabon.di.uoa.gr/MSG1" 1.46 -ENDPOINT_SCRIPT=../../scripts/endpoint 1.47 - 1.48 -logFile="chain.log" 1.49 - 1.50 +# If no arguments are given it returns miliseconds from 1970-01-01 00:00:00 UTC 1.51 +# Else if a time (miliseconds form ...) is given it returns the delta between 1.52 +# given time and current time 1.53 function timer() 1.54 { 1.55 if [[ $# -eq 0 ]]; then 1.56 @@ -42,12 +52,63 @@ 1.57 fi 1.58 } 1.59 1.60 -# find out the postgres service to use 1.61 -postgres=$(ls -1 /etc/init.d/| grep postgres | head -1) 1.62 +# Handle the postgres service 1.63 +# -$1: Command for the service 1.64 +function handlePostgresService() 1.65 +{ 1.66 + # find out the postgres service to use 1.67 + postgres=$(ls -1 /etc/init.d/| grep postgres | head -1) 1.68 1.69 -tomcat= 1.70 -function chooseTomcat() 1.71 + echo "Postgres service received command: $1" 1.72 + sudo service ${postgres} $1 1.73 +} 1.74 + 1.75 +# Handled a postgres database 1.76 +# -$1: Command (create/drop/store) 1.77 +# -$2: Dump file to store (if store is given as $1) 1.78 +# 'spatial' to create a spatial database (if create is given as $1) 1.79 +function handlePostgresDatabase() { 1.80 + local command=$1 1.81 + local db=$2 1.82 + shift; shift 1.83 + local options="$*" 1.84 + case "${command}" in 1.85 + create) 1.86 + echo "Creating database ${db}..." 1.87 + createdb -U postgres ${db} ${options} 1.88 + ;; 1.89 + drop) 1.90 + if test ! -z "${options}"; then 1.91 + echo "ERROR: dropdb takes no extra options" 1.92 + echo "options: ${options}" 1.93 + exit -1 1.94 + fi 1.95 + echo "Dropping database ${db}..." 1.96 + dropdb -U postgres ${db} 1.97 + ;; 1.98 + vacuum) 1.99 + if test "${options}" = "analyze"; then 1.100 + psql -U postgres ${DB} -c 'VACUUM ANALYZE' 1.101 + else 1.102 + psql -U postgres ${DB} -c 'VACUUM' 1.103 + fi 1.104 + ;; 1.105 + run) 1.106 + if test ! [ -f "${options}" ]; then 1.107 + echo "ERROR: No dump file to run" 1.108 + exit -1 1.109 + fi 1.110 + echo "Storing dump file ${options} in database ${db}..." 1.111 + psql ${db} -f ${options} 1.112 + ;; 1.113 + esac 1.114 +} 1.115 + 1.116 +# Handle the tomcat service 1.117 +# -$1: Command for the service 1.118 +function handleTomcatService() 1.119 { 1.120 + # find out the tomcat service to use 1.121 if test -s /etc/fedora-release ; then 1.122 tomcat="tomcat" 1.123 #elif test -s /etc/centos-release ; then 1.124 @@ -56,220 +117,303 @@ 1.125 #elif test -s /etc/SuSE-release ; then 1.126 #elif test -s /etc/gentoo-release ; then 1.127 elif test -s /etc/lsb-release ; then # Ubuntu 1.128 - tomcat=$(ls -1 /etc/init.d/| grep tomcat | head -1) 1.129 + tomcat=$(ls -1 /etc/init.d/| grep tomcat | head -1) 1.130 elif test -s /etc/debian_version ; then 1.131 tomcat="tomcat" 1.132 fi 1.133 1.134 # check for service availability 1.135 if ! test -e "/etc/init.d/${tomcat}"; then 1.136 - tomcat= 1.137 + echo "ERROR: No tomcat service found" 1.138 + exit -1 1.139 fi 1.140 + 1.141 + echo "Service ${tomcat} received command: $1" 1.142 + sudo service ${tomcat} $1 1.143 } 1.144 1.145 -insertMunicipalities=`cat ${LOC}/insertMunicipalities.rq` 1.146 -deleteSeaHotspots=`cat ${LOC}/deleteSeaHotspots.rq` # | sed 's/\"/\\\"/g'` 1.147 -refinePartialSeaHotspots=`cat ${LOC}/refinePartialSeaHotspots.rq` # | sed 's/\"/\\\"/g'` 1.148 -invalidForFires=`cat ${LOC}/landUseInvalidForFires.rq` 1.149 -refineTimePersistence=`cat ${LOC}/refineTimePersistence.rq` # | sed 's/\"/\\\"/g'` 1.150 -discover=`cat ${LOC}/discover.rq` 1.151 -#InsertMunicipalities =`cat ${LOC}/InsertMunicipalities.sparql` # | sed 's/\"/\\\"/g'` 1.152 +# get the main version of postgres 1.153 +function getPostgresMainVersion() { 1.154 + echo $(sudo service ${postgres} status | grep -o '.\..' | cut -b 1) 1.155 +} 1.156 1.157 -# Initialize (stop tomcat, restart postgres, drop/create database, start tomcat) 1.158 -chooseTomcat 1.159 -echo "stopping tomcat" 1.160 -if test -z "${tomcat}"; then 1.161 - # work-around for babis (standalone tomcat, with start_tomcat.sh and stop_tomcat.sh scripts) 1.162 - stop_tomcat.sh 1.163 -else 1.164 - sudo service ${tomcat} stop 1.165 -fi 1.166 +# It stores the backgroud data 1.167 +# - $1: database 1.168 +# - $2: backgound data file 1.169 +function storeBackgroundData() { 1.170 + local db=$1 1.171 + local bgFile=$2 1.172 1.173 -sudo service ${postgres} restart 1.174 + if test -f ${bgFile}; then 1.175 + handlePostgresDatabase run ${db} ${bgFile} 1.176 + elif test "${bgFile:0:7}" = "http://"; then 1.177 + #curl -s ${bgFile} | tar xz -O | psql -d ${DB} 1.178 + wget ${bgFile} -O /tmp/bgFile$$.tar.gz 1.179 + tar xz -O /tmp/bgFile$$.tar.gz 1.180 + 1.181 + handlePostgresDatabase run ${db} /tmp/bgFile$$.tar.tz 1.182 + rm /tmp/bgFiile$$.tar.gz 1.183 + else 1.184 + echo "Backgound file not foung" 1.185 + exit -1 1.186 + fi 1.187 + handlePostgresDatabase vacuum ${db} analyze 1.188 +} 1.189 + 1.190 +# Handle Stabon Endpoint 1.191 +# - $1: endpoint 1.192 +# - $2: command (store/query) 1.193 +# - $2: file/query 1.194 +function handleStrabonEndpoint(){ 1.195 + endpoint=$1 1.196 + command=$2 1.197 + option=$3 1.198 + 1.199 + endpointScript=${loc}/../../scripts/endpoint 1.200 + case ${command} in 1.201 + store) 1.202 + url=${options} 1.203 + 1.204 + # print timestamp 1.205 + echo -n "${TIMESTAMP} " >> ${logFile} 1.206 + 1.207 + tmr1=$(timer) 1.208 + ${endpointScript} store ${endpoint} N-Triples -u ${url} 1.209 + tmr2=$(timer) 1.210 + printf '%s ' $((tmr2-tmr1)) >> ${logFile} 1.211 + ;; 1.212 + query) 1.213 + query=${option} 1.214 + 1.215 + tmr1=$(timer) 1.216 + ${endpointScript} query ${endpoint} "${query}" 1.217 + tmr2=$(timer) 1.218 + printf '%s ' $((tmr2-tmr1)) >> ${logFile} 1.219 + ;; 1.220 + update) 1.221 + update=${option} 1.222 + tmr1=$(timer) 1.223 + ${endpointScript} update ${endpoint} "${update}" 1.224 + tmr2=$(timer) 1.225 + printf '%s ' $((tmr2-tmr1)) >> stderr.txt 1.226 + ;; 1.227 + *) 1.228 + echo "ERROR: ..." 1.229 + exit -1 1.230 + ;; 1.231 + esac 1.232 +} 1.233 + 1.234 +# $1: queryFile 1.235 +function makeUpdate() { 1.236 + queryFile=$1 1.237 + instansiateScript=${loc}/../scripts/instantiate.sh 1.238 +} 1.239 +# default values 1.240 +endpoint="http://pathway.di.uoa.gr:8080/endpoint" 1.241 +db="NOA2012" 1.242 +hotspots_url="http://pathway.di.uoag.r/hotspots/2012" 1.243 +bgFile="http://dev.strabon.di.uoa.gr/rdf/Kallikratis-Coastline-Corine-dump-postgres-9.tgz" 1.244 +logFile="runChain.log" 1.245 + 1.246 +# read script options 1.247 +while test $# -gt 0 -a "X${1:0:1}" == "X-"; do 1.248 + case "${1}" in 1.249 + --help) 1.250 + help 1.251 + exit 0 1.252 + ;; 1.253 + -e|--endpoint) 1.254 + shift 1.255 + endpoint=${1} 1.256 + shift 1.257 + ;; 1.258 + -d|--db) 1.259 + shift 1.260 + db=${1} 1.261 + shift 1.262 + ;; 1.263 + -h|--hotspots) 1.264 + shift 1.265 + hotspots_url=${1} 1.266 + shift 1.267 + ;; 1.268 + -b|--background) 1.269 + shift 1.270 + bgFile=${1} 1.271 + shift 1.272 + ;; 1.273 + -l|--log) 1.274 + shift 1.275 + logFile=${1} 1.276 + shift 1.277 + ;; 1.278 + *) 1.279 + echo "unknown argument ${1}" 1.280 + help 1.281 + exit -1 1.282 + ;; 1.283 + esac 1.284 +done 1.285 + 1.286 +echo "endpoint: ${endpoint}" 1.287 +echo "db: ${db}" 1.288 +echo "hotspots: ${hotspots_url}" 1.289 +echo "background: ${bgFile}" 1.290 +echo "logFile: ${logfile}" 1.291 + 1.292 + 1.293 +# Get queries and updates 1.294 +insertMunicipalities=`cat ${loc}/insertMunicipalities.rq` 1.295 +deleteSeaHotspots=`cat ${loc}/deleteSeaHotspots.rq` # | sed 's/\"/\\\"/g'` 1.296 +refinePartialSeaHotspots=`cat ${loc}/refinePartialSeaHotspots.rq` # | sed 's/\"/\\\"/g'` 1.297 +invalidForFires=`cat ${loc}/landUseInvalidForFires.rq` 1.298 +refineTimePersistence=`cat ${loc}/refineTimePersistence.rq` # | sed 's/\"/\\\"/g'` 1.299 +discover=`cat ${loc}/discover.rq` 1.300 +InsertMunicipalities=`cat ${loc}/InsertMunicipalities.sparql` # | sed 's/\"/\\\"/g'` 1.301 + 1.302 + 1.303 +#Initialize (stop tomcat, restart postgres, drop/create database, start tomcat) 1.304 +handleTomcatService stop 1.305 +handlePostgresService restart 1.306 1.307 # get the main version of postgres 1.308 -POSTGRES_MAIN_VERSION=$(sudo service ${postgres} status | grep -o '.\..' | cut -b 1) 1.309 +postgres_main_version=$(getPostgresMainVersion) 1.310 1.311 -#echo "Dropping endpoint database"; 1.312 -dropdb -U postgres ${DB} 1.313 +handlePostgresDatabase drop ${db} 1.314 +handlePostgresDatabase create ${db} 1.315 1.316 -#echo "Creating endpoint database" 1.317 -createdb -U postgres ${DB} 1.318 +storeBackgroundData ${db} ${bgFile} 1.319 1.320 -# load data 1.321 -#curl -s http://dev.strabon.di.uoa.gr/rdf/Kallikratis-Coastline-Corine-dump-postgres-${POSTGRES_MAIN_VERSION}.tgz | tar xz -O | psql -d ${DB} 1.322 -psql -U postgres -d ${DB} -f /tmp/Kallikratis-Coastline-ExcludeArea-dump.sql 1.323 -#psql -U postgres ${DB} -c 'VACUUM ANALYZE' 1.324 +handleTomcatService start 1.325 1.326 -echo "starting tomcat" 1.327 -if test -z "${tomcat}"; then 1.328 - # work-around for babis (standalone tomcat, with start_tomcat.sh and stop_tomcat.sh scripts) 1.329 - start_tomcat.sh 1.330 - sleep 2 1.331 -else 1.332 - sudo service ${tomcat} start 1.333 -fi 1.334 +echo "Timestamp Store Municipalities DeleteInSea InvalidForFires RefineInCoast 1.335 +TimePersistence" $ > stderr.txt 1.336 1.337 -echo "initializing database" 1.338 -echo "Timestamp Store Municipalities DeleteInSea InvalidForFires RefineInCoast TimePersistence" > stderr.txt 1.339 +years="2007 2008 2010 2011" 1.340 +for year in "${years}"; do 1.341 +for month in `seq 5 9`; do 1.342 +for day in `seq 1 31`; do 1.343 +for hour in `seq 0 23`; do 1.344 +for minute in `seq 0 60 5`; do 1.345 1.346 - 1.347 -#./scripts/endpoint query ${ENDPOINT} "SELECT (COUNT(*) AS ?C) WHERE {?s ?p ?o}" 1.348 -#sudo -u postgres psql -d endpoint -c 'CREATE INDEX datetime_values_idx_value ON datetime_values USING btree(value)'; 1.349 -#sudo -u postgres psql -d endpoint -c 'VACUUM ANALYZE;'; 1.350 - 1.351 -#for y in 2007 2008 2010 2011; do 1.352 -for y in 2012; do 1.353 - # get hotpost URLS 1.354 - for hot in $(curl -s ${HOTSPOTS_URL}/${y}/ | grep -o '>HMSG.*\.nt' | colrm 1 1); do 1.355 - file="${HOTSPOTS_URL}/${y}/${hot}" 1.356 - 1.357 - time_status=$(echo ${hot} | egrep -o '[[:digit:]]{6}_[[:digit:]]{4}') 1.358 - 1.359 - # get sensor 1.360 - SENSOR=$(echo ${hot} | grep -o 'MSG.') 1.361 - 1.362 - # get satellite and set number of acquisitions per hour 1.363 - if test "${SENSOR}" = "MSG2"; then 1.364 - SAT="METEOSAT9" 1.365 - 1.366 - N_ACQUISITIONS=3.0 1.367 - else 1.368 - SAT="METEOSAT8" 1.369 - SENSOR="MSG1_RSS" 1.370 - 1.371 - N_ACQUISITIONS=7.0 1.372 - fi 1.373 - 1.374 - # get time information for acquisition and construct timestamp 1.375 - year="20$(expr substr ${time_status} 1 2)" 1.376 - month=$(expr substr ${time_status} 3 2) 1.377 - day=$(expr substr ${time_status} 5 2) 1.378 - time2=$(expr substr ${time_status} 8 2) 1.379 - time2="${time2}:$(expr substr ${time_status} 10 2)" 1.380 + month=$(printf "%02d" ${month}) 1.381 + day=$(printf "%02d" ${day})k 1.382 + hour=$(printf "%02d" ${hour}) 1.383 + minute=$(printf "%02d" ${minute})k 1.384 1.385 # construct timestamp 1.386 - TIMESTAMP="${year}-${month}-${day}T${time2}:00" 1.387 + timestamp="${year}-${month}-${day}T${hour}:${minute}:00" 1.388 1.389 - # store file 1.390 - echo -n "storing " $file; echo; echo; 1.391 - # ${countTime} ./strabon -db endpoint store $file 1.392 + fileURL="HMSG1_RSS_039_s7_${year}${month}${day}_${hour}${minute}.hotspots.nt" 1.393 + handleStrabonEndpoint store ${fileURL} ${fileURL} 1.394 1.395 - # print timestamp 1.396 - echo -n "${TIMESTAMP} " >> stderr.txt 1.397 +## # sudo -u postgres psql -d endpoint -c 'VACUUM ANALYZE;'; 1.398 1.399 - tmr1=$(timer) 1.400 - ${ENDPOINT_SCRIPT} store ${ENDPOINT} N-Triples -u ${file} 1.401 - tmr2=$(timer) 1.402 - printf '%s ' $((tmr2-tmr1)) >> stderr.txt 1.403 1.404 - # sudo -u postgres psql -d endpoint -c 'VACUUM ANALYZE;'; 1.405 - 1.406 - echo;echo;echo;echo "File ${file} stored!" >> ${logFile} 1.407 - 1.408 - # insertMunicipalities 1.409 - echo -n "inserting Municipalities " ;echo; echo; echo; 1.410 - # query=`echo "${insertMunicipalities}" ` 1.411 - # ${countTime} ./strabon -db endpoint update "${query}" 1.412 - 1.413 - tmr1=$(timer) 1.414 - 1.415 - query=`echo "${insertMunicipalities}" | sed "s/TIMESTAMP/${TIMESTAMP}/g" | \ 1.416 - sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \ 1.417 - sed "s/SENSOR/${SENSOR}/g"` 1.418 - 1.419 - ${ENDPOINT_SCRIPT} update ${ENDPOINT} "${query}" 1.420 - 1.421 - tmr2=$(timer) 1.422 - printf '%s ' $((tmr2-tmr1)) >>stderr.txt 1.423 - echo;echo;echo;echo "File ${file} inserted Municipalities!" 1.424 - 1.425 - # execute an explicit VACUUM ANALYZE when a query takes longer than it should 1.426 - duration=$((tmr2-tmr1)) 1.427 - if test ${duration} -ge 30000; then 1.428 - psql -U postgres ${DB} -c 'VACUUM ANALYZE' 1.429 - echo "Explicit VACUUM ANALYZE" 1.430 - fi 1.431 - 1.432 - # deleteSeaHotspots 1.433 - echo -n "Going to deleteSeaHotspots ${TIMESTAMP} " ;echo; echo; echo; 1.434 - query=`echo "${deleteSeaHotspots}" | sed "s/TIMESTAMP/${TIMESTAMP}/g" | \ 1.435 - sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \ 1.436 - sed "s/SENSOR/${SENSOR}/g"` 1.437 - # ${countTime} ./strabon -db endpoint update "${query}" 1.438 - 1.439 - tmr1=$(timer) 1.440 - ${ENDPOINT_SCRIPT} update ${ENDPOINT} "${query}" 1.441 - 1.442 - tmr2=$(timer) 1.443 - printf '%s ' $((tmr2-tmr1)) >>stderr.txt 1.444 - echo;echo;echo;echo "File ${file} deleteSeaHotspots done!" 1.445 - 1.446 - # invalidForFires 1.447 - echo -n "invalidForFires ${TIMESTAMP} " ; echo; echo ; echo; 1.448 - query=`echo "${invalidForFires}" | sed "s/TIMESTAMP/${TIMESTAMP}/g" | \ 1.449 - sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \ 1.450 - sed "s/SENSOR/${SENSOR}/g"` 1.451 - 1.452 - # ${countTime} ./strabon -db endpoint update "${query}" 1.453 - tmr1=$(timer) 1.454 - ${ENDPOINT_SCRIPT} update ${ENDPOINT} "${query}" 1.455 - tmr2=$(timer) 1.456 - printf '%s ' $((tmr2-tmr1)) >>stderr.txt 1.457 - echo "File ${file} invalidForFires done!" 1.458 - 1.459 - # refinePartialSeaHotspots 1.460 - echo -n "refinePartialSeaHotspots ${TIMESTAMP} " ; echo; echo ; echo; 1.461 - query=`echo "${refinePartialSeaHotspots}" | sed "s/TIMESTAMP/${TIMESTAMP}/g" | \ 1.462 - sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \ 1.463 - sed "s/SENSOR/${SENSOR}/g" |\ 1.464 - sed "s/SAT/${SAT}/g"` 1.465 - # ${countTime} ./strabon -db endpoint update "${query}" 1.466 - tmr1=$(timer) 1.467 - ${ENDPOINT_SCRIPT} update ${ENDPOINT} "${query}" 1.468 - tmr2=$(timer) 1.469 - printf '%s ' $((tmr2-tmr1)) >>stderr.txt 1.470 - 1.471 - echo "File ${file} refinePartialSeaHotspots done!" 1.472 - # echo "Continue?" 1.473 - # read a 1.474 - 1.475 - # refineTimePersistence 1.476 - echo -n "Going to refineTimePersistence ${TIMESTAMP} ";echo;echo;echo; 1.477 - min_acquisition_time=`date --date="${year}-${month}-${day} ${time2}:00 EEST -30 minutes" +%Y-%m-%dT%H:%M:00` 1.478 - query=`echo "${refineTimePersistence}" | sed "s/TIMESTAMP/${TIMESTAMP}/g" | \ 1.479 - sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \ 1.480 - sed "s/SENSOR/${SENSOR}/g" | \ 1.481 - sed "s/ACQUISITIONS_IN_HALF_AN_HOUR/${N_ACQUISITIONS}/g" | \ 1.482 - sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g" |\ 1.483 - sed "s/SAT/${SAT}/g"` 1.484 - 1.485 - #sudo -u postgres psql -d ${DB} -c 'VACUUM ANALYZE;'; 1.486 - 1.487 - tmr1=$(timer) 1.488 - ${ENDPOINT_SCRIPT} update ${ENDPOINT} "${query}" 1.489 - tmr2=$(timer) 1.490 - printf '%s \n' $((tmr2-tmr1)) >>stderr.txt 1.491 - echo;echo;echo;echo "File ${file} timePersistence done!" 1.492 - # echo "Continue?" 1.493 - # read a 1.494 - 1.495 - 1.496 - # discover 1.497 - echo -n "Going to discover ${TIMESTAMP} ";echo;echo;echo; 1.498 - min_acquisition_time=`date --date="${year}-${month}-${day} 00:00 EEST" +%Y-%m-%dT%H:%M:00` 1.499 - max_acquisition_time=`date --date="${year}-${month}-${day} 23:59 EEST" +%Y-%m-%dT%H:%M:00` 1.500 - query=`echo "${discover}" | \ 1.501 - sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \ 1.502 - sed "s/SENSOR/${SENSOR}/g" | \ 1.503 - sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g" |\ 1.504 - sed "s/MAX_ACQUISITION_TIME/${max_acquisition_time}/g"` 1.505 - 1.506 - tmr1=$(timer) 1.507 - ${ENDPOINT_SCRIPT} query ${ENDPOINT} "${query}" 1.508 - tmr2=$(timer) 1.509 - printf '%s \n' $((tmr2-tmr1)) >>discover.txt 1.510 - echo;echo;echo;echo "Discovered hotspots done!" 1.511 - done 1.512 -done 1.513 - 1.514 +## # insertMunicipalities 1.515 +## echo -n "inserting Municipalities " ;echo; echo; echo; 1.516 +## # query=`echo "${insertMunicipalities}" ` 1.517 +## # ${countTime} ./strabon -db endpoint update "${query}" 1.518 +## 1.519 +## tmr1=$(timer) 1.520 +## 1.521 +## query=`echo "${insertMunicipalities}" | sed "s/TIMESTAMP/${TIMESTAMP}/g" | \ 1.522 +## sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \ 1.523 +## sed "s/SENSOR/${SENSOR}/g"` 1.524 +## 1.525 +## ${ENDPOINT_SCRIPT} update ${ENDPOINT} "${query}" 1.526 +## 1.527 +## tmr2=$(timer) 1.528 +## printf '%s ' $((tmr2-tmr1)) >>stderr.txt 1.529 +## echo;echo;echo;echo "File ${file} inserted Municipalities!" 1.530 +## 1.531 +## # execute an explicit VACUUM ANALYZE when a query takes longer than it should 1.532 +## duration=$((tmr2-tmr1)) 1.533 +## if test ${duration} -ge 30000; then 1.534 +## psql -U postgres ${DB} -c 'VACUUM ANALYZE' 1.535 +## echo "Explicit VACUUM ANALYZE" 1.536 +## fi 1.537 +## 1.538 +## # deleteSeaHotspots 1.539 +## echo -n "Going to deleteSeaHotspots ${TIMESTAMP} " ;echo; echo; echo; 1.540 +## query=`echo "${deleteSeaHotspots}" | sed "s/TIMESTAMP/${TIMESTAMP}/g" | \ 1.541 +## sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \ 1.542 +## sed "s/SENSOR/${SENSOR}/g"` 1.543 +## # ${countTime} ./strabon -db endpoint update "${query}" 1.544 +## 1.545 +## tmr1=$(timer) 1.546 +## ${ENDPOINT_SCRIPT} update ${ENDPOINT} "${query}" 1.547 +## 1.548 +## tmr2=$(timer) 1.549 +## printf '%s ' $((tmr2-tmr1)) >>stderr.txt 1.550 +## echo;echo;echo;echo "File ${file} deleteSeaHotspots done!" 1.551 +## 1.552 +## # invalidForFires 1.553 +## echo -n "invalidForFires ${TIMESTAMP} " ; echo; echo ; echo; 1.554 +## query=`echo "${invalidForFires}" | sed "s/TIMESTAMP/${TIMESTAMP}/g" | \ 1.555 +## sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \ 1.556 +## sed "s/SENSOR/${SENSOR}/g"` 1.557 +## 1.558 +## # ${countTime} ./strabon -db endpoint update "${query}" 1.559 +## tmr1=$(timer) 1.560 +## ${ENDPOINT_SCRIPT} update ${ENDPOINT} "${query}" 1.561 +## tmr2=$(timer) 1.562 +## printf '%s ' $((tmr2-tmr1)) >>stderr.txt 1.563 +## echo "File ${file} invalidForFires done!" 1.564 +## 1.565 +## # refinePartialSeaHotspots 1.566 +## echo -n "refinePartialSeaHotspots ${TIMESTAMP} " ; echo; echo ; echo; 1.567 +## query=`echo "${refinePartialSeaHotspots}" | sed "s/TIMESTAMP/${TIMESTAMP}/g" | \ 1.568 +## sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \ 1.569 +## sed "s/SENSOR/${SENSOR}/g" |\ 1.570 +## sed "s/SAT/${SAT}/g"` 1.571 +## # ${countTime} ./strabon -db endpoint update "${query}" 1.572 +## tmr1=$(timer) 1.573 +## ${ENDPOINT_SCRIPT} update ${ENDPOINT} "${query}" 1.574 +## tmr2=$(timer) 1.575 +## printf '%s ' $((tmr2-tmr1)) >>stderr.txt 1.576 +## 1.577 +## echo "File ${file} refinePartialSeaHotspots done!" 1.578 +## # echo "Continue?" 1.579 +## # read a 1.580 +## 1.581 +## # refineTimePersistence 1.582 +## echo -n "Going to refineTimePersistence ${TIMESTAMP} ";echo;echo;echo; 1.583 +## min_acquisition_time=`date --date="${year}-${month}-${day} ${time2}:00 EEST -30 minutes" +%Y-%m-%dT%H:%M:00` 1.584 +## query=`echo "${refineTimePersistence}" | sed "s/TIMESTAMP/${TIMESTAMP}/g" | \ 1.585 +## sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \ 1.586 +## sed "s/SENSOR/${SENSOR}/g" | \ 1.587 +## sed "s/ACQUISITIONS_IN_HALF_AN_HOUR/${N_ACQUISITIONS}/g" | \ 1.588 +## sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g" |\ 1.589 +## sed "s/SAT/${SAT}/g"` 1.590 +## 1.591 +## #sudo -u postgres psql -d ${DB} -c 'VACUUM ANALYZE;'; 1.592 +## 1.593 +## tmr1=$(timer) 1.594 +## ${ENDPOINT_SCRIPT} update ${ENDPOINT} "${query}" 1.595 +## tmr2=$(timer) 1.596 +## printf '%s \n' $((tmr2-tmr1)) >>stderr.txt 1.597 +## echo;echo;echo;echo "File ${file} timePersistence done!" 1.598 +## # echo "Continue?" 1.599 +## # read a 1.600 +## 1.601 +## 1.602 +## # discover 1.603 +## echo -n "Going to discover ${TIMESTAMP} ";echo;echo;echo; 1.604 +## min_acquisition_time=`date --date="${year}-${month}-${day} 00:00 EEST" +%Y-%m-%dT%H:%M:00` 1.605 +## max_acquisition_time=`date --date="${year}-${month}-${day} 23:59 EEST" +%Y-%m-%dT%H:%M:00` 1.606 +## query=`echo "${discover}" | \ 1.607 +## sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \ 1.608 +## sed "s/SENSOR/${SENSOR}/g" | \ 1.609 +## sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g" |\ 1.610 +## sed "s/MAX_ACQUISITION_TIME/${max_acquisition_time}/g"` 1.611 +## 1.612 +## tmr1=$(timer) 1.613 +## ${ENDPOINT_SCRIPT} query ${ENDPOINT} "${query}" 1.614 +## tmr2=$(timer) 1.615 +## printf '%s \n' $((tmr2-tmr1)) >>discover.txt 1.616 +## echo;echo;echo;echo "Discovered hotspots done!" 1.617 +## done 1.618 +##done 1.619 +##