Strabon

view examples/teleios/runChain.sh @ 1482:bd7c59599449

Changed the getSRID functions as in the default branch.
author George Stamoulis <gstam@di.uoa.gr>
date Sat Feb 21 12:38:36 2015 +0200 (2015-02-21)
parents e9521141694b
children
line source
1 #! /bin/bash
2 #
3 # This Source Code Form is subject to the terms of the Mozilla Public
4 # License, v. 2.0. If a copy of the MPL was not distributed with this file, you
5 # can obtain one at http://mozilla.org/MPL/2.0/.
6 #
7 # Copyright (C) 2010, 2011, 2012, 2013 Pyravlos Team
8 #
9 # http://www.strabon.di.uoa.gr/
10 #
11 # Author: George Garbis <ggarbis@di.uoa.gr>
12 # Author: Charalampos (Babis) Nikolaou <charnik@di.uoa.gr>
13 # Author: Manos Karpathiotakis <mk@di.uoa.gr>
14 # Author: Konstantina Bereta <Konstantina.Bereta@di.uoa.gr>
15 #
17 # If tomcat is standalone then environment variable TOMCATPATH should be set
19 # Example run command: examples/teleios/runChain.sh -b http://dev.strabon.di.uoa.gr/rdf/data-dump-postgres-9.tgz -l ${HOME}/runChain.log -e http://pathway.di.uoa.gr:8080/endpoint
21 # Command name
22 cmd="$(basename ${0})"
23 # Get the directory where the script resides
24 loc="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
26 function help() {
27 echo "Usage: ${cmd} [OPTIONS] "
28 echo
29 echo "Execute NOA chain with refinements and measure time."
30 echo
31 echo "OPTIONS can be any of the following"
32 echo " -d,--db : PostGIS database"
33 echo " -e,--endpoint : Strabon Endpoint"
34 echo " -h,--hotposts : URL where hotspots are stored"
35 echo " -b,--background : Background data"
36 echo " -l,--log : Log file"
37 echo " -c,--chain : Processing chain of hotspots"
38 echo " -p,--persistence : Value of persistence of discoverFires query"
39 echo " -r,--repeat_in_persistence : Value of repeat_in_persistence of discoverFires query"
40 }
42 # If no arguments are given it returns miliseconds from 1970-01-01 00:00:00 UTC
43 # Else if a time (miliseconds form ...) is given it returns the delta between
44 # given time and current time
45 function timer()
46 {
47 if [[ $# -eq 0 ]]; then
48 t=$(date '+%s%N')
49 t=$((t/1000000))
50 echo $t
51 else
52 local stime=$1
53 etime=$(date '+%s%N')
54 etime=$((etime/1000000))
56 if [[ -z "$stime" ]]; then stime=$etime; fi
57 dt=$((etime - stime)) #dt in milliseconds
58 dM=$((dt%1000))
59 Dt=$((dt/1000)) #delta t in seconds
60 ds=$((Dt % 60))
61 dm=$(((Dt / 60) % 60))
62 dh=$((Dt / 3600))
63 printf '%d:%02d:%02d.%03d' $dh $dm $ds $dM
64 fi
65 }
67 # Handle the postgres service
68 # -$1: Command for the service
69 function handlePostgresService()
70 {
71 # find out the postgres service to use
72 postgres=$(ls -1 /etc/init.d/| grep postgres | head -1)
74 echo "Service ${postgres} received command: $1"
75 sudo service ${postgres} $1
76 }
78 # Handled a postgres database
79 # -$1: Command (create/drop/store)
80 # -$2: Dump file to store (if runscript is given as command)
81 # or 'spatial' to create a spatial database (if create is given as command)
82 function handlePostgresDatabase() {
83 local command=$1
84 local db=$2
85 shift; shift
86 local options="$*"
87 case "${command}" in
88 create)
89 if test "${options}" = "spatial"; then
90 options="-T template_postgis"
91 elif test ! -z "${options}"; then
92 echo "ERROR: only spatial is allowed for create option"
93 echo "options: ${options}"
94 exit -1
95 fi
96 echo "Creating database ${db}... with options ${options}"
97 createdb -U postgres ${db} ${options}
98 ;;
99 drop)
100 if test ! -z "${options}"; then
101 echo "ERROR: dropdb takes no extra options"
102 echo "options: ${options}"
103 exit -1
104 fi
105 echo "Dropping database ${db}..."
106 dropdb -U postgres ${db}
107 ;;
108 vacuum)
109 if test "${options}" = "analyze"; then
110 psql -U postgres ${db} -c 'VACUUM ANALYZE'
111 echo "VACUUM ANALYZE ${db}"
112 else
113 psql -U postgres ${db} -c 'VACUUM'
114 echo "VACUUM ${db}"
115 fi
116 ;;
117 runscript)
118 if test ! -f "${options}"; then
119 echo "ERROR: No dump file to run"
120 exit -1
121 fi
122 echo "Storing dump file ${options} in database ${db}..."
123 psql -U postgres ${db} -f ${options}
124 ;;
125 esac
126 }
128 # Handle the tomcat service
129 # -$1: Command for the service
130 function handleTomcatService()
131 {
132 if test ! -z "${TOMCATPATH}" ; then
133 case "${1}" in
134 start)
135 ${TOMCATPATH}/bin/startup.sh
136 ;;
137 stop)
138 ${TOMCATPATH}/bin/shutdown.sh
139 ;;
140 restart)
141 ${TOMCATPATH}/bin/startup.sh
142 ${TOMCATPATH}/bin/shutdown.sh
143 ;;
144 esac
145 return
146 # find out the tomcat service to use
147 elif test -s /etc/fedora-release ; then
148 tomcat="tomcat"
149 #elif test -s /etc/centos-release ; then
150 #elif test -s /etc/yellowdog-release ; then
151 #elif test -s /etc/redhat-release ; then
152 #elif test -s /etc/SuSE-release ; then
153 #elif test -s /etc/gentoo-release ; then
154 elif test -s /etc/lsb-release ; then # Ubuntu
155 tomcat=$(ls -1 /etc/init.d/| grep tomcat | head -1)
156 elif test -s /etc/debian_version ; then
157 tomcat="tomcat"
158 fi
160 # check for service availability
161 if ! test -e "/etc/init.d/${tomcat}"; then
162 echo "ERROR: No tomcat service found"
163 exit -1
164 fi
166 echo "Service ${tomcat} received command: $1"
167 sudo service ${tomcat} $1
168 }
171 # get the main version of postgres
172 function getPostgresMainVersion() {
173 echo $(sudo service ${postgres} status | grep -o '.\..' | cut -b 1)
174 }
176 # It stores the backgroud data
177 # - $1: database
178 # - $2: backgound data file
179 function storeBackgroundData() {
180 local db=$1
181 local bgFile=$2
183 if test -f ${bgFile}; then
184 handlePostgresDatabase runscript ${db} ${bgFile}
185 elif test "${bgFile:0:7}" = "http://"; then
186 curl -s ${bgFile} | tar xzf - -O > /tmp/bgFiles$$.sql
187 # wget ${bgFile} -O /tmp/bgFile$$.tar.gz
188 # tar xzf /tmp/bgFile$$.tar.gz
189 handlePostgresDatabase runscript ${db} /tmp/bgFiles$$.sql
190 # rm /tmp/bgFile$$.tar.gz
191 rm /tmp/bgFiles$$.sql
192 else
193 echo "Backgound file not found"
194 exit -1
195 fi
196 handlePostgresDatabase vacuum ${db} analyze
197 }
199 # Handle Stabon Endpoint
200 # - $1: endpoint
201 # - $2: command (store/query)
202 # - $2: file/query
203 function handleStrabonEndpoint(){
204 endpoint=$1
205 command=$2
206 options=$3
208 endpointScript=${loc}/../../scripts/endpoint
209 case ${command} in
210 store)
211 url=${options}
213 tmr1=$(timer)
214 #${endpointScript} store ${endpoint} N-Triples -u ${url}
215 #read t
216 ${endpointScript} store ${endpoint} N-Triples -u ${url}
217 tmr2=$(timer)
219 # execute an explicit VACUUM ANALYZE when a query takes longer than it should
220 duration=$((tmr2-tmr1))
221 if test ${duration} -ge 30000; then
222 handlePostgresDatabase vacuum ${db} analyze
223 # psql -U postgres ${DB} -c 'VACUUM ANALYZE'
224 # echo "Explicit VACUUM ANALYZE"
225 tmr2=$(timer)
226 fi
227 printf '%s ' $((tmr2-tmr1)) >> ${logFile}
228 ;;
229 query)
230 query=${options}
231 tmr1=$(timer)
232 #${endpointScript} query ${endpoint} "${query}"
233 #read t
234 ${endpointScript} query ${endpoint} "${query}"
235 tmr2=$(timer)
236 printf '%s ' $((tmr2-tmr1)) >> ${logFile}
237 ;;
238 update)
239 update=${options}
240 tmr1=$(timer)
241 #${endpointScript} update ${endpoint} "${update}"
242 #read t
243 ${endpointScript} update ${endpoint} "${update}"
244 tmr2=$(timer)
245 printf '%s ' $((tmr2-tmr1)) >> ${logFile}
246 ;;
247 *)
248 echo "ERROR: Unknown endpoint command"
249 exit -1
250 ;;
251 esac
252 }
254 # default values
255 endpoint="http://teleios3.di.uoa.gr:8080/endpoint"
256 db="NOA2012"
257 hotspotsURL="http://jose.di.uoa.gr/rdf/hotspots/MSG1"
258 # ./examples/teleios/data/data-dump-9.sql
259 bgFile="http://dev.strabon.di.uoa.gr/rdf/data-dump-9.sql"
260 logFile="${HOME}/runChain.log"
262 chain="DynamicThresholds"
263 persistence=10
264 repeatInPers=3
266 # read script options
267 while test $# -gt 0 -a "X${1:0:1}" == "X-"; do
268 case "${1}" in
269 --help)
270 help
271 exit 0
272 ;;
273 -e|--endpoint)
274 shift
275 endpoint=${1}
276 shift
277 ;;
278 -d|--db)
279 shift
280 db=${1}
281 shift
282 ;;
283 -h|--hotspots)
284 shift
285 hotspots_url=${1}
286 shift
287 ;;
288 -b|--background)
289 shift
290 bgFile=${1}
291 shift
292 ;;
293 -l|--log)
294 shift
295 logFile=${1}
296 shift
297 ;;
298 -c|--chain)
299 shift
300 chain=${1}
301 shift
302 ;;
303 -p|--persistence)
304 shift
305 persistence=${1}
306 shift
307 ;;
308 -r|--repeat_in_persistence)
309 shift
310 repeat_in_persistence=${1}
311 shift
312 ;;
313 *)
314 echo "unknown argument ${1}"
315 help
316 exit -1
317 ;;
318 esac
319 done
321 echo "endpoint: ${endpoint}"
322 echo "db: ${db}"
323 echo "hotspots: ${hotspotsURL}"
324 echo "background: ${bgFile}"
325 echo "logFile: ${logFile}"
327 echo > ${logFile}
328 instantiate=${loc}/instantiate.sh
330 #Initialize (stop tomcat, restart postgres, drop/create database, start tomcat)
331 handleTomcatService stop
332 handlePostgresService restart
334 handlePostgresDatabase drop ${db}
335 handlePostgresDatabase create ${db}
337 storeBackgroundData ${db} ${bgFile} # ~/Temp/Kallikratis-Coastline-Corine-postgres-9.sql
339 handleTomcatService start
340 # Wait until tomcat server is up
341 until [ "`curl --silent --show-error --connect-timeout 1 -I ${endpoint} | grep 'Coyote'`" != "" ]; do sleep 1; done;
344 #${loc}/../../scripts/endpoint query ${endpoint} size
345 #exit -1
346 echo "Timestamp Store Municipalities DeleteInSea InvalidForFires RefineInCoast TimePersistence DiscoverHotspots" > ${logFile}
347 echo > ${HOME}/discoverFires.log
348 echo > ${HOME}/discover.log
350 years="2012" #"2007 2008 2010 2011"
351 for y in ${years}; do
352 # hotspots="`ls /var/www/hotspots/${y} | sort | grep -o 'HMSG.*\.nt'`"
353 # get hotpost URLS
354 for hot in $(curl -s ${hotspotsURL}/${y}/ | grep -o '>HMSG.*\.nt' | colrm 1 1); do
355 # for hot in ${hotspots}; do
356 file="${hotspotsURL}/${y}/${hot}"
358 time_status=$(echo ${hot} | egrep -o '[[:digit:]]{6}_[[:digit:]]{4}')
360 # get sensor
361 sensor=$(echo ${hot} | grep -o 'MSG.')
363 # get satellite and set number of acquisitions per hour
364 if test "${sensor}" = "MSG1"; then
365 sensor="MSG1_RSS"
366 fi
368 # get time information for acquisition and construct timestamp
369 year="20$(expr substr ${time_status} 1 2)"
370 month=$(expr substr ${time_status} 3 2)
371 day=$(expr substr ${time_status} 5 2)
372 time2=$(expr substr ${time_status} 8 2)
373 time2="${time2}:$(expr substr ${time_status} 10 2)"
375 # construct timestamp
376 timestamp="${year}-${month}-${day}T${time2}:00"
377 # print timestamp
378 echo -n "${timestamp} " >> ${logFile}
380 handleStrabonEndpoint ${endpoint} store ${file}
381 echo "Processing File ${file}" ; # read t
383 # Insert Municipalities
384 update="`${instantiate} -t ${timestamp} -c ${chain} -s ${sensor} ${loc}/insertMunicipalities.rq`"
385 # echo "Insert Municipalities: ${update}" ; read t
386 handleStrabonEndpoint ${endpoint} update "${update}"
389 # Delete Sea Hotspots
390 update="`${instantiate} -t ${timestamp} -c ${chain} -s ${sensor} ${loc}/deleteSeaHotspots.rq`"
391 # echo "Delete Sea Hotspots: ${update}" ; read t
392 handleStrabonEndpoint ${endpoint} update "${update}"
394 # Invalid For Fires
395 update="`${instantiate} -t ${timestamp} -c ${chain} -s ${sensor} ${loc}/landUseInvalidForFires.rq`"
396 # echo "Invalid For Fires: ${update}" ; read t
397 handleStrabonEndpoint ${endpoint} update "${update}"
399 # # Delete Reflections
400 # minTime=`date --date="${year}-${month}-${day} ${time2}:00 EEST -60 minutes" +%Y-%m-%dT%H:%M:00`
401 # update="`${instantiate} -t ${timestamp} -c ${chain} -s ${sensor} -m ${minTime} ${loc}/deleteReflections.rq`"
402 ## echo "Delete Reflections: ${update}" ;
403 # handleStrabonEndpoint ${endpoint} update "${update}"
405 # Refine Partial Sea Hotspots
406 update="`${instantiate} -t ${timestamp} -c ${chain} -s ${sensor} ${loc}/refinePartialSeaHotspots.rq`"
407 # echo "Refine Partial Sea Hotspots: ${update}" ; read t
408 handleStrabonEndpoint ${endpoint} update "${update}"
410 # Refine Time Persistence
411 minTime=`date --date="${year}-${month}-${day} ${time2}:00 EEST -30 minutes" +%Y-%m-%dT%H:%M:00`
412 update="`${instantiate} -t ${timestamp} -c ${chain} -s ${sensor} -m ${minTime} ${loc}/refineTimePersistence.rq`"
413 # echo "Refine Time Persistence: ${update}" ; read t
414 handleStrabonEndpoint ${endpoint} update "${update}" #2>&1 | tee ${HOME}/timePersistence.log
416 #psql -U postgres -d ${DB} -c 'VACUUM ANALYZE;';
418 # Discover
419 minTime=`date --date="${year}-${month}-${day} 00:00 EEST" +%Y-%m-%dT%H:%M:00`
420 maxTime=`date --date="${year}-${month}-${day} 23:59 EEST" +%Y-%m-%dT%H:%M:00`
421 query="`${instantiate} -c ${chain} -s ${sensor} -m ${minTime} -M ${maxTime} ${loc}/discover.rq`"
422 # echo "Discover: ${query}" ; #read t
423 handleStrabonEndpoint ${endpoint} query "${query}" &>> ${HOME}/discover.log
424 #
425 # # Discover Fires
426 # minTime=`date --date="${year}-${month}-${day} 00:00 EEST" +%Y-%m-%dT%H:%M:00`
427 # maxTime=`date --date="${year}-${month}-${day} 23:59 EEST" +%Y-%m-%dT%H:%M:00`
428 # query="`${instantiate} -c ${chain} -s ${sensor} -m ${minTime} -M ${maxTime} -p 10 -r 3 ${loc}/discoverFires.rq`"
429 ## echo "Discover Fires: ${query}" ; #read t
430 # handleStrabonEndpoint ${endpoint} query "${query}" &>> ${HOME}/discoverFires.log
432 # Add a new line
433 echo >> ${logFile}
434 done
435 done