Strabon

view scripts/archive/v2.2/runNoaRefinementChain-papos.sh @ 1277:92b90ac06d39

[maven-release-plugin] prepare release v3.2.9
author Babis Nikolaou <charnik@di.uoa.gr>
date Sat Oct 19 23:37:23 2013 +0300 (2013-10-19)
parents 515413720e57
children
line source
1 #!/bin/bash
2 LOC="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
4 ############################ CONFIGURATION #############################
5 ENDPOINT="http://localhost:8080/endpoint"
6 DB="endpoint"
8 HOTSPOTS_URL="http://localhost/rdf"
9 URLDIR=( "msg1_rss" "msg2")
10 URLPREFIX=( "HMSG1_RSS_IR_039_s7_" "HMSG2_IR_039_s7_")
11 SENSOR=( "MSG1_RSS" "MSG2")
12 SATELITE=( "METEOSAT8" "METEOSAT9")
13 ACQ_HALF_HOUR=( "7.0" "3.0")
14 SUFFIX=".hotspots.n3"
15 PROCESSING_CHAIN="DynamicThresholds"
18 # log files
19 logFile="chain.log"
20 timings="chain-times.log"
21 timingsDiscover="discover.txt"
23 # stSPARQL statements
24 insertMunicipalities=`cat ${LOC}/insertMunicipalities.sparql`
25 deleteSeaHotspots=`cat ${LOC}/deleteSeaHotspots.sparql`
26 invalidForFires=`cat ${LOC}/landUseInvalidForFires.sparql`
27 refinePartialSeaHotspots=`cat ${LOC}/refinePartialSeaHotspots.sparql`
28 refineTimePersistence=`cat ${LOC}/refineTimePersistence.sparql`
29 discover=`cat ${LOC}/discover.sparql`
30 # | sed 's/\"/\\\"/g'`
31 ########################################################################
33 function timer()
34 {
35 if [[ $# -eq 0 ]]; then
36 t=$(date '+%s%N')
37 t=$((t/1000000))
38 echo $t
39 else
40 local stime=$1
41 etime=$(date '+%s%N')
42 etime=$((etime/1000000))
44 if [[ -z "$stime" ]]; then stime=$etime; fi
45 dt=$((etime - stime)) #dt in milliseconds
46 dM=$((dt%1000))
47 Dt=$((dt/1000)) #delta t in seconds
48 ds=$((Dt % 60))
49 dm=$(((Dt / 60) % 60))
50 dh=$((Dt / 3600))
51 printf '%d:%02d:%02d.%03d' $dh $dm $ds $dM
52 fi
53 }
55 # find out the postgres service to use
56 postgres=$(ls -1 /etc/init.d/| grep postgres | head -1)
58 tomcat=
59 function chooseTomcat()
60 {
61 if test -s /etc/fedora-release ; then
62 tomcat="tomcat"
63 #elif test -s /etc/centos-release ; then
64 #elif test -s /etc/yellowdog-release ; then
65 #elif test -s /etc/redhat-release ; then
66 #elif test -s /etc/SuSE-release ; then
67 #elif test -s /etc/gentoo-release ; then
68 elif test -s /etc/lsb-release ; then # Ubuntu
69 if test -s /etc/init.d/tomcat6 ; then
70 tomcat="tomcat6"
71 elif test -s /etc/init.d/tomcat7 ; then
72 tomcat="tomcat7"
73 fi
74 elif test -s /etc/debian_version ; then
75 tomcat="tomcat"
76 fi
78 # check for service availability
79 if ! test -e "/etc/init.d/${tomcat}"; then
80 tomcat=
81 fi
82 }
84 # Initialize (stop tomcat, restart postgres, drop/create database, start tomcat)
85 chooseTomcat
86 echo "stopping tomcat"
87 if test -z "${tomcat}"; then
88 # work-around for babis (standalone tomcat, with start_tomcat.sh and stop_tomcat.sh scripts)
89 stop_tomcat.sh
90 else
91 sudo service ${tomcat} stop
92 fi
94 sudo service ${postgres} restart
96 # get the main version of postgres
97 POSTGRES_MAIN_VERSION=$(sudo service ${postgres} status | grep -o '.\..' | cut -b 1)
99 echo "Dropping endpoint database";
100 dropdb -U postgres ${DB}
102 echo "Creating endpoint database"
103 createdb -U postgres ${DB}
105 # load background data
106 echo "initializing database"
107 curl -s http://dev.strabon.di.uoa.gr/rdf/Kallikratis-Coastline-Corine-dump-postgres-${POSTGRES_MAIN_VERSION}.tgz | tar xz -O | psql -U postgres -d ${DB}
108 psql ${DB} -U postgres -c 'VACUUM ANALYZE '
110 echo "starting tomcat"
111 if test -z "${tomcat}"; then
112 # work-around for babis (standalone tomcat, with start_tomcat.sh and stop_tomcat.sh scripts)
113 start_tomcat.sh
114 else
115 sudo service ${tomcat} start
116 fi
118 # the chain :)
119 echo "Store Municipalities DeleteInSea InvalidForFires RefineInCoast TimePersistence" > ${timings}
120 for (( i = 0 ; i < ${#URLDIR[@]} ; i++ )) do
121 dir=${URLDIR[$i]}
122 prefix=${URLPREFIX[$i]}
123 sensor=${SENSOR[$i]}
124 satelite=${SATELITE[$i]}
125 acquisitions=${ACQ_HALF_HOUR[$i]}
126 # get hotpost URLS
127 for hot in $(curl -s ${HOTSPOTS_URL}/${dir}/ | grep -o ">${prefix}.*\.n3" | colrm 1 1); do
128 echo $hot
129 file="${HOTSPOTS_URL}/${dir}/${hot}"
130 echo $file
132 # get time information for acquisition
133 offset=$(( ${#prefix} + 1 ))
134 year=$(expr substr ${hot} ${offset} 2)
135 month=$(expr substr ${hot} $(( ${offset} + 2 )) 2)
136 day=$(expr substr ${hot} $(( ${offset} + 4 )) 2)
137 t1=$(expr substr ${hot} $(( ${offset} + 7 )) 2)
138 t2=$(expr substr ${hot} $(( ${offset} + 9 )) 2)
139 time2="${t1}:${t2}"
141 printf "$hot " >> ${timings}
143 # store file
144 echo -n "storing " $file; echo; echo;
145 # ${countTime} ./strabon -db endpoint store $file
147 tmr1=$(timer)
148 ../endpoint store ${ENDPOINT} N-Triples -u ${file}
149 tmr2=$(timer)
150 printf '%s ' $((tmr2-tmr1)) >> ${timings}
152 # sudo -u postgres psql -d endpoint -c 'VACUUM ANALYZE;';
154 echo;echo;echo;echo "File ${file} stored!" >> ${logFile}
156 # insertMunicipalities
157 echo -n "inserting Municipalities " ;echo; echo; echo;
158 # query=`echo "${insertMunicipalities}" `
159 # ${countTime} ./strabon -db endpoint update "${query}"
161 tmr1=$(timer)
163 query=`echo "${insertMunicipalities}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \
164 sed "s/PROCESSING_CHAIN/${PROCESSING_CHAIN}/g" | \
165 sed "s/SENSOR/${sensor}/g"`
167 ../endpoint update ${ENDPOINT} "${query}"
169 tmr2=$(timer)
170 printf '%s ' $((tmr2-tmr1)) >> ${timings}
171 echo;echo;echo;echo "File ${file} inserted Municipalities!"
173 # deleteSeaHotspots
174 echo -n "Going to deleteSeaHotspots 20${year}-${month}-${day}T${time2}:00 " ;echo; echo; echo;
175 query=`echo "${deleteSeaHotspots}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \
176 sed "s/PROCESSING_CHAIN/${PROCESSING_CHAIN}/g" | \
177 sed "s/SENSOR/${sensor}/g"`
178 # ${countTime} ./strabon -db endpoint update "${query}"
180 tmr1=$(timer)
181 ../endpoint update ${ENDPOINT} "${query}"
183 tmr2=$(timer)
184 printf '%s ' $((tmr2-tmr1)) >> ${timings}
185 echo;echo;echo;echo "File ${file} deleteSeaHotspots done!"
187 # echo "Continue?"
188 # read a
189 # invalidForFires
190 echo -n "invalidForFires 20${year}-${month}-${day}T${time2}:00 " ; echo; echo ; echo;
191 query=`echo "${invalidForFires}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \
192 sed "s/PROCESSING_CHAIN/${PROCESSING_CHAIN}/g" | \
193 sed "s/SENSOR/${sensor}/g" |\
194 sed "s/SAT/${satelite}/g"`
195 # ${countTime} ./strabon -db endpoint update "${query}"
196 tmr1=$(timer)
197 ../endpoint update ${ENDPOINT} "${query}"
198 tmr2=$(timer)
199 printf '%s ' $((tmr2-tmr1)) >> ${timings}
200 echo "File ${file} invalidForFires done!"
202 # refinePartialSeaHotspots
203 echo -n "refinePartialSeaHotspots 20${year}-${month}-${day}T${time2}:00 " ; echo; echo ; echo;
204 query=`echo "${refinePartialSeaHotspots}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \
205 sed "s/PROCESSING_CHAIN/${PROCESSING_CHAIN}/g" | \
206 sed "s/SENSOR/${sensor}/g" |\
207 sed "s/SAT/${satelite}/g"`
208 # ${countTime} ./strabon -db endpoint update "${query}"
209 tmr1=$(timer)
210 ../endpoint update ${ENDPOINT} "${query}"
211 tmr2=$(timer)
212 printf '%s ' $((tmr2-tmr1)) >> ${timings}
214 echo "File ${file} refinePartialSeaHotspots done!"
215 # echo "Continue?"
216 # read a
218 # refineTimePersistence
219 echo -n "Going to refineTimePersistence 20${year}-${month}-${day}T${time2}:00 ";echo;echo;echo;
220 min_acquisition_time=`date --date="20${year}-${month}-${day} ${time2}:00 EEST -30 minutes" +%Y-%m-%dT%H:%m:00`
221 query=`echo "${refineTimePersistence}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \
222 sed "s/PROCESSING_CHAIN/${PROCESSING_CHAIN}/g" | \
223 sed "s/SENSOR/${sensor}/g" | \
224 sed "s/ACQUISITIONS_IN_HALF_AN_HOUR/${acquisitions}/g" | \
225 sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g" |\
226 sed "s/SAT/${satelite}/g"`
228 #sudo -u postgres psql -d ${DB} -c 'VACUUM ANALYZE;';
230 tmr1=$(timer)
231 ../endpoint update ${ENDPOINT} "${query}"
232 tmr2=$(timer)
233 printf '%s \n' $((tmr2-tmr1)) >> ${timings}
234 echo;echo;echo;echo "File ${file} timePersistence done!"
235 # echo "Continue?"
236 # read a
239 # discover
240 echo -n "Going to discover 20${year}-${month}-${day}T${time2}:00 ";echo;echo;echo;
241 min_acquisition_time=`date --date="20${year}-${month}-${day} 00:00 EEST" +%Y-%m-%dT%H:%m:00`
242 max_acquisition_time=`date --date="20${year}-${month}-${day} 23:59 EEST" +%Y-%m-%dT%H:%m:00`
243 query=`echo "${discover}" | \
244 sed "s/PROCESSING_CHAIN/${PROCESSING_CHAIN}/g" | \
245 sed "s/SENSOR/${sensor}/g" | \
246 sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g" |\
247 sed "s/MAX_ACQUISITION_TIME/${max_acquisition_time}/g"`
249 tmr1=$(timer)
250 ../endpoint query ${ENDPOINT} "${query}"
251 tmr2=$(timer)
252 printf '%s \n' $((tmr2-tmr1)) >> ${timingsDiscover}
253 echo;echo;echo;echo "Discovered hotspots done!"
254 done
255 done