Strabon

view scripts/archive/v2.2/runNoaRefinementChain.sh @ 1277:92b90ac06d39

[maven-release-plugin] prepare release v3.2.9
author Babis Nikolaou <charnik@di.uoa.gr>
date Sat Oct 19 23:37:23 2013 +0300 (2013-10-19)
parents 86e5ba1d801a b76d08b43131
children
line source
1 #!/bin/bash
2 LOC="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
4 ENDPOINT="http://localhost:8080/endpoint"
5 DB="endpoint"
7 #HOTSPOTS_URL="http://jose.di.uoa.gr/rdf/hotspots/MSG2"
8 HOTSPOTS_URL="http://jose.di.uoa.gr/rdf/hotspots/MSG1"
10 logFile="chain.log"
12 function timer()
13 {
14 if [[ $# -eq 0 ]]; then
15 t=$(date '+%s%N')
16 t=$((t/1000000))
17 echo $t
18 else
19 local stime=$1
20 etime=$(date '+%s%N')
21 etime=$((etime/1000000))
23 if [[ -z "$stime" ]]; then stime=$etime; fi
24 dt=$((etime - stime)) #dt in milliseconds
25 dM=$((dt%1000))
26 Dt=$((dt/1000)) #delta t in seconds
27 ds=$((Dt % 60))
28 dm=$(((Dt / 60) % 60))
29 dh=$((Dt / 3600))
30 printf '%d:%02d:%02d.%03d' $dh $dm $ds $dM
31 fi
32 }
34 # find out the postgres service to use
35 postgres=$(ls -1 /etc/init.d/| grep postgres | head -1)
37 tomcat=
38 function chooseTomcat()
39 {
40 if test -s /etc/fedora-release ; then
41 tomcat="tomcat"
42 #elif test -s /etc/centos-release ; then
43 #elif test -s /etc/yellowdog-release ; then
44 #elif test -s /etc/redhat-release ; then
45 #elif test -s /etc/SuSE-release ; then
46 #elif test -s /etc/gentoo-release ; then
47 elif test -s /etc/lsb-release ; then # Ubuntu
48 tomcat=$(ls -1 /etc/init.d/| grep tomcat | head -1)
49 elif test -s /etc/debian_version ; then
50 tomcat="tomcat"
51 fi
53 # check for service availability
54 if ! test -e "/etc/init.d/${tomcat}"; then
55 tomcat=
56 fi
57 }
59 insertMunicipalities=`cat ${LOC}/insertMunicipalities.sparql`
60 deleteSeaHotspots=`cat ${LOC}/deleteSeaHotspots.sparql` # | sed 's/\"/\\\"/g'`
61 refinePartialSeaHotspots=`cat ${LOC}/refinePartialSeaHotspots.sparql` # | sed 's/\"/\\\"/g'`
62 refineTimePersistence=`cat ${LOC}/refineTimePersistence.sparql` # | sed 's/\"/\\\"/g'`
63 invalidForFires=`cat ${LOC}/landUseInvalidForFires.sparql`
64 discover=`cat ${LOC}/discover.sparql`
65 #InsertMunicipalities =`cat ${LOC}/InsertMunicipalities.sparql` # | sed 's/\"/\\\"/g'`
67 # Initialize (stop tomcat, restart postgres, drop/create database, start tomcat)
68 chooseTomcat
69 echo "stopping tomcat"
70 if test -z "${tomcat}"; then
71 # work-around for babis (standalone tomcat, with start_tomcat.sh and stop_tomcat.sh scripts)
72 stop_tomcat.sh
73 else
74 sudo service ${tomcat} stop
75 fi
77 sudo service ${postgres} restart
79 # get the main version of postgres
80 POSTGRES_MAIN_VERSION=$(sudo service ${postgres} status | grep -o '.\..' | cut -b 1)
82 echo "Dropping endpoint database";
83 dropdb ${DB}
85 echo "Creating endpoint database"
86 createdb ${DB}
88 # load data
89 curl -s http://dev.strabon.di.uoa.gr/rdf/Kallikratis-Coastline-Corine-dump-postgres-${POSTGRES_MAIN_VERSION}.tgz | tar xz -O | psql -d ${DB}
90 psql ${DB} -c 'VACUUM ANALYZE'
92 echo "starting tomcat"
93 if test -z "${tomcat}"; then
94 # work-around for babis (standalone tomcat, with start_tomcat.sh and stop_tomcat.sh scripts)
95 start_tomcat.sh
96 sleep 2
97 else
98 sudo service ${tomcat} start
99 fi
101 echo "initializing database"
102 echo "Timestamp Store Municipalities DeleteInSea InvalidForFires RefineInCoast TimePersistence" > stderr.txt
105 #./scripts/endpoint query ${ENDPOINT} "SELECT (COUNT(*) AS ?C) WHERE {?s ?p ?o}"
106 #sudo -u postgres psql -d endpoint -c 'CREATE INDEX datetime_values_idx_value ON datetime_values USING btree(value)';
107 #sudo -u postgres psql -d endpoint -c 'VACUUM ANALYZE;';
109 #for y in 2007 2008 2010 2011; do
110 for y in 2012; do
111 # get hotpost URLS
112 for hot in $(curl -s ${HOTSPOTS_URL}/${y}/ | grep -o '>HMSG.*\.nt' | colrm 1 1); do
113 file="${HOTSPOTS_URL}/${y}/${hot}"
115 time_status=$(echo ${hot} | egrep -o '[[:digit:]]{6}_[[:digit:]]{4}')
117 # get sensor
118 SENSOR=$(echo ${hot} | grep -o 'MSG.')
120 # get satellite and set number of acquisitions per hour
121 if test "${SENSOR}" = "MSG2"; then
122 SAT="METEOSAT9"
124 N_ACQUISITIONS=3.0
125 else
126 SAT="METEOSAT8"
127 SENSOR="MSG1_RSS"
129 N_ACQUISITIONS=7.0
130 fi
132 # get time information for acquisition and construct timestamp
133 year="20$(expr substr ${time_status} 1 2)"
134 month=$(expr substr ${time_status} 3 2)
135 day=$(expr substr ${time_status} 5 2)
136 time2=$(expr substr ${time_status} 8 2)
137 time2="${time2}:$(expr substr ${time_status} 10 2)"
139 # construct timestamp
140 TIMESTAMP="${year}-${month}-${day}T${time2}:00"
142 # store file
143 echo -n "storing " $file; echo; echo;
144 # ${countTime} ./strabon -db endpoint store $file
146 # print timestamp
147 echo -n "${TIMESTAMP} " >> stderr.txt
149 tmr1=$(timer)
150 ../endpoint store ${ENDPOINT} N-Triples -u ${file}
151 tmr2=$(timer)
152 printf '%s ' $((tmr2-tmr1)) >> stderr.txt
154 # sudo -u postgres psql -d endpoint -c 'VACUUM ANALYZE;';
156 echo;echo;echo;echo "File ${file} stored!" >> ${logFile}
158 # insertMunicipalities
159 echo -n "inserting Municipalities " ;echo; echo; echo;
160 # query=`echo "${insertMunicipalities}" `
161 # ${countTime} ./strabon -db endpoint update "${query}"
163 tmr1=$(timer)
165 query=`echo "${insertMunicipalities}" | sed "s/TIMESTAMP/${TIMESTAMP}/g" | \
166 sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
167 sed "s/SENSOR/${SENSOR}/g"`
169 ../endpoint update ${ENDPOINT} "${query}"
171 tmr2=$(timer)
172 printf '%s ' $((tmr2-tmr1)) >>stderr.txt
173 echo;echo;echo;echo "File ${file} inserted Municipalities!"
175 # execute an explicit VACUUM ANALYZE when a query takes longer than it should
176 duration=$((tmr2-tmr1))
177 if test ${duration} -ge 30000; then
178 psql ${DB} -c 'VACUUM ANALYZE'
179 echo "Explicit VACUUM ANALYZE"
180 fi
182 # deleteSeaHotspots
183 echo -n "Going to deleteSeaHotspots ${TIMESTAMP} " ;echo; echo; echo;
184 query=`echo "${deleteSeaHotspots}" | sed "s/TIMESTAMP/${TIMESTAMP}/g" | \
185 sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
186 sed "s/SENSOR/${SENSOR}/g"`
187 # ${countTime} ./strabon -db endpoint update "${query}"
189 tmr1=$(timer)
190 ../endpoint update ${ENDPOINT} "${query}"
192 tmr2=$(timer)
193 printf '%s ' $((tmr2-tmr1)) >>stderr.txt
194 echo;echo;echo;echo "File ${file} deleteSeaHotspots done!"
196 # invalidForFires
197 echo -n "invalidForFires ${TIMESTAMP} " ; echo; echo ; echo;
198 query=`echo "${invalidForFires}" | sed "s/TIMESTAMP/${TIMESTAMP}/g" | \
199 sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
200 sed "s/SENSOR/${SENSOR}/g"`
202 # ${countTime} ./strabon -db endpoint update "${query}"
203 tmr1=$(timer)
204 ../endpoint update ${ENDPOINT} "${query}"
205 tmr2=$(timer)
206 printf '%s ' $((tmr2-tmr1)) >>stderr.txt
207 echo "File ${file} invalidForFires done!"
209 # refinePartialSeaHotspots
210 echo -n "refinePartialSeaHotspots ${TIMESTAMP} " ; echo; echo ; echo;
211 query=`echo "${refinePartialSeaHotspots}" | sed "s/TIMESTAMP/${TIMESTAMP}/g" | \
212 sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
213 sed "s/SENSOR/${SENSOR}/g" |\
214 sed "s/SAT/${SAT}/g"`
215 # ${countTime} ./strabon -db endpoint update "${query}"
216 tmr1=$(timer)
217 ../endpoint update ${ENDPOINT} "${query}"
218 tmr2=$(timer)
219 printf '%s ' $((tmr2-tmr1)) >>stderr.txt
221 echo "File ${file} refinePartialSeaHotspots done!"
222 # echo "Continue?"
223 # read a
225 # refineTimePersistence
226 echo -n "Going to refineTimePersistence ${TIMESTAMP} ";echo;echo;echo;
227 min_acquisition_time=`date --date="${year}-${month}-${day} ${time2}:00 EEST -30 minutes" +%Y-%m-%dT%H:%M:00`
228 query=`echo "${refineTimePersistence}" | sed "s/TIMESTAMP/${TIMESTAMP}/g" | \
229 sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
230 sed "s/SENSOR/${SENSOR}/g" | \
231 sed "s/ACQUISITIONS_IN_HALF_AN_HOUR/${N_ACQUISITIONS}/g" | \
232 sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g" |\
233 sed "s/SAT/${SAT}/g"`
235 #sudo -u postgres psql -d ${DB} -c 'VACUUM ANALYZE;';
237 tmr1=$(timer)
238 ../endpoint update ${ENDPOINT} "${query}"
239 tmr2=$(timer)
240 printf '%s \n' $((tmr2-tmr1)) >>stderr.txt
241 echo;echo;echo;echo "File ${file} timePersistence done!"
242 # echo "Continue?"
243 # read a
246 # discover
247 echo -n "Going to discover ${TIMESTAMP} ";echo;echo;echo;
248 min_acquisition_time=`date --date="${year}-${month}-${day} 00:00 EEST" +%Y-%m-%dT%H:%M:00`
249 max_acquisition_time=`date --date="${year}-${month}-${day} 23:59 EEST" +%Y-%m-%dT%H:%M:00`
250 query=`echo "${discover}" | \
251 sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
252 sed "s/SENSOR/${SENSOR}/g" | \
253 sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g" |\
254 sed "s/MAX_ACQUISITION_TIME/${max_acquisition_time}/g"`
256 tmr1=$(timer)
257 ../endpoint query ${ENDPOINT} "${query}"
258 tmr2=$(timer)
259 printf '%s \n' $((tmr2-tmr1)) >>discover.txt
260 echo;echo;echo;echo "Discovered hotspots done!"
261 done
262 done