Strabon

view scripts/v2.2/runNoaRefinementChain.sh @ 583:86e5ba1d801a

merge
author Teleios Repository <opteleios@teleios-repo.acsys.it>
date Sun Jul 22 14:37:38 2012 +0200 (2012-07-22)
parents 119576ba1127 e614ad7569c5
children
line source
1 #!/bin/bash
2 LOC="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
4 ENDPOINT="http://localhost:8080/endpoint"
5 DB="endpoint"
7 #HOTSPOTS_URL="http://jose.di.uoa.gr/rdf/hotspots/MSG2"
8 HOTSPOTS_URL="http://jose.di.uoa.gr/rdf/hotspots/MSG1"
10 logFile="chain.log"
12 function timer()
13 {
14 if [[ $# -eq 0 ]]; then
15 t=$(date '+%s%N')
16 t=$((t/1000000))
17 echo $t
18 else
19 local stime=$1
20 etime=$(date '+%s%N')
21 etime=$((etime/1000000))
23 if [[ -z "$stime" ]]; then stime=$etime; fi
24 dt=$((etime - stime)) #dt in milliseconds
25 dM=$((dt%1000))
26 Dt=$((dt/1000)) #delta t in seconds
27 ds=$((Dt % 60))
28 dm=$(((Dt / 60) % 60))
29 dh=$((Dt / 3600))
30 printf '%d:%02d:%02d.%03d' $dh $dm $ds $dM
31 fi
32 }
34 # find out the postgres service to use
35 postgres=$(ls -1 /etc/init.d/| grep postgres | head -1)
37 tomcat=
38 function chooseTomcat()
39 {
40 if test -s /etc/fedora-release ; then
41 tomcat="tomcat"
42 #elif test -s /etc/centos-release ; then
43 #elif test -s /etc/yellowdog-release ; then
44 #elif test -s /etc/redhat-release ; then
45 #elif test -s /etc/SuSE-release ; then
46 #elif test -s /etc/gentoo-release ; then
47 elif test -s /etc/lsb-release ; then # Ubuntu
48 tomcat=$(ls -1 /etc/init.d/| grep tomcat | head -1)
49 elif test -s /etc/debian_version ; then
50 tomcat="tomcat"
51 fi
53 # check for service availability
54 if ! test -e "/etc/init.d/${tomcat}"; then
55 tomcat=
56 fi
57 }
59 insertMunicipalities=`cat ${LOC}/insertMunicipalities.sparql`
60 deleteSeaHotspots=`cat ${LOC}/deleteSeaHotspots.sparql` # | sed 's/\"/\\\"/g'`
61 refinePartialSeaHotspots=`cat ${LOC}/refinePartialSeaHotspots.sparql` # | sed 's/\"/\\\"/g'`
62 refineTimePersistence=`cat ${LOC}/refineTimePersistence.sparql` # | sed 's/\"/\\\"/g'`
63 invalidForFires=`cat ${LOC}/landUseInvalidForFires.sparql`
64 discover=`cat ${LOC}/discover.sparql`
65 #InsertMunicipalities =`cat ${LOC}/InsertMunicipalities.sparql` # | sed 's/\"/\\\"/g'`
67 # Initialize (stop tomcat, restart postgres, drop/create database, start tomcat)
68 chooseTomcat
69 echo "stopping tomcat"
70 if test -z "${tomcat}"; then
71 # work-around for babis (standalone tomcat, with start_tomcat.sh and stop_tomcat.sh scripts)
72 stop_tomcat.sh
73 else
74 sudo service ${tomcat} stop
75 fi
77 sudo service ${postgres} restart
79 # get the main version of postgres
80 POSTGRES_MAIN_VERSION=$(sudo service ${postgres} status | grep -o '.\..' | cut -b 1)
82 echo "Dropping endpoint database";
83 dropdb ${DB}
85 echo "Creating endpoint database"
86 createdb ${DB}
88 # load data
89 curl -s http://dev.strabon.di.uoa.gr/rdf/Kallikratis-Coastline-Corine-dump-postgres-${POSTGRES_MAIN_VERSION}.tgz | tar xz -O | psql -d ${DB}
90 psql ${DB} -c 'VACUUM ANALYZE'
92 echo "starting tomcat"
93 if test -z "${tomcat}"; then
94 # work-around for babis (standalone tomcat, with start_tomcat.sh and stop_tomcat.sh scripts)
95 start_tomcat.sh
96 sleep 2
97 else
98 sudo service ${tomcat} start
99 fi
101 echo "initializing database"
102 echo "Timestamp Store Municipalities DeleteInSea InvalidForFires RefineInCoast TimePersistence" > stderr.txt
105 #./scripts/endpoint query ${ENDPOINT} "SELECT (COUNT(*) AS ?C) WHERE {?s ?p ?o}"
106 #sudo -u postgres psql -d endpoint -c 'CREATE INDEX datetime_values_idx_value ON datetime_values USING btree(value)';
107 #sudo -u postgres psql -d endpoint -c 'VACUUM ANALYZE;';
109 #for y in 2007 2008 2010 2011 ;do
110 for y in 2012; do
111 # get hotpost URLS
112 for hot in $(curl -s ${HOTSPOTS_URL}/${y}/ | grep -o '>HMSG.*\.nt' | colrm 1 1); do
113 file="${HOTSPOTS_URL}/${y}/${hot}"
115 time_status=$(echo ${hot} | egrep -o '[[:digit:]]{6}_[[:digit:]]{4}')
117 # get sensor
118 SENSOR=$(echo ${hot} | grep -o 'MSG.')
120 # get satellite
121 if test "${SENSOR}" = "MSG2"; then
122 SAT="METEOSAT9"
123 else
124 SAT="METEOSAT8"
125 SENSOR="MSG1_RSS"
126 fi
128 # get time information for acquisition and construct timestamp
129 year="20$(expr substr ${time_status} 1 2)"
130 month=$(expr substr ${time_status} 3 2)
131 day=$(expr substr ${time_status} 5 2)
132 time2=$(expr substr ${time_status} 8 2)
133 time2="${time2}:$(expr substr ${time_status} 10 2)"
135 # construct timestamp
136 TIMESTAMP="${year}-${month}-${day}T${time2}:00"
138 # store file
139 echo -n "storing " $file; echo; echo;
140 # ${countTime} ./strabon -db endpoint store $file
142 # print timestamp
143 echo -n "${TIMESTAMP} " >> stderr.txt
145 tmr1=$(timer)
146 ../endpoint store ${ENDPOINT} N-Triples -u ${file}
147 tmr2=$(timer)
148 printf '%s ' $((tmr2-tmr1)) >> stderr.txt
150 # sudo -u postgres psql -d endpoint -c 'VACUUM ANALYZE;';
152 echo;echo;echo;echo "File ${file} stored!" >> ${logFile}
154 # insertMunicipalities
155 echo -n "inserting Municipalities " ;echo; echo; echo;
156 # query=`echo "${insertMunicipalities}" `
157 # ${countTime} ./strabon -db endpoint update "${query}"
159 tmr1=$(timer)
161 query=`echo "${insertMunicipalities}" | sed "s/TIMESTAMP/${year}-${month}-${day}T${time2}:00/g" | \
162 sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
163 sed "s/SENSOR/${SENSOR}/g"`
165 ../endpoint update ${ENDPOINT} "${query}"
167 tmr2=$(timer)
168 printf '%s ' $((tmr2-tmr1)) >>stderr.txt
169 echo;echo;echo;echo "File ${file} inserted Municipalities!"
171 # execute an explicit VACUUM ANALYZE when a query takes longer than it should
172 duration=$((tmr2-tmr1))
173 if test ${duration} -ge 30000; then
174 psql ${DB} -c 'VACUUM ANALYZE'
175 echo "Explicit VACUUM ANALYZE"
176 fi
178 # deleteSeaHotspots
179 echo -n "Going to deleteSeaHotspots ${year}-${month}-${day}T${time2}:00 " ;echo; echo; echo;
180 query=`echo "${deleteSeaHotspots}" | sed "s/TIMESTAMP/${year}-${month}-${day}T${time2}:00/g" | \
181 sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
182 sed "s/SENSOR/${SENSOR}/g"`
183 # ${countTime} ./strabon -db endpoint update "${query}"
185 tmr1=$(timer)
186 ../endpoint update ${ENDPOINT} "${query}"
188 tmr2=$(timer)
189 printf '%s ' $((tmr2-tmr1)) >>stderr.txt
190 echo;echo;echo;echo "File ${file} deleteSeaHotspots done!"
192 # echo "Continue?"
193 # read a
194 # invalidForFires
195 echo -n "invalidForFires ${year}-${month}-${day}T${time2}:00 " ; echo; echo ; echo;
196 query=`echo "${invalidForFires}" | sed "s/TIMESTAMP/${year}-${month}-${day}T${time2}:00/g" | \
197 sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
198 sed "s/SENSOR/${SENSOR}/g" |\
199 sed "s/SAT/${SAT}/g"`
200 # ${countTime} ./strabon -db endpoint update "${query}"
201 tmr1=$(timer)
202 ../endpoint update ${ENDPOINT} "${query}"
203 tmr2=$(timer)
204 printf '%s ' $((tmr2-tmr1)) >>stderr.txt
205 echo "File ${file} invalidForFires done!"
207 # refinePartialSeaHotspots
208 echo -n "refinePartialSeaHotspots ${year}-${month}-${day}T${time2}:00 " ; echo; echo ; echo;
209 query=`echo "${refinePartialSeaHotspots}" | sed "s/TIMESTAMP/${year}-${month}-${day}T${time2}:00/g" | \
210 sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
211 sed "s/SENSOR/${SENSOR}/g" |\
212 sed "s/SAT/${SAT}/g"`
213 # ${countTime} ./strabon -db endpoint update "${query}"
214 tmr1=$(timer)
215 ../endpoint update ${ENDPOINT} "${query}"
216 tmr2=$(timer)
217 printf '%s ' $((tmr2-tmr1)) >>stderr.txt
219 echo "File ${file} refinePartialSeaHotspots done!"
220 # echo "Continue?"
221 # read a
223 # refineTimePersistence
224 echo -n "Going to refineTimePersistence ${year}-${month}-${day}T${time2}:00 ";echo;echo;echo;
225 min_acquisition_time=`date --date="${year}-${month}-${day} ${time2}:00 EEST -30 minutes" +%Y-%m-%dT%H:%m:00`
226 query=`echo "${refineTimePersistence}" | sed "s/TIMESTAMP/${year}-${month}-${day}T${time2}:00/g" | \
227 sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
228 sed "s/SENSOR/${SENSOR}/g" | \
229 sed "s/ACQUISITIONS_IN_HALF_AN_HOUR/3.0/g" | \
230 sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g" |\
231 sed "s/SAT/${SAT}/g"`
233 #sudo -u postgres psql -d ${DB} -c 'VACUUM ANALYZE;';
235 tmr1=$(timer)
236 ../endpoint update ${ENDPOINT} "${query}"
237 tmr2=$(timer)
238 printf '%s \n' $((tmr2-tmr1)) >>stderr.txt
239 echo;echo;echo;echo "File ${file} timePersistence done!"
240 # echo "Continue?"
241 # read a
244 # discover
245 echo -n "Going to discover ${year}-${month}-${day}T${time2}:00 ";echo;echo;echo;
246 min_acquisition_time=`date --date="${year}-${month}-${day} 00:00 EEST" +%Y-%m-%dT%H:%m:00`
247 max_acquisition_time=`date --date="${year}-${month}-${day} 23:59 EEST" +%Y-%m-%dT%H:%m:00`
248 query=`echo "${discover}" | \
249 sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
250 sed "s/SENSOR/${SENSOR}/g" | \
251 sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g" |\
252 sed "s/MAX_ACQUISITION_TIME/${max_acquisition_time}/g"`
254 tmr1=$(timer)
255 ../endpoint query ${ENDPOINT} "${query}"
256 tmr2=$(timer)
257 printf '%s \n' $((tmr2-tmr1)) >>discover.txt
258 echo;echo;echo;echo "Discovered hotspots done!"
260 done
261 done