Strabon

view scripts/v2.2/runNoaRefinementChain.sh @ 471:08d8767b08d5

beautify output
author Babis Nikolaou <charnik@di.uoa.gr>
date Thu Jul 19 23:37:56 2012 +0300 (2012-07-19)
parents 2e3c729225e5
children 27a92cb6315c
line source
1 #!/bin/bash
2 LOC="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
4 ENDPOINT="http://localhost:8080/endpoint"
5 DB="endpoint"
7 #HOTSPOTS_URL="http://jose.di.uoa.gr/rdf/hotspots"
8 HOTSPOTS_URL="http://jose.di.uoa.gr/rdf/hotspots/msg1"
10 logFile="chain.log"
12 function timer()
13 {
14 if [[ $# -eq 0 ]]; then
15 t=$(date '+%s%N')
16 t=$((t/1000000))
17 echo $t
18 else
19 local stime=$1
20 etime=$(date '+%s%N')
21 etime=$((etime/1000000))
23 if [[ -z "$stime" ]]; then stime=$etime; fi
24 dt=$((etime - stime)) #dt in milliseconds
25 dM=$((dt%1000))
26 Dt=$((dt/1000)) #delta t in seconds
27 ds=$((Dt % 60))
28 dm=$(((Dt / 60) % 60))
29 dh=$((Dt / 3600))
30 printf '%d:%02d:%02d.%03d' $dh $dm $ds $dM
31 fi
32 }
34 # find out the postgres service to use
35 postgres=$(ls -1 /etc/init.d/| grep postgres | head -1)
37 tomcat=
38 function chooseTomcat()
39 {
40 if test -s /etc/fedora-release ; then
41 tomcat="tomcat"
42 #elif test -s /etc/centos-release ; then
43 #elif test -s /etc/yellowdog-release ; then
44 #elif test -s /etc/redhat-release ; then
45 #elif test -s /etc/SuSE-release ; then
46 #elif test -s /etc/gentoo-release ; then
47 elif test -s /etc/lsb-release ; then # Ubuntu
48 tomcat=$(ls -1 /etc/init.d/| grep tomcat | head -1)
49 elif test -s /etc/debian_version ; then
50 tomcat="tomcat"
51 fi
53 # check for service availability
54 if ! test -e "/etc/init.d/${tomcat}"; then
55 tomcat=
56 fi
57 }
59 insertMunicipalities=`cat ${LOC}/insertMunicipalities.sparql`
60 deleteSeaHotspots=`cat ${LOC}/deleteSeaHotspots.sparql` # | sed 's/\"/\\\"/g'`
61 refinePartialSeaHotspots=`cat ${LOC}/refinePartialSeaHotspots.sparql` # | sed 's/\"/\\\"/g'`
62 refineTimePersistence=`cat ${LOC}/refineTimePersistence.sparql` # | sed 's/\"/\\\"/g'`
63 invalidForFires=`cat ${LOC}/landUseInvalidForFires.sparql`
64 discover=`cat ${LOC}/discover.sparql`
65 #InsertMunicipalities =`cat ${LOC}/InsertMunicipalities.sparql` # | sed 's/\"/\\\"/g'`
67 # Initialize (stop tomcat, restart postgres, drop/create database, start tomcat)
68 chooseTomcat
69 echo "stopping tomcat"
70 if test -z "${tomcat}"; then
71 # work-around for babis (standalone tomcat, with start_tomcat.sh and stop_tomcat.sh scripts)
72 stop_tomcat.sh
73 else
74 sudo service ${tomcat} stop
75 fi
77 sudo service ${postgres} restart
79 # get the main version of postgres
80 POSTGRES_MAIN_VERSION=$(sudo service ${postgres} status | grep -o '.\..' | cut -b 1)
82 echo "Dropping endpoint database";
83 dropdb ${DB}
85 echo "Creating endpoint database"
86 createdb ${DB}
88 # load data
89 curl -s http://dev.strabon.di.uoa.gr/rdf/Kallikratis-Coastline-Corine-dump-postgres-${POSTGRES_MAIN_VERSION}.tgz | tar xz -O | psql -d ${DB}
90 psql ${DB} -c 'VACUUM ANALYZE '
92 echo "starting tomcat"
93 if test -z "${tomcat}"; then
94 # work-around for babis (standalone tomcat, with start_tomcat.sh and stop_tomcat.sh scripts)
95 start_tomcat.sh
96 sleep 2
97 else
98 sudo service ${tomcat} start
99 fi
101 echo "initializing database"
102 echo "Timestamp Store Municipalities DeleteInSea InvalidForFires RefineInCoast TimePersistence" > stderr.txt
105 #./scripts/endpoint query ${ENDPOINT} "SELECT (COUNT(*) AS ?C) WHERE {?s ?p ?o}"
106 #sudo -u postgres psql -d endpoint -c 'CREATE INDEX datetime_values_idx_value ON datetime_values USING btree(value)';
107 #sudo -u postgres psql -d endpoint -c 'VACUUM ANALYZE;';
109 #for y in 2007 2008 2010 2011 ;do
110 for y in 2012; do
111 # get hotpost URLS
112 for hot in $(curl -s ${HOTSPOTS_URL}/${y}/ | grep -o '>HMSG.*\.nt' | colrm 1 1); do
113 file="${HOTSPOTS_URL}/${y}/${hot}"
115 time_status=$(echo ${hot} | egrep -o '[[:digit:]]{6}_[[:digit:]]{4}')
117 # get sensor
118 SENSOR=$(echo ${hot} | grep -o 'MSG.')
120 # get satellite
121 if test "${SENSOR}" = "MSG2"; then
122 SAT="METEOSAT9"
123 else
124 SAT="METEOSAT8"
125 SENSOR="MSG1_RSS"
126 fi
128 # get time information for acquisition and construct timestamp
129 year="20$(expr substr ${time_status} 1 2)"
130 month=$(expr substr ${time_status} 3 2)
131 day=$(expr substr ${time_status} 5 2)
132 time2=$(expr substr ${time_status} 8 2)
133 time2="${time2}:$(expr substr ${time_status} 10 2)"
135 # construct timestamp
136 TIMESTAMP="${year}-${month}-${day}T${time2}:00"
138 # store file
139 echo -n "storing " $file; echo; echo;
140 # ${countTime} ./strabon -db endpoint store $file
142 # print timestamp
143 echo -n "${TIMESTAMP} " >> stderr.txt
145 tmr1=$(timer)
146 ../endpoint store ${ENDPOINT} N-Triples -u ${file}
147 tmr2=$(timer)
148 printf '%s ' $((tmr2-tmr1)) >> stderr.txt
150 # sudo -u postgres psql -d endpoint -c 'VACUUM ANALYZE;';
152 echo;echo;echo;echo "File ${file} stored!" >> ${logFile}
154 # insertMunicipalities
155 echo -n "inserting Municipalities " ;echo; echo; echo;
156 # query=`echo "${insertMunicipalities}" `
157 # ${countTime} ./strabon -db endpoint update "${query}"
159 tmr1=$(timer)
161 query=`echo "${insertMunicipalities}" | sed "s/TIMESTAMP/${year}-${month}-${day}T${time2}:00/g" | \
162 sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
163 sed "s/SENSOR/${SENSOR}/g"`
165 ../endpoint update ${ENDPOINT} "${query}"
167 tmr2=$(timer)
168 printf '%s ' $((tmr2-tmr1)) >>stderr.txt
169 echo;echo;echo;echo "File ${file} inserted Municipalities!"
171 # deleteSeaHotspots
172 echo -n "Going to deleteSeaHotspots ${year}-${month}-${day}T${time2}:00 " ;echo; echo; echo;
173 query=`echo "${deleteSeaHotspots}" | sed "s/TIMESTAMP/${year}-${month}-${day}T${time2}:00/g" | \
174 sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
175 sed "s/SENSOR/${SENSOR}/g"`
176 # ${countTime} ./strabon -db endpoint update "${query}"
178 tmr1=$(timer)
179 ../endpoint update ${ENDPOINT} "${query}"
181 tmr2=$(timer)
182 printf '%s ' $((tmr2-tmr1)) >>stderr.txt
183 echo;echo;echo;echo "File ${file} deleteSeaHotspots done!"
185 # echo "Continue?"
186 # read a
187 # invalidForFires
188 echo -n "invalidForFires ${year}-${month}-${day}T${time2}:00 " ; echo; echo ; echo;
189 query=`echo "${invalidForFires}" | sed "s/TIMESTAMP/${year}-${month}-${day}T${time2}:00/g" | \
190 sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
191 sed "s/SENSOR/${SENSOR}/g" |\
192 sed "s/SAT/${SAT}/g"`
193 # ${countTime} ./strabon -db endpoint update "${query}"
194 tmr1=$(timer)
195 ../endpoint update ${ENDPOINT} "${query}"
196 tmr2=$(timer)
197 printf '%s ' $((tmr2-tmr1)) >>stderr.txt
198 echo "File ${file} invalidForFires done!"
200 # refinePartialSeaHotspots
201 echo -n "refinePartialSeaHotspots ${year}-${month}-${day}T${time2}:00 " ; echo; echo ; echo;
202 query=`echo "${refinePartialSeaHotspots}" | sed "s/TIMESTAMP/${year}-${month}-${day}T${time2}:00/g" | \
203 sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
204 sed "s/SENSOR/${SENSOR}/g" |\
205 sed "s/SAT/${SAT}/g"`
206 # ${countTime} ./strabon -db endpoint update "${query}"
207 tmr1=$(timer)
208 ../endpoint update ${ENDPOINT} "${query}"
209 tmr2=$(timer)
210 printf '%s ' $((tmr2-tmr1)) >>stderr.txt
212 echo "File ${file} refinePartialSeaHotspots done!"
213 # echo "Continue?"
214 # read a
216 # refineTimePersistence
217 echo -n "Going to refineTimePersistence ${year}-${month}-${day}T${time2}:00 ";echo;echo;echo;
218 min_acquisition_time=`date --date="${year}-${month}-${day} ${time2}:00 EEST -30 minutes" +%Y-%m-%dT%H:%m:00`
219 query=`echo "${refineTimePersistence}" | sed "s/TIMESTAMP/${year}-${month}-${day}T${time2}:00/g" | \
220 sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
221 sed "s/SENSOR/${SENSOR}/g" | \
222 sed "s/ACQUISITIONS_IN_HALF_AN_HOUR/3.0/g" | \
223 sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g" |\
224 sed "s/SAT/${SAT}/g"`
226 #sudo -u postgres psql -d ${DB} -c 'VACUUM ANALYZE;';
228 tmr1=$(timer)
229 ../endpoint update ${ENDPOINT} "${query}"
230 tmr2=$(timer)
231 printf '%s \n' $((tmr2-tmr1)) >>stderr.txt
232 echo;echo;echo;echo "File ${file} timePersistence done!"
233 # echo "Continue?"
234 # read a
237 # discover
238 echo -n "Going to discover ${year}-${month}-${day}T${time2}:00 ";echo;echo;echo;
239 min_acquisition_time=`date --date="${year}-${month}-${day} 00:00 EEST" +%Y-%m-%dT%H:%m:00`
240 max_acquisition_time=`date --date="${year}-${month}-${day} 23:59 EEST" +%Y-%m-%dT%H:%m:00`
241 query=`echo "${discover}" | \
242 sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
243 sed "s/SENSOR/${SENSOR}/g" | \
244 sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g" |\
245 sed "s/MAX_ACQUISITION_TIME/${max_acquisition_time}/g"`
247 tmr1=$(timer)
248 ../endpoint query ${ENDPOINT} "${query}"
249 tmr2=$(timer)
250 printf '%s \n' $((tmr2-tmr1)) >>discover.txt
251 echo;echo;echo;echo "Discovered hotspots done!"
253 done
254 done