Strabon
changeset 941:21c54b20f4fa temporals
1)suspected there was an error when storing many NQUAD statements as string. It could not be reproduced but some related tests were added. 2) rewritten triples are now stored immediately as they are created to avoid the respective cost in memory
| author | Konstantina Bereta <Konstantina.Bereta@di.uoa.gr> |
|---|---|
| date | Thu Mar 28 13:45:39 2013 +0200 (2013-03-28) |
| parents | a1e2561b7c2c |
| children | 6f223f6b7aeb |
| files | endpoint/src/main/java/eu/earthobservatory/org/StrabonEndpoint/StrabonBeanWrapper.java runtime/src/main/java/eu/earthobservatory/runtime/generaldb/NQuadsTranslator.java runtime/src/main/java/eu/earthobservatory/runtime/generaldb/QuadRDFHandler.java runtime/src/main/java/eu/earthobservatory/runtime/generaldb/Strabon.java runtime/src/test/java/eu/earthobservatory/runtime/postgres/temporals/storeTests.java |
line diff
1.1 --- a/endpoint/src/main/java/eu/earthobservatory/org/StrabonEndpoint/StrabonBeanWrapper.java Wed Mar 27 13:30:17 2013 +0200 1.2 +++ b/endpoint/src/main/java/eu/earthobservatory/org/StrabonEndpoint/StrabonBeanWrapper.java Thu Mar 28 13:45:39 2013 +0200 1.3 @@ -295,7 +295,7 @@ 1.4 in= new ByteArrayInputStream(source_data.getBytes()); 1.5 } 1.6 //ByteArrayInputStream in = new ByteArrayInputStream(); 1.7 - NQuadsTranslator translator = new NQuadsTranslator(); 1.8 + NQuadsTranslator translator = new NQuadsTranslator(conn); 1.9 1.10 Collection<Statement> statements = translator.translate(in, ""); 1.11 for(Statement st: statements)
2.1 --- a/runtime/src/main/java/eu/earthobservatory/runtime/generaldb/NQuadsTranslator.java Wed Mar 27 13:30:17 2013 +0200 2.2 +++ b/runtime/src/main/java/eu/earthobservatory/runtime/generaldb/NQuadsTranslator.java Thu Mar 28 13:45:39 2013 +0200 2.3 @@ -22,6 +22,7 @@ 2.4 2.5 2.6 import org.openrdf.model.Statement; 2.7 +import org.openrdf.repository.sail.SailRepositoryConnection; 2.8 import org.openrdf.rio.RDFHandlerException; 2.9 import org.openrdf.rio.RDFParseException; 2.10 import org.openrdf.rio.helpers.StatementCollector; 2.11 @@ -33,21 +34,16 @@ 2.12 2.13 private NQuadsParser parser; 2.14 private QuadRDFHandler handler; 2.15 - private StringBuffer handledTriples; 2.16 - 2.17 - 2.18 - 2.19 - public NQuadsTranslator() { 2.20 + private SailRepositoryConnection connection; 2.21 + 2.22 + public NQuadsTranslator(SailRepositoryConnection connection) { 2.23 super(); 2.24 this.parser = new NQuadsParser(); 2.25 - this.handler = new QuadRDFHandler(); 2.26 - this.handledTriples = new StringBuffer(1024); 2.27 + this.handler = new QuadRDFHandler(connection); 2.28 + this.connection = connection; 2.29 } 2.30 2.31 - public StringBuffer getHandledTriples() { 2.32 - return handledTriples; 2.33 - } 2.34 - 2.35 + 2.36 public Collection<Statement> translate(InputStream is,String baseURI) 2.37 { 2.38 Collection<Statement> statements = null; 2.39 @@ -56,8 +52,6 @@ 2.40 try { 2.41 2.42 parser.parse(is, "http://test.base.uri"); 2.43 - handledTriples = handler.getTriples(); 2.44 - // System.out.println("HANDLED TRIPLES: "+handledTriples.toString()); 2.45 } catch (RDFParseException e) { 2.46 // TODO Auto-generated catch block 2.47 e.printStackTrace();
3.1 --- a/runtime/src/main/java/eu/earthobservatory/runtime/generaldb/QuadRDFHandler.java Wed Mar 27 13:30:17 2013 +0200 3.2 +++ b/runtime/src/main/java/eu/earthobservatory/runtime/generaldb/QuadRDFHandler.java Thu Mar 28 13:45:39 2013 +0200 3.3 @@ -15,12 +15,18 @@ 3.4 */ 3.5 package eu.earthobservatory.runtime.generaldb; 3.6 3.7 +import java.io.IOException; 3.8 +import java.io.StringReader; 3.9 import java.text.ParseException; 3.10 3.11 //import org.junit.Assert; 3.12 import org.openrdf.model.Resource; 3.13 import org.openrdf.model.Statement; 3.14 +import org.openrdf.model.impl.URIImpl; 3.15 import org.openrdf.query.algebra.evaluation.function.temporal.stsparql.relation.TemporalConstants; 3.16 +import org.openrdf.repository.RepositoryException; 3.17 +import org.openrdf.repository.sail.SailRepositoryConnection; 3.18 +import org.openrdf.rio.RDFFormat; 3.19 //import org.openrdf.model.URI; 3.20 import org.openrdf.rio.RDFHandlerException; 3.21 import org.openrdf.rio.RDFParseException; 3.22 @@ -28,19 +34,20 @@ 3.23 3.24 public class QuadRDFHandler extends StatementCollector { 3.25 3.26 - private StringBuffer triples = new StringBuffer(4096); 3.27 + private SailRepositoryConnection connection= null; 3.28 + 3.29 + 3.30 + 3.31 + public QuadRDFHandler(SailRepositoryConnection connection) { 3.32 + super(); 3.33 + this.connection = connection; 3.34 + } 3.35 3.36 - 3.37 - @Override 3.38 + @Override 3.39 public void startRDF() throws RDFHandlerException { 3.40 super.startRDF(); 3.41 - triples.append("\n"); 3.42 } 3.43 - 3.44 - public StringBuffer getTriples() 3.45 - { 3.46 - return triples; 3.47 - }; 3.48 + 3.49 3.50 @Override 3.51 public void endRDF() throws RDFHandlerException { 3.52 @@ -65,12 +72,20 @@ 3.53 Resource graph = parser.createValidTimeURI(validPeriod); 3.54 3.55 String triple = "<"+graph.toString()+">"+ " <http://strdf.di.uoa.gr/ontology#hasValidTime> "+ validPeriod+ " .\n" ; 3.56 - 3.57 - if (!triples.toString().contains(triple)) 3.58 - { 3.59 - triples.append(triple); 3.60 - 3.61 - } 3.62 + 3.63 + 3.64 + try { 3.65 + //connection.add(new URIImpl("<"+graph.toString()+">"),new URIImpl(" <http://strdf.di.uoa.gr/ontology#hasValidTime>"), new URIImpl(validPeriod)); 3.66 + StringReader reader = new StringReader(triple); 3.67 + connection.add(reader, "null", RDFFormat.NTRIPLES); 3.68 + } catch (RepositoryException e) { 3.69 + // TODO Auto-generated catch block 3.70 + System.out.println("Error in QuadRDFHandler: could not store rewritter triple"); 3.71 + e.printStackTrace(); 3.72 + } catch (IOException e) { 3.73 + // TODO Auto-generated catch block 3.74 + e.printStackTrace(); 3.75 + } 3.76 3.77 3.78 } catch (RDFParseException e) {
4.1 --- a/runtime/src/main/java/eu/earthobservatory/runtime/generaldb/Strabon.java Wed Mar 27 13:30:17 2013 +0200 4.2 +++ b/runtime/src/main/java/eu/earthobservatory/runtime/generaldb/Strabon.java Thu Mar 28 13:45:39 2013 +0200 4.3 @@ -74,6 +74,7 @@ 4.4 public static final String FORMAT_GEOJSON = "GeoJSON"; 4.5 public static final String FORMAT_EXP = "EXP"; 4.6 public static final String FORMAT_HTML = "HTML"; 4.7 + public static final int LINES_IN_BATCH = 100; 4.8 4.9 public static final String NEWLINE = "\n"; 4.10 4.11 @@ -517,7 +518,7 @@ 4.12 BufferedReader br = new BufferedReader(reader); 4.13 while ((line = br.readLine()) != null) { 4.14 counter++; 4.15 - if(counter%500 ==0 ) 4.16 + if(counter%LINES_IN_BATCH ==0 ) 4.17 { 4.18 storeString(batch.toString(), baseURI, context, format); 4.19 batch = new StringBuilder(); 4.20 @@ -608,7 +609,7 @@ 4.21 if(format.equals(RDFFormat.NQUADS)) 4.22 { 4.23 ByteArrayInputStream in = new ByteArrayInputStream(text.getBytes()); 4.24 - NQuadsTranslator translator = new NQuadsTranslator(); 4.25 + NQuadsTranslator translator = new NQuadsTranslator(con1); 4.26 4.27 Collection<Statement> statements = translator.translate(in, baseURI); 4.28 Iterator iterator = statements.iterator(); 4.29 @@ -634,8 +635,9 @@ 4.30 } 4.31 4.32 } 4.33 - StringReader quadGraphReader = new StringReader(translator.getHandledTriples().toString()); 4.34 - con1.add(quadGraphReader, "", RDFFormat.NTRIPLES); 4.35 + //StringReader quadGraphReader = new StringReader(translator.getHandledTriples().toString()); 4.36 + //System.out.println("HANDLED TRIPLES" +translator.getHandledTriples().toString()); 4.37 + //con1.add(quadGraphReader, "", RDFFormat.NTRIPLES); 4.38 return; 4.39 } 4.40
5.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000 5.2 +++ b/runtime/src/test/java/eu/earthobservatory/runtime/postgres/temporals/storeTests.java Thu Mar 28 13:45:39 2013 +0200 5.3 @@ -0,0 +1,128 @@ 5.4 +package eu.earthobservatory.runtime.postgres.temporals; 5.5 + 5.6 + 5.7 +import static org.junit.Assert.*; 5.8 + 5.9 +import java.io.IOException; 5.10 +import java.io.InputStream; 5.11 +import java.sql.SQLException; 5.12 +import java.util.ArrayList; 5.13 +import java.util.Properties; 5.14 + 5.15 +import org.junit.AfterClass; 5.16 +import org.junit.BeforeClass; 5.17 +import org.junit.Test; 5.18 +import org.openrdf.query.MalformedQueryException; 5.19 +import org.openrdf.query.QueryEvaluationException; 5.20 +import org.openrdf.query.TupleQueryResult; 5.21 +import org.openrdf.query.TupleQueryResultHandlerException; 5.22 +//import org.openrdf.query.resultio.Format; 5.23 +import eu.earthobservatory.utils.Format; 5.24 +import org.openrdf.query.resultio.TupleQueryResultWriter; 5.25 +import org.openrdf.query.resultio.stSPARQLQueryResultWriterFactory; 5.26 +import org.openrdf.repository.RepositoryException; 5.27 +import org.openrdf.rio.RDFFormat; 5.28 +import org.openrdf.rio.RDFHandlerException; 5.29 +import org.openrdf.rio.RDFParseException; 5.30 + 5.31 + 5.32 +import eu.earthobservatory.runtime.generaldb.*; 5.33 +import eu.earthobservatory.runtime.postgres.temporals.TemplateTests; 5.34 + 5.35 + 5.36 +/** 5.37 + * A set of simple tests on query rewriting functionality 5.38 + * 5.39 + * @author Konstantina Bereta <Konstantina.Bereta@di.uoa.gr> 5.40 + */ 5.41 +public class storeTests { 5.42 + 5.43 + protected static Strabon strabon; 5.44 + //using a new database to to the stores 5.45 + private static final String database = "store-test"; 5.46 + public static java.sql.Connection conn = null; 5.47 + public static String jdbcDriver = null; 5.48 + public static String serverName = null; 5.49 + public static String username = null; 5.50 + public static String password = null; 5.51 + public static Integer port = null; 5.52 + 5.53 + 5.54 + 5.55 + 5.56 + protected static final String prefixes = 5.57 + "PREFIX strdf: <http://strdf.di.uoa.gr/ontology#> \n" + 5.58 + "PREFIX xsd: <http://www.w3.org/2001/XMLSchema#> \n" + 5.59 + "PREFIX noa: <http://teleios.di.uoa.gr/ontologies/noaOntology.owl#> \n"; 5.60 + 5.61 + 5.62 + @BeforeClass 5.63 + public static void beforeClass() throws Exception 5.64 + { 5.65 + // Read properties 5.66 + Properties properties = new Properties(); 5.67 + InputStream propertiesStream = SimpleTests.class.getResourceAsStream("/databases.properties"); 5.68 + properties.load(propertiesStream); 5.69 + 5.70 + serverName = properties.getProperty("temporal.postgis.serverName"); 5.71 + port = Integer.parseInt(properties.getProperty("temporal.postgis.port")); 5.72 + username = properties.getProperty("temporal.postgis.username"); 5.73 + password = properties.getProperty("temporal.postgis.password"); 5.74 + strabon = new eu.earthobservatory.runtime.postgis.Strabon(database, username, password, port, serverName, true); 5.75 + 5.76 + } 5.77 + 5.78 + @AfterClass 5.79 + public static void afterClass() throws SQLException 5.80 + { 5.81 + TemplateTests.afterClass(strabon); 5.82 + } 5.83 + 5.84 + 5.85 + 5.86 + @Test 5.87 + public void testStoreString() 5.88 + { 5.89 + String text1 = "<http://example.org/itemOfString1> <http://example.org/id> \"String111\" \"[2005-11-19T12:41:00,2010-11-19T13:41:00]\"^^<http://strdf.di.uoa.gr/ontology#period> . \n" + 5.90 + "<http://example.org/itemOfString2> <http://example.org/id> \"String112\" \"[2005-11-19T12:42:00,2010-11-19T13:41:00]\"^^<http://strdf.di.uoa.gr/ontology#period>."; 5.91 + 5.92 + String text2 = "<http://example.org/itemOfString1> <http://example.org/id> \"String121\" \"[2002-11-19T12:41:00,2010-11-19T13:41:00]\"^^<http://strdf.di.uoa.gr/ontology#period> . \n" + 5.93 + "<http://example.org/itemOfString22> <http://example.org/id> \"String122\" \"[2002-11-19T12:42:00,2010-11-19T13:41:00]\"^^<http://strdf.di.uoa.gr/ontology#period>."; 5.94 + try { 5.95 + strabon.storeInRepo(text1, "NQUADS"); 5.96 + strabon.storeInRepo(text2, "NQUADS"); 5.97 + 5.98 + } catch (RDFParseException e) { 5.99 + // TODO Auto-generated catch block 5.100 + e.printStackTrace(); 5.101 + } catch (RepositoryException e) { 5.102 + // TODO Auto-generated catch block 5.103 + e.printStackTrace(); 5.104 + } catch (RDFHandlerException e) { 5.105 + // TODO Auto-generated catch block 5.106 + e.printStackTrace(); 5.107 + } catch (IOException e) { 5.108 + // TODO Auto-generated catch block 5.109 + e.printStackTrace(); 5.110 + } catch (InvalidDatasetFormatFault e) { 5.111 + // TODO Auto-generated catch block 5.112 + e.printStackTrace(); 5.113 + } 5.114 + 5.115 + } 5.116 + 5.117 + @Test 5.118 + public void testStoreURL() throws RDFParseException, RepositoryException, RDFHandlerException, IOException, InvalidDatasetFormatFault 5.119 + { 5.120 + strabon.storeInRepo("http://manolee.di.uoa.gr/temporals/temporal-periods.nq", "NQUADS"); 5.121 + 5.122 + } 5.123 + 5.124 + @Test 5.125 + public void testStoreFile() throws RDFParseException, RepositoryException, RDFHandlerException, IOException, InvalidDatasetFormatFault 5.126 + { 5.127 + strabon.storeInRepo("/temporal-periods.nq", "NQUADS"); 5.128 + 5.129 + } 5.130 + 5.131 +} 5.132 \ No newline at end of file