lots of code cleanup, and new test code snippets for CLjure, Scala, a…

…nd JRuby
intermezzo-fr · Apr 6, 2010 · 96a3a62 · 96a3a62
1 parent 94fa7ee
commit 96a3a62
Show file tree

Hide file tree

Showing 8 changed files with 66 additions and 6 deletions.
diff --git a/knowledgebooks.jar b/knowledgebooks.jar
diff --git a/src/com/knowledgebooks/info_spiders/WebSpider.java b/src/com/knowledgebooks/info_spiders/WebSpider.java
@@ -26,7 +26,7 @@ public WebSpider(String root_url, int max_returned_pages) throws Exception {
     Set<String> already_visited = new HashSet<String>();
     urls.add(root_url);
     int num_fetched = 0;
-    while (num_fetched < max_returned_pages && !urls.isEmpty()) {
+    while (num_fetched <= max_returned_pages && !urls.isEmpty()) {
       try {
         String url_str = urls.remove(0);
         if (url_str.toLowerCase().indexOf(host) > -1 && url_str.indexOf("https:") == -1 && !already_visited.contains(url_str)) {

diff --git a/test/TestEmbeddedSesame.java b/test/TestEmbeddedSesame.java
@@ -36,16 +36,16 @@ public TestEmbeddedSesame() throws Exception {
     ts.deleteRepository("test-repo1");
     ts.createRepository("test-repo1");
     ts.registerFreetextPredicate("http://example.org/ontology/name");  // do this before adding triples
-    ts.registerFreetextPredicate("http://knowledgebooks.com/ontology/#summary");  // do this before adding triples
+    ts.registerFreetextPredicate("http://knowledgebooks.com/ontology/summary");  // do this before adding triples
     ts.initializeGeoLocation(3d);
 
-    ts.addTriple("<http://kbsportal.com/oak_creek_flooding>", "<http://knowledgebooks.com/ontology/#storyType>", "<http://knowledgebooks.com/ontology/#disaster>");
-    ts.addTriple("<http://kbsportal.com/oak_creek_flooding>", "<http://knowledgebooks.com/ontology/#summary>", "Oak Creek flooded last week affecting 5 businesses");
+    ts.addTriple("<http://kbsportal.com/oak_creek_flooding>", "<http://knowledgebooks.com/ontology/storyType>", "<http://knowledgebooks.com/ontology/disaster>");
+    ts.addTriple("<http://kbsportal.com/oak_creek_flooding>", "<http://knowledgebooks.com/ontology/summary>", "Oak Creek flooded last week affecting 5 businesses");
     ts.addTriple("<http://kbsportal.com/oak_creek_flooding>", "http://example.org/ontology/name", "Beth");
     ts.addTriple("http://example.org/people/alice", Triple.RDF_TYPE, "http://example.org/people/alice");
     ts.addTriple("http://example.org/people/alice", "http://example.org/ontology/name", "Alice");
     ts.addTriple("http://example.org/people/alice", Triple.RDF_LOCATION, ts.latLonToLiteral(+37.783333, -122.433334));
-    String sparql_query = "SELECT ?subject ?object WHERE { ?subject <http://knowledgebooks.com/ontology/#storyType> ?object . }";
+    String sparql_query = "SELECT ?subject ?object WHERE { ?subject <http://knowledgebooks.com/ontology/storyType> ?object . }";
     List<List<String>> results = ts.query(sparql_query);
     for (List<String> result : results) System.out.println("result: " + result);
     results = ts.getLocations(+37.113333d, -122.113334d, 500d);

diff --git a/test/TestScalaSparqlClient.scala b/test/TestScalaSparqlClient.scala
@@ -0,0 +1,26 @@
+import com.knowledgebooks.rdf.SparqlClient
+
+/**
+ * Copyright Mark Watson 2008-2010. All Rights Reserved.
+ * License: AGPL version 3 (http://www.gnu.org/licenses/agpl-3.0.txt)
+ * Alternative commercial license used under special arrangement (contact markw <at> markwatson <dot> com):
+ *    http://markwatson.com/commerciallicense.txt
+ */
+
+
+object TestScalaSparqlClient {
+  def main(args: Array[String]) {
+    val sparql =
+"""PREFIX foaf: <http://xmlns.com/foaf/0.1/>
+PREFIX dbpedia2: <http://dbpedia.org/property/>
+PREFIX dbpedia: <http://dbpedia.org/>
+SELECT ?name ?person WHERE {
+     ?person dbpedia2:birthPlace <http://dbpedia.org/resource/California> .
+     ?person foaf:name ?name .
+}
+LIMIT 10
+"""
+    val results = new SparqlClient("http://dbpedia.org/sparql", sparql)
+    println(results.variableBindings)
+  }
+}
diff --git a/test/TestScalaWebSpider.scala b/test/TestScalaWebSpider.scala
@@ -0,0 +1,17 @@
+import com.knowledgebooks.info_spiders.WebSpider
+
+/**
+ * Copyright Mark Watson 2008-2010. All Rights Reserved.
+ * License: AGPL version 3 (http://www.gnu.org/licenses/agpl-3.0.txt)
+ * Alternative commercial license used under special arrangement (contact markw <at> markwatson <dot> com):
+ *     http://markwatson.com/commerciallicense.txt
+ */
+
+
+object TestScalaWebSpider {
+  def main(args: Array[String]) {
+    val results = new WebSpider("http://www.knowledgebooks.com", 2)
+    println(results.url_content_lists.get(0))
+    println(results.url_content_lists.get(1))
+  }
+}
diff --git a/test/TestWebSpider.java b/test/TestWebSpider.java
@@ -11,7 +11,7 @@
  */
 public class TestWebSpider {
   static public void main(String[] args) throws Exception {
-    WebSpider ws = new WebSpider("http://www.knowledgebooks.com", 20);
+    WebSpider ws = new WebSpider("http://www.knowledgebooks.com", 3);
     for (List<String> ls : ws.url_content_lists) {
       String url = ls.get(0);
       String text = ls.get(1);

diff --git a/test/test-webspider-cloujure.clj b/test/test-webspider-cloujure.clj
@@ -0,0 +1,8 @@
+(import '(com.knowledgebooks.info_spiders WebSpider))
+
+(defn get-pages [starting-url max-pages]
+  (let [ws (new WebSpider starting-url max-pages)]
+    (map seq (.url_content_lists ws))))
+
+(println (get-pages "http://www.knowledgebooks.com" 2))
+
diff --git a/test/test_ruby_web_spider.rb b/test/test_ruby_web_spider.rb
@@ -0,0 +1,9 @@
+require 'java'
+(Dir.glob("lib/*.jar")).each do |fname|
+  require fname
+end
+require "knowledgebooks.jar"
+require 'pp'
+
+results = com.knowledgebooks.info_spiders.WebSpider.new("http://www.knowledgebooks.com", 2)
+pp results.url_content_lists