url pattern matcher

web-archive-group · Mar 22, 2016 · d0e9212 · d0e9212
1 parent 93783c4
commit d0e9212
Showing 1 changed file with 7 additions and 0 deletions.
diff --git a/code/Warcbase/Graphx-URL-Pattern.scala b/code/Warcbase/Graphx-URL-Pattern.scala
@@ -0,0 +1,7 @@
+import org.warcbase.spark.rdd.RecordRDD._
+import org.warcbase.spark.matchbox.RecordLoader
+import org.warcbase.spark.matchbox.ExtractGraph
+
+val recs = RecordLoader.loadArchives("/collections/webarchives/geocities/warcs/*", sc).keepUrlPatterns(Set("http://geocities.com/EnchantedForest/.*".r))
+val graph = ExtractGraph(recs)
+graph.writeAsJson("nodes-accession01-static", "links-accession01-static")