Working on making things more command friendly

1. CommandLineArgument allows one to parse the command line arguments or just load one from config.txt 2. Fixed up some documentation that was missing. 3. A few unit tests but more are needed.
fras2560 · Feb 2, 2018 · 1e94b35 · 1e94b35
1 parent 388a68c
commit 1e94b35
Show file tree

Hide file tree

Showing 12 changed files with 719 additions and 135 deletions.
diff --git a/config.txt b/config.txt
@@ -0,0 +1,14 @@
+# the index directory where output directories are stored
+indexDirectory: /home/d6fraser/git/UWResearch/resources/index/wikipedia_formula/
+# the the query file to use
+queriesFile: /home/d6fraser/git/UWResearch/resources/query/NTCIR11-Math-Wikipedia.xml
+# the judgements of the queries to use
+judgements: /home/d6fraser/git/UWResearch/resources/results/NTCIR11-wikipedia-formula-11.txt
+# the file path to the documents directory
+documentsDirectory: /home/d6fraser/Documents/Research/Datasets/NTCIR11_wikipedia_formula/
+# the folder where to put logfiles
+logFile: /home/d6fraser/git/UWResearch/resources/logs/
+# the folder where to put results and outputs
+resultsFile: /home/d6fraser/git/UWResearch/resources/output
+# the folder where to put query results
+queriesOutputFile: /home/d6fraser/git/UWResearch/resources/output
diff --git a/src/index/ConvertConfig.java b/src/index/ConvertConfig.java
@@ -100,6 +100,9 @@ public ConvertConfig(){
         this.initConfig();
     }
 
+    /**
+     * Initializes the Configuration
+     */
     private void initConfig(){
         this.windowSize = 1;
         this.shortened = true;
@@ -139,6 +142,10 @@ public boolean getMathBM25(){
         return this.mathbm25;
     }
 
+    /**
+     * Retursn a Config that can be used for searching
+     * @return ConvertConfig the config to use for searching
+     */
     public ConvertConfig getSearchConfig(){
         ConvertConfig searchConfig = this.copy();
         searchConfig.setBooleanAttribute(ConvertConfig.SYNONYMS, false);
@@ -256,6 +263,7 @@ public void setWindowSize(int n){
             this.windowSize = n;
         }
     }
+
     /**
      * Getter for different attributes
      * @param attribute the Attribute to get
@@ -587,6 +595,12 @@ public void loadConfig(Path directory) throws IOException, ConvertConfigExceptio
         }
     }
 
+    /**
+     * An Exception raised when config do not match or settings are set that conflicts
+     * @author Dallas Fraser
+     * @see ConvertConfigException
+     * @since 2017-09-06
+     */
     public class ConvertConfigException extends Exception{
         /**
          * 

diff --git a/src/index/ConvertMathML.java b/src/index/ConvertMathML.java
@@ -78,6 +78,14 @@ public Reader convert() throws IOException, InterruptedException{
         return this.convert(config);
     }
 
+    /**
+     * Converts the MathML file to Tuples using Tangent
+     * Returns a temporary file
+     * @param config the config file with the features to use
+     * @return Path the path to the temporary file
+     * @throws IOException raised when issues with file occur
+     * @throws InterruptedException raised when the process has an interruption
+     */
     public Path convertPath(ConvertConfig config) throws IOException, InterruptedException{
      // the output file
         Path new_file = Functions.createtempFile(this.file);
@@ -126,7 +134,7 @@ public Path convertPath(ConvertConfig config) throws IOException, InterruptedExc
      * @param config the configuration of features to be used when converting
      * @exception IOException
      * @exception InterruptedException
-     * @return path to the file that was converted
+     * @return Reader the reader to the output stream
      */
     public Reader convert(ConvertConfig config) throws IOException, InterruptedException{
         // the output file
@@ -165,7 +173,7 @@ public Reader convert(ConvertConfig config) throws IOException, InterruptedExcep
             }
             if (stdError.ready()) {
                 if ((s = stdError.readLine()) != null) {
-                    this.logger.log(Level.SEVERE, s);
+                    this.logger.log(Level.SEVERE, this.file.toString() + ":" + s);
                 }
             }
         }

diff --git a/src/index/FileStatistics.java b/src/index/FileStatistics.java
@@ -1,3 +1,18 @@
+/*
+ * Copyright 2017 Dallas Fraser
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package index;
 
 import java.io.IOException;

diff --git a/src/index/IndexThreadConsumer.java b/src/index/IndexThreadConsumer.java
@@ -1,20 +1,47 @@
+/*
+ * Copyright 2017 Dallas Fraser
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package index;
 
 import java.io.IOException;
 import java.util.concurrent.BlockingQueue;
 import org.apache.lucene.index.IndexWriter;
 
-
+/** A Thread that Indexes Files
+ * @author Dallas Fraser
+ * @since 2018-02-02
+ */
 public class IndexThreadConsumer implements Runnable{
     private BlockingQueue <IndexThreadObject>queue;
     private IndexWriter writer;
     private ConvertConfig config;
+    /**
+     * Constructor
+     * @param queue a queue holds the files to index
+     * @param writer the index to write to
+     * @param config the config with the features to use
+     */
     public IndexThreadConsumer(BlockingQueue <IndexThreadObject>queue, IndexWriter writer, ConvertConfig config){
         this.queue = queue;
         this.writer =  writer;
         this.config = config;
     }
 
+    /**
+     * The functions called when running the thread
+     */
     @Override
     public void run() {
         // TODO Auto-generated method stub

diff --git a/src/index/IndexThreadObject.java b/src/index/IndexThreadObject.java
@@ -1,18 +1,46 @@
+/*
+ * Copyright 2017 Dallas Fraser
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package index;
 
 import java.nio.file.Path;
 import java.nio.file.Paths;
 
+/** An Object used to store task information that is stored on the queue
+ * @author Dallas Fraser
+ * @since 2018-02-02
+ */
 public class IndexThreadObject {
     public static String COMPLETE = "COMPLETE";
     public static String MOVE_ON = "MOVE_ON";
     private String path;
     private long lastModified;
+    /**
+     * Constructor
+     * @param path the path to the file to index
+     * @param lastModified the time the file was last modified
+     */
     public IndexThreadObject(String path, long lastModified){
         this.path = path;
         this.lastModified = lastModified;
     }
 
+    /**
+     * Returns true if the tasks is signal the queue is complete
+     * @return boolean True if it is down
+     */
     public boolean complete(){
         boolean done = false;
         if (this.path.equals(IndexThreadObject.COMPLETE)){
@@ -21,10 +49,18 @@ public boolean complete(){
         return done;
     }
 
+    /**
+     * Returns the time the file was last modified
+     * @return long the time last modified
+     */
     public long getLastModified(){
         return this.lastModified;
     }
 
+    /**
+     * Returns the path to the file to index
+     * @return Path the path to the file
+     */
     public Path getFilePath(){
         return Paths.get(this.path);
     }

diff --git a/src/index/IndexThreadProducer.java b/src/index/IndexThreadProducer.java
@@ -1,3 +1,18 @@
+/*
+ * Copyright 2017 Dallas Fraser
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package index;
 
 import java.io.IOException;
@@ -11,16 +26,31 @@
 import utilities.Constants;
 import utilities.Functions;
 
+/**
+ * A Thread that waslks a document folder and adds file to index to a queue
+ * @author Dallas Fraser
+ * @since 2018-02-02
+ */
 public class IndexThreadProducer implements Runnable{
     private BlockingQueue <IndexThreadObject> queue;
     private Path documents;
     private int consumers;
+    /**
+     * Constructor
+     * @param queue the queue to add files to
+     * @param documents the directory that holds the documents
+     * @param consumers the number of threads that are indexing
+     */
     public IndexThreadProducer(BlockingQueue<IndexThreadObject> queue, Path documents, int consumers){
         this.queue = queue;
         this.documents = documents;
         this.consumers = consumers;
     }
 
+    /**
+     * The functions called when running the thread
+     * Walks the documents directory and then adds a stop signal for each consumer thread
+     */
     @Override
     public void run() {
         // add all the documents to the thread
@@ -32,6 +62,10 @@ public void run() {
         // System.out.println("Producer exiting");
     }
 
+    /**
+     * Walks the file path and adds files to a queue to be indexed
+     * @param path the documents directory
+     */
     public void indexDocs(Path path){
         if (Files.isDirectory(path)) {
             try {