Adding doc changes

README.md

@@ -4,7 +4,7 @@ The Edison Web interface is a Node-based web application that allows one to sear
 
 This application requires the following to generate its components and to run:
    - node
-   - npm (node package manager)
+   - npm (node package manager - version >= 3.8.3)
    - bash
    - java (version > 1.6)
    - mvn
@@ -13,13 +13,15 @@ This application requires the following to generate its components and to run:
 Steps to run the application:
 
 1) Extract the javadoc from the online javadoc repository. 
-   - Replace the base_javadoc_url variable with the appropriate base_url (The url that produces the API overview page)
+   - Replace the base_javadoc_url variable in the 'extract_javadoc.sh' script in the javadoc_scraper folder with the appropriate base_url (The url that produces the API overview page)
    - The current default is: "http://cogcomp.cs.illinois.edu/software/doc/illinois-edison/apidocs/"
    - Run './extract_javadoc.sh' from the 'javadoc_scraper' directory
 
 2) Use the jdparser maven package to parser and generate the 'data.json' file that will be ingested and presented by the web interface.
    - Ensure that the base_url variable in the 'src/test/java/cz/zcu/validator/jdparser/JDParserTest.java' file contains the base url with the root package path of the LREC feature classes.
    - The current default is: "http://cogcomp.cs.illinois.edu/software/doc/apidocs/edu/illinois/cs/cogcomp/edison/features/"
+   - Ensure that the github_url_base_raw variable in the 'src/test/java/cz/zcu/validator/jdparser/JDParserTest.java' file contains the base raw github url from which we would append a "Test[FeatureExtractorName]" to download the Test file to be presented.
+   - The current default is: "https://raw.githubusercontent.com/IllinoisCogComp/illinois-cogcomp-nlp/master/edison/src/test/java/edu/illinois/cs/cogcomp/edison/features/lrec/" 
    - Run 'mvn test' from the 'javadoc_scraper' directory, and this will generate the 'data.json' file in the base directory of our web application
 
 3) Start the Web Application 
@@ -32,4 +34,8 @@ Steps to run the application:
  - jade
  - bootstrap
  - [jquery-searchable](https://github.com/stidges/jquery-searchable)
-
+
+##Troubleshooting
+ - 'Error: Cannot find module 'npmlog'. 
+    - This error usually occurs in outdated versions of npm (typically < 2.5.8) so be sure to run 'sudo npm install npm -g' to fix this issue.
+    - This error could also occur in some Mac installations and the best means to resolve the issue in that case is to a well-supported package installer such as 'homebrew' to handle your npm installation on Mac   

javadoc_scraper/robots.txt

@@ -0,0 +1 @@
+User-agent: *

javadoc_scraper/src/main/java/cz/zcu/validator/jdparser/api/JsoupJavaDocParser.java

@@ -109,17 +109,18 @@ public void setJavadoc_url(String javadoc_url) {
 	 */
 	private JavaDocsFactory javaDocFactory = null;
 
-    private String base_url, repo_path_string, path_to_write;
+    private String base_url, repo_path_string, base_github_url_raw, path_to_write;
 
 	//Old approach to extracting classes that we require.
 	//String[] fexes = {"LabelOneAfter", "LabelOneBefore", "LabelTwoAfter", "POSWindow"};
 
-    public JsoupJavaDocParser(String base_url, String repo_path_string, String path_to_write) {
+    public JsoupJavaDocParser(String base_url, String repo_path_string, String base_github_url_raw, String path_to_write) {
 
 		this.skeletonFactory = new SkeletonFactory();
 		this.javaDocFactory = new JavaDocsFactory();
 		this.base_url = base_url;
 		this.repo_path_string = repo_path_string;
+		this.base_github_url_raw = base_github_url_raw;
 		this.path_to_write = path_to_write;
 	}
 
@@ -202,13 +203,13 @@ public Collection<ClassRepresentation> parseDataFromJavaDoc(
 
 			ClassRepresentation a = (ClassRepresentation) reps[i];
 			String java_code = "";
-			String base_raw_url = "https://raw.githubusercontent.com/pauljv92/testedison/master/edison/src/test/java/edu/illinois/cs/cogcomp/edison/features/lrec/";
+			//String base_raw_url = "https://raw.githubusercontent.com/pauljv92/testedison/master/edison/src/test/java/edu/illinois/cs/cogcomp/edison/features/lrec/";
 
 			JavaDocClass jdoc = a.getJavaDocClass();
 
 			try {
 
-				   URL url = new URL(base_raw_url+"Test"+jdoc.getName()+".java");
+				   URL url = new URL(this.base_github_url_raw+"Test"+jdoc.getName()+".java");
 				   Scanner s = new Scanner(url.openStream());
 				   while(s.hasNext()) {
 					    // operate

javadoc_scraper/src/main/java/cz/zcu/validator/jdparser/api/Main.java

@@ -17,7 +17,7 @@ public class Main {
 	 */
 	public static void main(String[] args) {
 
-	    JsoupJavaDocParser parser = new JsoupJavaDocParser("None", "/test/","../data.json");
+	    JsoupJavaDocParser parser = new JsoupJavaDocParser("None", "/test/","../data.json", "http://github.com/test");
 
 		try {
 

javadoc_scraper/src/test/java/cz/zcu/validator/jdparser/JDParserTest.java

@@ -20,10 +20,13 @@ public void standardJavaDocDocumentsTest() {
     	String base_url = "http://cogcomp.cs.illinois.edu/software/doc/apidocs/edu/illinois/cs/cogcomp/edison/features/"; //This URL will be expected to the base URL of the public repository. We will extract the testcode to be presented on the web interface from here.
 
         String repository_path = "/lrec/"; //We will be using string to check for Classes whose path contains this substring. 
-
+
+	//Raw Github URL base for Tests
+	String github_url_base_raw = "https://raw.githubusercontent.com/IllinoisCogComp/illinois-cogcomp-nlp/master/edison/src/test/java/edu/illinois/cs/cogcomp/edison/features/lrec/";
+
 	String path_to_write_json = "../data.json";
 
-    	JsoupJavaDocParser parser = new JsoupJavaDocParser(base_url, repository_path, path_to_write_json);
+    	JsoupJavaDocParser parser = new JsoupJavaDocParser(base_url, repository_path, github_url_base_raw, path_to_write_json);
 
         try {
         	// parses javadoc directory of the Edison project