Updated readme and example.

Author: tijme

README.md

@@ -19,11 +19,9 @@ A very useful web crawler for vulnerability scanning. Not Your Average Web Crawl
 
 **Current limitations:**
 - Only works on Python 3.6 or higher.
-- Multiprocessing is not yet working.
-- Maximum recursion depth exception when crawling too much resources.
+- Ignore similar requests option not working.
 
 **Future development:**
-
 - Fix current limitations.
 - Performance improvements.
 - Support XHR crawling.
@@ -40,6 +38,7 @@ First make sure you're on [Python 3.6](https://www.python.org/) or higher. Then
 ## Example usage
 ```python
 from nyawc.Options import Options
+from nyawc.Queue import QueueItem
 from nyawc.Crawler import Crawler, CrawlerActions
 from nyawc.http.Request import Request
 
@@ -50,18 +49,19 @@ def cb_crawler_after_finish(queue):
     print("Crawler finished. Found " + str(queue.get_count()) + " requests.")
 
     for queue_item in queue.get_all():
-        print(queue_item.request.method + ": " + queue_item.request.url + " (" + str(queue_item.request.data) + ")")
+        print("[" + queue_item.request.method + "] " + queue_item.request.url + " (PostData: " + str(queue_item.request.data) + ")")
 
 def cb_request_before_start(queue, queue_item):
     # return CrawlerActions.DO_SKIP_TO_NEXT
     # return CrawlerActions.DO_STOP_CRAWLING
+
     return CrawlerActions.DO_CONTINUE_CRAWLING
 
 def cb_request_after_finish(queue, queue_item, new_queue_items):
     percentage = str(int(queue.get_progress()))
     total_requests = str(queue.get_count())
 
-    print("At " + percentage + "% of " + total_requests + " requests (" + queue_item.request.url + ").")
+    print("At " + percentage + "% of " + total_requests + " requests ([" + str(queue_item.response.status_code) + "] " + queue_item.request.url + ").")
 
     # return CrawlerActions.DO_STOP_CRAWLING
     return CrawlerActions.DO_CONTINUE_CRAWLING
@@ -77,13 +77,13 @@ options.callbacks.request_after_finish = cb_request_after_finish # Called after
 
 # Scope options
 options.scope.protocol_must_match = False # Only crawl pages with the same protocol as the startpoint (e.g. only https). Default is False.
-options.scope.subdomain_must_match = True # Only crawl pages with the same subdomain as the startpoint. If the startpoint is not a subdomain, no subdomains will be crawled. Default is True.
+options.scope.subdomain_must_match = False # Only crawl pages with the same subdomain as the startpoint. If the startpoint is not a subdomain, no subdomains will be crawled. Default is True.
 options.scope.domain_must_match = True # Only crawl pages with the same domain as the startpoint (e.g. only finnwea.com). Default is True.
 options.scope.ignore_similar_requests = True # Ignore similar requests like `?page=1` & `?page=2` or `/page/1` and `/page/2`. Default is True.
 options.scope.max_depth = None # The maximum search depth. For example, 2 would be the startpoint and all the pages found on it. Default is None (unlimited).
 
 # Performance options
-options.performance.max_processes = 8 # The maximum amount of simultaneous processes to use for crawling. Default is 8. 
+options.performance.max_threads = 8 # The maximum amount of simultaneous threads to use for crawling. Default is 4. 
 
 crawler = Crawler(options)
 crawler.start_with(Request("https://finnwea.com/"))

example.py

@@ -27,16 +27,14 @@
 from nyawc.Crawler import Crawler, CrawlerActions
 from nyawc.http.Request import Request
 
-import sys
-
 def cb_crawler_before_start():
     print("Crawler started.")
 
 def cb_crawler_after_finish(queue):
     print("Crawler finished. Found " + str(queue.get_count()) + " requests.")
 
     for queue_item in queue.get_all():
-        print(queue_item.request.method + ": " + queue_item.request.url + " (" + str(queue_item.request.data) + ")")
+        print("[" + queue_item.request.method + "] " + queue_item.request.url + " (PostData: " + str(queue_item.request.data) + ")")
 
 def cb_request_before_start(queue, queue_item):
     # return CrawlerActions.DO_SKIP_TO_NEXT
@@ -48,7 +46,7 @@ def cb_request_after_finish(queue, queue_item, new_queue_items):
     percentage = str(int(queue.get_progress()))
     total_requests = str(queue.get_count())
 
-    print("At " + percentage + "% of " + total_requests + " requests (" + str(queue_item.response.status_code) + "/" + queue_item.request.url + ").")
+    print("At " + percentage + "% of " + total_requests + " requests ([" + str(queue_item.response.status_code) + "] " + queue_item.request.url + ").")
 
     # return CrawlerActions.DO_STOP_CRAWLING
     return CrawlerActions.DO_CONTINUE_CRAWLING
@@ -70,7 +68,7 @@ def cb_request_after_finish(queue, queue_item, new_queue_items):
 options.scope.max_depth = None # The maximum search depth. For example, 2 would be the startpoint and all the pages found on it. Default is None (unlimited).
 
 # Performance options
-options.performance.max_threads = 16 # The maximum amount of simultaneous threads to use for crawling. Default is 4. 
+options.performance.max_threads = 8 # The maximum amount of simultaneous threads to use for crawling. Default is 4. 
 
 crawler = Crawler(options)
-crawler.start_with(Request("https://tweakers.net/"))
+crawler.start_with(Request("https://finnwea.com/"))

nyawc/Crawler.py

@@ -212,26 +212,40 @@ def __request_finish(self, queue_item, new_requests):
         if action == CrawlerActions.DO_CONTINUE_CRAWLING or action is None:
             return self.__spawn_new_requests()
 
+class CrawlerThread(threading.Thread):
+    """The crawler thread executes the HTTP request using the HTTP handler.
 
+    Attributes:
+        __callback (obj): The method to call when finished
+        __callback_lock (bool): The callback lock that prevents race conditions.
+        __queue_item (obj): The queue item containing a request to execute.
 
+    """
 
+    __callback = None
 
+    __callback_lock = None
 
-class CrawlerThread(threading.Thread):
+    __queue_item = None
 
-    __callback = None
+    def __init__(self, callback, callback_lock, queue_item):
+        """Constructs a crawler thread class
 
-    __queue_item = None
+        Args:
+            callback (obj): The method to call when finished
+            callback_lock (bool): The callback lock that prevents race conditions.
+            queue_item (obj): The queue item containing a request to execute.
 
-    __callback_lock = None
+        """
 
-    def __init__(self, callback, callback_lock, queue_item):
         threading.Thread.__init__(self)
         self.__callback = callback
         self.__queue_item = queue_item
         self.__callback_lock = callback_lock
 
     def run(self):
+        """Executes the HTTP call"""
+
         try:
             handler = Handler(self.__queue_item)
             self.__queue_item.response.raise_for_status()
@@ -247,12 +261,6 @@ def run(self):
         with self.__callback_lock:
             self.__callback(self.__queue_item, new_requests)
 
-
-
-
-
-
-
 class CrawlerActions:
     """The actions that crawler callbacks can return.