Tidying up for release

- Improved URL controller. Now returns the original URL instead of "None"
- Added the URL expander to the twitter_follow_links source

Author: azazelm3dj3d

threatingestor/sources/twitter.py

@@ -83,7 +83,7 @@ def run(self, saved_state):
                 try:
                     tweet['content'] = tweet['content'].replace(url['url'], url['expanded_url'])
                 except KeyError:
-                    # Attempts to expand the URL using custom script
+                    # Attempts to expand the URL if not available through Twitter
                     tweet['content'] = tweet['content'].replace(url['url'], UrlController.expand_url(url['url']))
 
             # Process tweet.

threatingestor/sources/twitter_follow_links.py

@@ -1,18 +1,16 @@
 from __future__ import absolute_import
 
-
 import re
 import requests
 import twitter
 from loguru import logger
-from threatingestor.sources import Source
 
+from threatingestor.sources import Source
+from threatingestor.utils.url_controller import UrlController
 
 TWEET_URL = 'https://twitter.com/{user}/status/{id}'
-
 WHITELIST_DOMAINS = r"pastebin\.com"
 
-
 class Plugin(Source):
 
     def __init__(self, name, api_key, api_secret_key, access_token, access_token_secret, defanged_only=True, **kwargs):
@@ -97,8 +95,8 @@ def run(self, saved_state):
                         logger.log('NOTIFY', f"Discovered paste: {location}")
 
                 except KeyError:
-                    # No url/expanded_url, continue without expanding.
-                    pass
+                    # Attempts to expand the URL if not available through Twitter
+                    tweet['content'] = tweet['content'].replace(url['url'], UrlController.expand_url(url['url']))
 
         return saved_state, artifacts
 

threatingestor/utils/url_controller.py

@@ -11,7 +11,7 @@ def expand_url(url):
         """
         Expand ingested URLs with this method.
 
-        If a URL is unknown or cannot be expanded, you'll get a "None" response.
+        If a URL is unknown or cannot be expanded, you'll get the original URL back.
 
         @param: url (Example: https://inquest.net)
 
@@ -23,4 +23,4 @@ def expand_url(url):
             return str(expanded_link)
         # If unable to expand the URL, this exception is thrown
         except exceptions.ExpandingErrorException:
-            return "None"
+            return str(url)