Merge pull request #36 from passivetotal/mock-requests

Mock requests & recent articles

Author: aeetos

CHANGELOG.md

@@ -1,5 +1,35 @@
 # Changelog
 
+## v2.5.3
+
+#### Enhancements
+
+- Better support for unit tests in client libraries with ability to set a 
+  session to override default request methods.
+- Add flexibility to library class instantiation to prefer keyword parameters
+  over config file keys. 
+- Support for new `create_date` Articles API data field and query parameter. Enables
+  searching for most recent articles instead of returning all of them at once, and
+  provides visiblity to situations where an article published in the past was recently
+  added to the Articles collection. 
+
+
+#### Breaking Changes
+
+- Previously, calls to `analyzer.AllArticles()` would return all articles without a date
+  limit. Now, it will return only articles created after the starting date set with
+  `analyzer.set_date_range()`. The current module-level default for all date-bounded queries
+  is 90 days back, so now this function will return all articles created in the last 90 days.
+- `age` property of an Article analyzer object is now based on `create_date` instead of publish
+  date.
+
+
+#### Bug Fixes
+
+[ none ]
+
+
+
 ## v2.5.2
 
 #### Enhancements

passivetotal/_version.py

@@ -1 +1 @@
-VERSION="2.5.2"
+VERSION="2.5.3"

passivetotal/analyzer/__init__.py

@@ -56,7 +56,7 @@ def init(**kwargs):
         if 'username' in kwargs and 'api_key' in kwargs:
             api_clients[name] = c(**kwargs)
         else:
-            api_clients[name] = c.from_config()
+            api_clients[name] = c.from_config(**kwargs)
         api_clients[name].exception_class = AnalyzerAPIError
         api_clients[name].set_context('python','passivetotal',VERSION,'analyzer')
     config['is_ready'] = True

passivetotal/analyzer/_common.py

@@ -429,7 +429,13 @@ def __init__(self, response):
             self.json = self.response.json()
         except Exception:
             self.json = {}
-        self.message = self.json.get('error', self.json.get('message', str(response)))
+        if self.json is None:
+            self.message = 'No JSON data in API response'
+        else:
+            try:
+                self.message = self.json.get('error', self.json.get('message', str(response)))
+            except Exception:
+                self.message = ''
 
     def __str__(self):
         return 'Error #{0.status_code} "{0.message}" ({0.url})'.format(self)

passivetotal/analyzer/articles.py

@@ -3,7 +3,7 @@
 from passivetotal.analyzer._common import (
     RecordList, Record, ForPandas
 )
-from passivetotal.analyzer import get_api
+from passivetotal.analyzer import get_api, get_config
 
 
 
@@ -71,20 +71,28 @@ class AllArticles(ArticlesList):
     By default, instantiating the class will automatically load the entire list
     of threat intelligence articles. Pass autoload=False to the constructor to disable
     this functionality.
+
+    Only articles created after the start date specified in the analyzer.set_date_range()
+    method will be returned unless a different created_after parameter is supplied to the object
+    constructor.
     """
 
-    def __init__(self, autoload = True):
+    def __init__(self, created_after=None, autoload=True):
         """Initialize a list of articles; will autoload by default.
-
         :param autoload: whether to automatically load articles upon instantiation (defaults to true)
         """
         super().__init__()
         if autoload:
-            self.load()
+            self.load(created_after)
 
-    def load(self):
-        """Query the API for articles and load them into an articles list."""
-        response = get_api('Articles').get_articles()
+    def load(self, created_after=None):
+        """Query the API for articles and load them into an articles list.
+        
+        :param created_after: only return articles created after this date (optional, defaults to date set by `analyzer.set_date_range()`
+        """
+        if created_after is None:
+            created_after = get_config('start_date')
+        response = get_api('Articles').get_articles(createdAfter=created_after)
         self.parse(response)
 
 
@@ -98,6 +106,7 @@ def __init__(self, api_response, query=None):
         self._summary = api_response.get('summary')
         self._type = api_response.get('type')
         self._publishdate = api_response.get('publishedDate')
+        self._createdate = api_response.get('createdDate')
         self._link = api_response.get('link')
         self._categories = api_response.get('categories')
         self._tags = api_response.get('tags')
@@ -115,13 +124,14 @@ def _api_get_details(self):
         response = get_api('Articles').get_details(self._guid)
         self._summary = response.get('summary')
         self._publishdate = response.get('publishedDate')
+        self._createdate = response.get('createdDate')
         self._tags = response.get('tags')
         self._categories = response.get('categories')
         self._indicators = response.get('indicators')
 
     def _get_dict_fields(self):
-        return ['guid','title','type','summary','str:date_published','age',
-                 'link','categories','tags','indicators','indicator_count',
+        return ['guid','title','type','summary','str:date_published','str:date_created',
+                 'age', 'link','categories','tags','indicators','indicator_count',
                  'indicator_types','str:ips','str:hostnames']
 
     def _ensure_details(self):
@@ -161,6 +171,7 @@ def to_dataframe(self, ensure_details=True, include_indicators=False):
             title           = self._title,
             type            = self._type,
             date_published  = self._publishdate,
+            date_created    = self._createdate,
             summary         = self._summary,
             link            = self._link,
             categories      = self._categories,
@@ -228,11 +239,18 @@ def date_published(self):
         date = datetime.fromisoformat(self._publishdate)
         return date
 
+    @property
+    def date_created(self):
+        """Date the article was created in the RiskIQ database."""
+        self._ensure_details()
+        date = datetime.fromisoformat(self._createdate)
+        return date
+    
     @property
     def age(self):
-        """Age of the article in days."""
+        """Age of the article in days, measured from create date."""
         now = datetime.now(timezone.utc)
-        interval = now - self.date_published
+        interval = now - self.date_created
         return interval.days
 
     @property

passivetotal/api.py

@@ -24,7 +24,8 @@ class Client(object):
 
     def __init__(self, username, api_key, server=DEFAULT_SERVER,
                  version=DEFAULT_VERSION, http_proxy=None, https_proxy=None,
-                 verify=True, headers=None, debug=False, exception_class=Exception):
+                 verify=True, headers=None, debug=False, exception_class=Exception,
+                 session=None):
         """Initial loading of the client.
 
         :param str username: API username in email address format
@@ -63,18 +64,25 @@ def __init__(self, username, api_key, server=DEFAULT_SERVER,
             self.verify = False
         self.exception_class = exception_class
         self.set_context('python','passivetotal',VERSION)
+        self.session = session or requests.Session()
 
     @classmethod
-    def from_config(cls):
-        """Method to return back a loaded instance."""
+    def from_config(cls, **kwargs):
+        """Method to return back a loaded instance.
+        
+        kwargs override configuration file variables if provided and are passed to the object constructor.
+        """
+        arg_keys = ['username','api_key','server','version','http_proxy','https_proxy']
+        args = { k: kwargs.pop(k) if k in kwargs else None for k in arg_keys }
         config = Config()
         client = cls(
-            username=config.get('username'),
-            api_key=config.get('api_key'),
-            server=config.get('api_server'),
-            version=config.get('api_version'),
-            http_proxy=config.get('http_proxy'),
-            https_proxy=config.get('https_proxy'),
+            username    = args.get('username') or config.get('username'),
+            api_key     = args.get('api_key') or config.get('api_key'),
+            server      = args.get('server') or config.get('api_server'),
+            version     = args.get('version') or config.get('api_version'),
+            http_proxy  = args.get('http_proxy') or config.get('http_proxy'),
+            https_proxy = args.get('https_proxy') or config.get('https_proxy'),
+            **kwargs
         )
         return client
 
@@ -155,7 +163,7 @@ def _get(self, endpoint, action, *url_args, **url_params):
         if self.proxies:
             kwargs['proxies'] = self.proxies
         self.logger.debug("Requesting: %s, %s" % (api_url, str(kwargs)))
-        response = requests.get(api_url, **kwargs)
+        response = self.session.get(api_url, **kwargs)
         return self._json(response)
 
     def _get_special(self, endpoint, action, trail, data, *url_args, **url_params):
@@ -175,7 +183,7 @@ def _get_special(self, endpoint, action, trail, data, *url_args, **url_params):
                   'auth': (self.username, self.api_key)}
         if self.proxies:
             kwargs['proxies'] = self.proxies
-        response = requests.get(api_url, **kwargs)
+        response = self.session.get(api_url, **kwargs)
         return self._json(response)
 
     def _send_data(self, method, endpoint, action,
@@ -196,7 +204,7 @@ def _send_data(self, method, endpoint, action,
                   'auth': (self.username, self.api_key)}
         if self.proxies:
             kwargs['proxies'] = self.proxies
-        response = requests.request(method, api_url, **kwargs)
+        response = self.session.request(method, api_url, **kwargs)
         return self._json(response)