Initial cut

Author: jayvdb

README.md

@@ -1,2 +1,34 @@
-# https-everywhere-py
-Privacy for Pythons
+# https-everywhere : Privacy for Pythons
+
+This project primarily provides requests adapters for Chrome [HSTS Preload](http://hstspreload.org/) and [HTTPS Everywhere](https://github.com/EFForg/https-everywhere) rulesets.
+
+At this stage, the focus is on correct efficient loading of the approx 25,000 rulesets of HTTPS Everywhere for use with any requests.
+Emphasis is on converting those rulesets to simpler or more common rules to reduce memory requirements.
+
+Current list of data problems can be found in `https_everywhere/_fixme.py`.  Many of these have patches sent upstream to the main HTTPS Everywhere project.
+
+## Testing
+
+To test
+
+```sh
+git clone https://github.com/jayvdb/https-everywhere-py
+git clone https://github.com/EFForg/https-everywhere
+cd https-everywhere-py
+tox
+```
+(Note: `test_rules` takes a long time to begin.)
+
+## Not implemented
+
+- custom local ruleset channels
+- cookie support
+- credentials in urls, such as `http://eff:techprojects@chart.googleapis.com/`,
+  which interfers with many rules, and also prevents exclusions from being applied
+- efficient memory structure for target mapping
+- rules with `@platform='mixedcontent'`; approx 800 rulesets ignored
+- rules with `@default_off`; approx 300 rulesets ignored, but all are mixedcontent
+- ruleset targets containing wildcards in the middle of the domain names (`foo.*.com`), which doesnt exist in the default channel
+- ruleset targets containing a wildcard at beginning and end (`*.foo.*`), which doesnt exist in the default channel
+- overlapping rules, which only applies to voxmedia.com in the default channel when filtered to exclude rules with `@default_off` and `@platform`.
+- rules for IPs; there are two 1.0.0.1 and 1.1.1.1 in the default channel. See https://en.wikipedia.org/wiki/1.1.1.1

https_everywhere/__init__.py

@@ -0,0 +1 @@
+"""HTTPS Everywhere."""

https_everywhere/_chrome_preload_hsts.py

@@ -0,0 +1,72 @@
+import json
+import logging
+import os.path
+
+import requests
+
+from logzero import setup_logger
+
+from ._fetch import _storage_location
+
+logger = setup_logger(name="httpseverwhere.preload", level=logging.INFO)
+
+_github_url = "https://raw.githubusercontent.com/chromium/chromium/master/net/http/transport_security_state_static.json"
+
+
+def _fetch_preload():
+    filename = _storage_location("transport_security_state_static.json")
+    if os.path.exists(filename):
+        return filename
+
+    r = requests.get(_github_url)
+
+    with open(filename, "w") as f:
+        f.write(r.text)
+
+    return filename
+
+
+def _load_preload_data(filename):
+    with open(filename) as f:
+        lines = [line for line in f.readlines() if not line.lstrip().startswith("/")]
+        raw = "\n".join(lines)
+        data = json.loads(raw)
+        return data
+
+
+def _preload_including_subdomains():
+    filename = _fetch_preload()
+    data = _load_preload_data(filename)
+    data = data["entries"]
+    domains = set()
+
+    for entry in data:
+        name = entry["name"]
+        if "." not in name:
+            continue
+        includeSubdomains = entry.get("include_subdomains")
+        if not includeSubdomains:
+            continue
+
+        pos = name.rfind(".", 1)
+        pos = name.rfind(".", 1, pos)
+        if pos != -1:
+            check_domain = name[pos + 1 :]
+            if check_domain in domains:
+                logger.debug(
+                    "{} base {} already in the list".format(name, check_domain)
+                )
+                continue
+
+        parts = name.split(".")
+        if name in ["stats.g.doubleclick.net"]:
+            continue
+        if parts[-2:] == ["google", "com"]:
+            continue
+        assert len(parts) < 5, "{} ({}) is very long and {} not found".format(
+            name, parts[-2:], check_domain
+        )
+
+        domains.add(name)
+
+    return domains

https_everywhere/_fetch.py

@@ -0,0 +1,68 @@
+import gzip
+import json
+import os
+import os.path
+
+import appdirs
+import requests
+
+base_url = "https://www.https-rulesets.org/v1/"
+_ts = None
+
+
+def _storage_location(filename=None, timestamp=None):
+    cache_dir = appdirs.user_cache_dir("https-everywhere-py")
+    os.makedirs(cache_dir, exist_ok=True)
+
+    if timestamp:
+        filename = "default.rulesets.{}".format(timestamp)
+
+    if filename:
+        return os.path.join(cache_dir, filename)
+
+    return cache_dir
+
+
+def fetch_channel_ts():
+    ruleset_ts_url = base_url + "latest-rulesets-timestamp"
+    r = requests.get(ruleset_ts_url)
+    r.raise_for_status()
+    ts_string = r.text
+    return int(ts_string)
+
+
+def _get_local_ts():
+    global _ts
+    if not _ts:  # pragma: no cover
+        _ts = fetch_channel_ts()
+    return _ts
+
+
+def _get_local(timestamp=None):
+    if not timestamp:
+        timestamp = _get_local_ts()  # pragma: no cover
+    location = _storage_location(timestamp=timestamp)
+    if os.path.exists(location):
+        with open(location) as f:
+            return json.load(f)
+
+
+def fetch_update(timestamp=None):
+    if not timestamp:  # pragma: no cover
+        timestamp = _get_local_ts()
+    data = _get_local(timestamp)
+    if data:
+        return data
+    ruleset_url = "{}rulesets-signature.{}.sha256".format(base_url, timestamp)
+    r = requests.get(ruleset_url)
+    r.raise_for_status()
+    ruleset_url = "{}default.rulesets.{}.gz".format(base_url, timestamp)
+    r = requests.get(
+        ruleset_url, headers={"Accept-Encoding": "gzip, deflate, br"}, stream=True
+    )
+    r.raise_for_status()
+    location = _storage_location(timestamp=timestamp)
+    data = gzip.GzipFile(fileobj=r.raw).read()
+    with open(location, "wb") as f:
+        f.write(data)
+    return json.loads(data)